path: root/utf8.c
diff options
authorLars Schneider <>2018-04-15 18:16:06 (GMT)
committerJunio C Hamano <>2018-04-16 02:40:56 (GMT)
commitc6e48652f69f6955bbbb423100e0df2a49467db8 (patch)
tree5cd147a495304941965ac2f6c2af74ec233ecd2c /utf8.c
parent10ecb82e4f1f507d5f122e00fd4829b30953f853 (diff)
utf8: add function to detect a missing UTF-16/32 BOM
If the endianness is not defined in the encoding name, then let's be strict and require a BOM to avoid any encoding confusion. The is_missing_required_utf_bom() function returns true if a required BOM is missing. The Unicode standard instructs to assume big-endian if there in no BOM for UTF-16/32 [1][2]. However, the W3C/WHATWG encoding standard used in HTML5 recommends to assume little-endian to "deal with deployed content" [3]. Strictly requiring a BOM seems to be the safest option for content in Git. This function is used in a subsequent commit. [1] [2] Section 3.10, D98, page 132 [3] Signed-off-by: Lars Schneider <> Signed-off-by: Junio C Hamano <>
Diffstat (limited to 'utf8.c')
1 files changed, 13 insertions, 0 deletions
diff --git a/utf8.c b/utf8.c
index 83af3a9..25d366d 100644
--- a/utf8.c
+++ b/utf8.c
@@ -586,6 +586,19 @@ int has_prohibited_utf_bom(const char *enc, const char *data, size_t len)
+int is_missing_required_utf_bom(const char *enc, const char *data, size_t len)
+ return (
+ (same_utf_encoding(enc, "UTF-16")) &&
+ !(has_bom_prefix(data, len, utf16_be_bom, sizeof(utf16_be_bom)) ||
+ has_bom_prefix(data, len, utf16_le_bom, sizeof(utf16_le_bom)))
+ ) || (
+ (same_utf_encoding(enc, "UTF-32")) &&
+ !(has_bom_prefix(data, len, utf32_be_bom, sizeof(utf32_be_bom)) ||
+ has_bom_prefix(data, len, utf32_le_bom, sizeof(utf32_le_bom)))
+ );
* Returns first character length in bytes for multi-byte `text` according to
* `encoding`.