diff options
| -rw-r--r-- | Documentation/gitattributes.txt | 4 | ||||
| -rw-r--r-- | compat/precompose_utf8.c | 2 | ||||
| -rwxr-xr-x | t/t0028-working-tree-encoding.sh | 12 | ||||
| -rw-r--r-- | utf8.c | 42 | ||||
| -rw-r--r-- | utf8.h | 2 |
5 files changed, 48 insertions, 14 deletions
diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index b8392fc..a2310fb 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -344,7 +344,9 @@ automatic line ending conversion based on your platform. Use the following attributes if your '*.ps1' files are UTF-16 little endian encoded without BOM and you want Git to use Windows line endings -in the working directory. Please note, it is highly recommended to +in the working directory (use `UTF-16-LE-BOM` instead of `UTF-16LE` if +you want UTF-16 little endian with BOM). +Please note, it is highly recommended to explicitly define the line endings with `eol` if the `working-tree-encoding` attribute is used to avoid ambiguity. diff --git a/compat/precompose_utf8.c b/compat/precompose_utf8.c index de61c15..136250f 100644 --- a/compat/precompose_utf8.c +++ b/compat/precompose_utf8.c @@ -79,7 +79,7 @@ void precompose_argv(int argc, const char **argv) size_t namelen; oldarg = argv[i]; if (has_non_ascii(oldarg, (size_t)-1, &namelen)) { - newarg = reencode_string_iconv(oldarg, namelen, ic_precompose, NULL); + newarg = reencode_string_iconv(oldarg, namelen, ic_precompose, 0, NULL); if (newarg) argv[i] = newarg; } diff --git a/t/t0028-working-tree-encoding.sh b/t/t0028-working-tree-encoding.sh index 7e87b5a..e58ecbf 100755 --- a/t/t0028-working-tree-encoding.sh +++ b/t/t0028-working-tree-encoding.sh @@ -11,9 +11,12 @@ test_expect_success 'setup test files' ' text="hallo there!\ncan you read me?" && echo "*.utf16 text working-tree-encoding=utf-16" >.gitattributes && + echo "*.utf16lebom text working-tree-encoding=UTF-16LE-BOM" >>.gitattributes && printf "$text" >test.utf8.raw && printf "$text" | iconv -f UTF-8 -t UTF-16 >test.utf16.raw && printf "$text" | iconv -f UTF-8 -t UTF-32 >test.utf32.raw && + printf "\377\376" >test.utf16lebom.raw && + printf "$text" | iconv -f UTF-8 -t UTF-32LE >>test.utf16lebom.raw && # Line ending tests printf "one\ntwo\nthree\n" >lf.utf8.raw && @@ -32,7 +35,8 @@ test_expect_success 'setup test files' ' # Add only UTF-16 file, we will add the UTF-32 file later cp test.utf16.raw test.utf16 && cp test.utf32.raw test.utf32 && - git add .gitattributes test.utf16 && + cp test.utf16lebom.raw test.utf16lebom && + git add .gitattributes test.utf16 test.utf16lebom && git commit -m initial ' @@ -51,6 +55,12 @@ test_expect_success 're-encode to UTF-16 on checkout' ' test_cmp_bin test.utf16.raw test.utf16 ' +test_expect_success 're-encode to UTF-16-LE-BOM on checkout' ' + rm test.utf16lebom && + git checkout test.utf16lebom && + test_cmp_bin test.utf16lebom.raw test.utf16lebom +' + test_expect_success 'check $GIT_DIR/info/attributes support' ' test_when_finished "rm -f test.utf32.git" && test_when_finished "git reset --hard HEAD" && @@ -4,6 +4,11 @@ /* This code is originally from http://www.cl.cam.ac.uk/~mgk25/ucs/ */ +static const char utf16_be_bom[] = {'\xFE', '\xFF'}; +static const char utf16_le_bom[] = {'\xFF', '\xFE'}; +static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'}; +static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'}; + struct interval { ucs_char_t first; ucs_char_t last; @@ -470,16 +475,17 @@ int utf8_fprintf(FILE *stream, const char *format, ...) #else typedef char * iconv_ibp; #endif -char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv, size_t *outsz_p) +char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv, + size_t bom_len, size_t *outsz_p) { size_t outsz, outalloc; char *out, *outpos; iconv_ibp cp; outsz = insz; - outalloc = st_add(outsz, 1); /* for terminating NUL */ + outalloc = st_add(outsz, 1 + bom_len); /* for terminating NUL */ out = xmalloc(outalloc); - outpos = out; + outpos = out + bom_len; cp = (iconv_ibp)in; while (1) { @@ -540,10 +546,30 @@ char *reencode_string_len(const char *in, size_t insz, { iconv_t conv; char *out; + const char *bom_str = NULL; + size_t bom_len = 0; if (!in_encoding) return NULL; + /* UTF-16LE-BOM is the same as UTF-16 for reading */ + if (same_utf_encoding("UTF-16LE-BOM", in_encoding)) + in_encoding = "UTF-16"; + + /* + * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM" + * Some users under Windows want the little endian version + */ + if (same_utf_encoding("UTF-16LE-BOM", out_encoding)) { + bom_str = utf16_le_bom; + bom_len = sizeof(utf16_le_bom); + out_encoding = "UTF-16LE"; + } else if (same_utf_encoding("UTF-16BE-BOM", out_encoding)) { + bom_str = utf16_be_bom; + bom_len = sizeof(utf16_be_bom); + out_encoding = "UTF-16BE"; + } + conv = iconv_open(out_encoding, in_encoding); if (conv == (iconv_t) -1) { in_encoding = fallback_encoding(in_encoding); @@ -553,9 +579,10 @@ char *reencode_string_len(const char *in, size_t insz, if (conv == (iconv_t) -1) return NULL; } - - out = reencode_string_iconv(in, insz, conv, outsz); + out = reencode_string_iconv(in, insz, conv, bom_len, outsz); iconv_close(conv); + if (out && bom_str && bom_len) + memcpy(out, bom_str, bom_len); return out; } #endif @@ -566,11 +593,6 @@ static int has_bom_prefix(const char *data, size_t len, return data && bom && (len >= bom_len) && !memcmp(data, bom, bom_len); } -static const char utf16_be_bom[] = {'\xFE', '\xFF'}; -static const char utf16_le_bom[] = {'\xFF', '\xFE'}; -static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'}; -static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'}; - int has_prohibited_utf_bom(const char *enc, const char *data, size_t len) { return ( @@ -27,7 +27,7 @@ void strbuf_utf8_replace(struct strbuf *sb, int pos, int width, #ifndef NO_ICONV char *reencode_string_iconv(const char *in, size_t insz, - iconv_t conv, size_t *outsz); + iconv_t conv, size_t bom_len, size_t *outsz); char *reencode_string_len(const char *in, size_t insz, const char *out_encoding, const char *in_encoding, |