From 19ee29401dfc414924f335c21ef9880513f3c58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Sat, 22 Aug 2015 21:06:12 +0200 Subject: t5004: test ZIP archives with many entries A ZIP file directory has a 16-bit field for the number of entries it contains. There are 64-bit extensions to deal with that. Demonstrate that git archive --format=zip currently doesn't use them and instead overflows the field. InfoZIP's unzip doesn't care about this field and extracts all files anyway. Software that uses the directory for presenting a filesystem like view quickly -- notably Windows -- depends on it, but doesn't lend itself to an automatic test case easily. Use InfoZIP's zipinfo, which probably isn't available everywhere but at least can provides *some* way to check this field. To speed things up a bit create and commit only a subset of the files and build a fake tree out of duplicates and pass that to git archive. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano diff --git a/t/t5004-archive-corner-cases.sh b/t/t5004-archive-corner-cases.sh index 654adda..c6bd729 100755 --- a/t/t5004-archive-corner-cases.sh +++ b/t/t5004-archive-corner-cases.sh @@ -115,4 +115,44 @@ test_expect_success 'archive empty subtree by direct pathspec' ' check_dir extract sub ' +ZIPINFO=zipinfo + +test_lazy_prereq ZIPINFO ' + n=$("$ZIPINFO" "$TEST_DIRECTORY"/t5004/empty.zip | sed -n "2s/.* //p") + test "x$n" = "x0" +' + +test_expect_failure ZIPINFO 'zip archive with many entries' ' + # add a directory with 256 files + mkdir 00 && + for a in 0 1 2 3 4 5 6 7 8 9 a b c d e f + do + for b in 0 1 2 3 4 5 6 7 8 9 a b c d e f + do + : >00/$a$b + done + done && + git add 00 && + git commit -m "256 files in 1 directory" && + + # duplicate it to get 65536 files in 256 directories + subtree=$(git write-tree --prefix=00/) && + for c in 0 1 2 3 4 5 6 7 8 9 a b c d e f + do + for d in 0 1 2 3 4 5 6 7 8 9 a b c d e f + do + echo "040000 tree $subtree $c$d" + done + done >tree && + tree=$(git mktree expect && + "$ZIPINFO" many.zip | head -2 | sed -n "2s/.* //p" >actual && + test_cmp expect actual +' + test_done -- cgit v0.10.2-6-g49f6 From 0f747f9d37e2b3d7a454029dfcf0065e6914e995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Sat, 22 Aug 2015 21:06:31 +0200 Subject: archive-zip: use a local variable to store the creator version Use a simpler conditional right next to the code which makes a higher creator version necessary -- namely symlink handling and support for executable files -- instead of a long line with a ternary operator. The resulting code has more lines but is simpler and allows reuse of the value easily. Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano diff --git a/archive-zip.c b/archive-zip.c index ffb3535..df004de 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -223,6 +223,7 @@ static int write_zip_entry(struct archiver_args *args, unsigned long size; int is_binary = -1; const char *path_without_prefix = path + args->baselen; + unsigned int creator_version = 0; crc = crc32(0, NULL, 0); @@ -251,6 +252,8 @@ static int write_zip_entry(struct archiver_args *args, method = 0; attr2 = S_ISLNK(mode) ? ((mode | 0777) << 16) : (mode & 0111) ? ((mode) << 16) : 0; + if (S_ISLNK(mode) || (mode & 0111)) + creator_version = 0x0317; if (S_ISREG(mode) && args->compression_level != 0 && size > 0) method = 8; @@ -303,8 +306,7 @@ static int write_zip_entry(struct archiver_args *args, } copy_le32(dirent.magic, 0x02014b50); - copy_le16(dirent.creator_version, - S_ISLNK(mode) || (S_ISREG(mode) && (mode & 0111)) ? 0x0317 : 0); + copy_le16(dirent.creator_version, creator_version); copy_le16(dirent.version, 10); copy_le16(dirent.flags, flags); copy_le16(dirent.compression_method, method); -- cgit v0.10.2-6-g49f6 From 88329ca8091b16f4bb01e3ea546c8c5e1a5016ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Sat, 22 Aug 2015 21:06:45 +0200 Subject: archive-zip: support more than 65535 entries Support more than 65535 entries cleanly by writing a "zip64 end of central directory record" (with a 64-bit field for the number of entries) before the usual "end of central directory record" (which contains only a 16-bit field). InfoZIP's zip does the same. Archives with 65535 or less entries are not affected. Programs that extract all files like InfoZIP's zip and 7-Zip ignored the field and could extract all files already. Software that relies on the ZIP file directory to show a list of contained files quickly to simulate to normal directory like Windows' built-in ZIP functionality only saw a subset of the included files. Windows supports ZIP64 since Vista according to https://en.wikipedia.org/wiki/Zip_%28file_format%29#ZIP64. Suggested-by: Johannes Schauer Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano diff --git a/archive-zip.c b/archive-zip.c index df004de..480360d 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -16,7 +16,9 @@ static unsigned int zip_dir_size; static unsigned int zip_offset; static unsigned int zip_dir_offset; -static unsigned int zip_dir_entries; +static uint64_t zip_dir_entries; + +static unsigned int max_creator_version; #define ZIP_DIRECTORY_MIN_SIZE (1024 * 1024) #define ZIP_STREAM (1 << 3) @@ -86,6 +88,28 @@ struct zip_extra_mtime { unsigned char _end[1]; }; +struct zip64_dir_trailer { + unsigned char magic[4]; + unsigned char record_size[8]; + unsigned char creator_version[2]; + unsigned char version[2]; + unsigned char disk[4]; + unsigned char directory_start_disk[4]; + unsigned char entries_on_this_disk[8]; + unsigned char entries[8]; + unsigned char size[8]; + unsigned char offset[8]; + unsigned char _end[1]; +}; + +struct zip64_dir_trailer_locator { + unsigned char magic[4]; + unsigned char disk[4]; + unsigned char offset[8]; + unsigned char number_of_disks[4]; + unsigned char _end[1]; +}; + /* * On ARM, padding is added at the end of the struct, so a simple * sizeof(struct ...) reports two bytes more than the payload size @@ -98,6 +122,12 @@ struct zip_extra_mtime { #define ZIP_EXTRA_MTIME_SIZE offsetof(struct zip_extra_mtime, _end) #define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \ (ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags)) +#define ZIP64_DIR_TRAILER_SIZE offsetof(struct zip64_dir_trailer, _end) +#define ZIP64_DIR_TRAILER_RECORD_SIZE \ + (ZIP64_DIR_TRAILER_SIZE - \ + offsetof(struct zip64_dir_trailer, creator_version)) +#define ZIP64_DIR_TRAILER_LOCATOR_SIZE \ + offsetof(struct zip64_dir_trailer_locator, _end) static void copy_le16(unsigned char *dest, unsigned int n) { @@ -113,6 +143,31 @@ static void copy_le32(unsigned char *dest, unsigned int n) dest[3] = 0xff & (n >> 030); } +static void copy_le64(unsigned char *dest, uint64_t n) +{ + dest[0] = 0xff & n; + dest[1] = 0xff & (n >> 010); + dest[2] = 0xff & (n >> 020); + dest[3] = 0xff & (n >> 030); + dest[4] = 0xff & (n >> 040); + dest[5] = 0xff & (n >> 050); + dest[6] = 0xff & (n >> 060); + dest[7] = 0xff & (n >> 070); +} + +static uint64_t clamp_max(uint64_t n, uint64_t max, int *clamped) +{ + if (n <= max) + return n; + *clamped = 1; + return max; +} + +static void copy_le16_clamp(unsigned char *dest, uint64_t n, int *clamped) +{ + copy_le16(dest, clamp_max(n, 0xffff, clamped)); +} + static void *zlib_deflate_raw(void *data, unsigned long size, int compression_level, unsigned long *compressed_size) @@ -282,6 +337,9 @@ static int write_zip_entry(struct archiver_args *args, sha1_to_hex(sha1)); } + if (creator_version > max_creator_version) + max_creator_version = creator_version; + if (buffer && method == 8) { out = deflated = zlib_deflate_raw(buffer, size, args->compression_level, @@ -439,20 +497,49 @@ static int write_zip_entry(struct archiver_args *args, return 0; } +static void write_zip64_trailer(void) +{ + struct zip64_dir_trailer trailer64; + struct zip64_dir_trailer_locator locator64; + + copy_le32(trailer64.magic, 0x06064b50); + copy_le64(trailer64.record_size, ZIP64_DIR_TRAILER_RECORD_SIZE); + copy_le16(trailer64.creator_version, max_creator_version); + copy_le16(trailer64.version, 45); + copy_le32(trailer64.disk, 0); + copy_le32(trailer64.directory_start_disk, 0); + copy_le64(trailer64.entries_on_this_disk, zip_dir_entries); + copy_le64(trailer64.entries, zip_dir_entries); + copy_le64(trailer64.size, zip_dir_offset); + copy_le64(trailer64.offset, zip_offset); + + copy_le32(locator64.magic, 0x07064b50); + copy_le32(locator64.disk, 0); + copy_le64(locator64.offset, zip_offset + zip_dir_offset); + copy_le32(locator64.number_of_disks, 1); + + write_or_die(1, &trailer64, ZIP64_DIR_TRAILER_SIZE); + write_or_die(1, &locator64, ZIP64_DIR_TRAILER_LOCATOR_SIZE); +} + static void write_zip_trailer(const unsigned char *sha1) { struct zip_dir_trailer trailer; + int clamped = 0; copy_le32(trailer.magic, 0x06054b50); copy_le16(trailer.disk, 0); copy_le16(trailer.directory_start_disk, 0); - copy_le16(trailer.entries_on_this_disk, zip_dir_entries); - copy_le16(trailer.entries, zip_dir_entries); + copy_le16_clamp(trailer.entries_on_this_disk, zip_dir_entries, + &clamped); + copy_le16_clamp(trailer.entries, zip_dir_entries, &clamped); copy_le32(trailer.size, zip_dir_offset); copy_le32(trailer.offset, zip_offset); copy_le16(trailer.comment_length, sha1 ? 40 : 0); write_or_die(1, zip_dir, zip_dir_offset); + if (clamped) + write_zip64_trailer(); write_or_die(1, &trailer, ZIP_DIR_TRAILER_SIZE); if (sha1) write_or_die(1, sha1_to_hex(sha1), 40); diff --git a/t/t5004-archive-corner-cases.sh b/t/t5004-archive-corner-cases.sh index c6bd729..cca2338 100755 --- a/t/t5004-archive-corner-cases.sh +++ b/t/t5004-archive-corner-cases.sh @@ -122,7 +122,7 @@ test_lazy_prereq ZIPINFO ' test "x$n" = "x0" ' -test_expect_failure ZIPINFO 'zip archive with many entries' ' +test_expect_success ZIPINFO 'zip archive with many entries' ' # add a directory with 256 files mkdir 00 && for a in 0 1 2 3 4 5 6 7 8 9 a b c d e f -- cgit v0.10.2-6-g49f6