summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJunio C Hamano <gitster@pobox.com>2015-05-26 20:24:45 (GMT)
committerJunio C Hamano <gitster@pobox.com>2015-05-26 20:24:46 (GMT)
commit38ccaf93bbf5a99dbff908068292ffaa5bafe25e (patch)
treecd1b379e0ebd26c6b5576e023c67338c41462d4e
parenta26d48a46e8d7df58e7a91e7aac6795e29ddbd6d (diff)
parentaeb6f8b3a2bbfd8b48a967139fbf4581e5345182 (diff)
downloadgit-38ccaf93bbf5a99dbff908068292ffaa5bafe25e.zip
git-38ccaf93bbf5a99dbff908068292ffaa5bafe25e.tar.gz
git-38ccaf93bbf5a99dbff908068292ffaa5bafe25e.tar.bz2
Merge branch 'nd/untracked-cache'
Teach the index to optionally remember already seen untracked files to speed up "git status" in a working tree with tons of cruft. * nd/untracked-cache: (24 commits) git-status.txt: advertisement for untracked cache untracked cache: guard and disable on system changes mingw32: add uname() t7063: tests for untracked cache update-index: test the system before enabling untracked cache update-index: manually enable or disable untracked cache status: enable untracked cache untracked-cache: temporarily disable with $GIT_DISABLE_UNTRACKED_CACHE untracked cache: mark index dirty if untracked cache is updated untracked cache: print stats with $GIT_TRACE_UNTRACKED_STATS untracked cache: avoid racy timestamps read-cache.c: split racy stat test to a separate function untracked cache: invalidate at index addition or removal untracked cache: load from UNTR index extension untracked cache: save to an index extension ewah: add convenient wrapper ewah_serialize_strbuf() untracked cache: don't open non-existent .gitignore untracked cache: mark what dirs should be recursed/saved untracked cache: record/validate dir mtime and reuse cached output untracked cache: make a wrapper around {open,read,close}dir() ...
-rw-r--r--.gitignore1
-rw-r--r--Documentation/git-status.txt5
-rw-r--r--Documentation/git-update-index.txt14
-rw-r--r--Documentation/technical/index-format.txt62
-rw-r--r--Makefile1
-rw-r--r--builtin/commit.c5
-rw-r--r--builtin/update-index.c188
-rw-r--r--cache.h6
-rw-r--r--compat/mingw.c11
-rw-r--r--compat/mingw.h9
-rw-r--r--dir.c990
-rw-r--r--dir.h82
-rw-r--r--ewah/ewah_io.c13
-rw-r--r--ewah/ewok.h2
-rw-r--r--git-compat-util.h1
-rw-r--r--read-cache.c55
-rw-r--r--split-index.c11
-rwxr-xr-xt/t7063-status-untracked-cache.sh353
-rw-r--r--test-dump-untracked-cache.c62
-rw-r--r--unpack-trees.c7
-rw-r--r--wt-status.c2
21 files changed, 1822 insertions, 58 deletions
diff --git a/.gitignore b/.gitignore
index a052419..422c538 100644
--- a/.gitignore
+++ b/.gitignore
@@ -184,6 +184,7 @@
/test-delta
/test-dump-cache-tree
/test-dump-split-index
+/test-dump-untracked-cache
/test-scrap-cache-tree
/test-genrandom
/test-hashmap
diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt
index 5221f95..335f312 100644
--- a/Documentation/git-status.txt
+++ b/Documentation/git-status.txt
@@ -66,7 +66,10 @@ When `-u` option is not used, untracked files and directories are
shown (i.e. the same as specifying `normal`), to help you avoid
forgetting to add newly created files. Because it takes extra work
to find untracked files in the filesystem, this mode may take some
-time in a large working tree. You can use `no` to have `git status`
+time in a large working tree.
+Consider enabling untracked cache and split index if supported (see
+`git update-index --untracked-cache` and `git update-index
+--split-index`), Otherwise you can use `no` to have `git status`
return more quickly without showing untracked files.
+
The default can be changed using the status.showUntrackedFiles
diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt
index aff0179..1a296bc 100644
--- a/Documentation/git-update-index.txt
+++ b/Documentation/git-update-index.txt
@@ -170,6 +170,20 @@ may not support it yet.
the shared index file. This mode is designed for very large
indexes that take a significant amount of time to read or write.
+--untracked-cache::
+--no-untracked-cache::
+ Enable or disable untracked cache extension. This could speed
+ up for commands that involve determining untracked files such
+ as `git status`. The underlying operating system and file
+ system must change `st_mtime` field of a directory if files
+ are added or deleted in that directory.
+
+--force-untracked-cache::
+ For safety, `--untracked-cache` performs tests on the working
+ directory to make sure untracked cache can be used. These
+ tests can take a few seconds. `--force-untracked-cache` can be
+ used to skip the tests.
+
\--::
Do not interpret any more arguments as options.
diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt
index 35112e4..b7093af 100644
--- a/Documentation/technical/index-format.txt
+++ b/Documentation/technical/index-format.txt
@@ -233,3 +233,65 @@ Git index format
The remaining index entries after replaced ones will be added to the
final index. These added entries are also sorted by entry name then
stage.
+
+== Untracked cache
+
+ Untracked cache saves the untracked file list and necessary data to
+ verify the cache. The signature for this extension is { 'U', 'N',
+ 'T', 'R' }.
+
+ The extension starts with
+
+ - A sequence of NUL-terminated strings, preceded by the size of the
+ sequence in variable width encoding. Each string describes the
+ environment where the cache can be used.
+
+ - Stat data of $GIT_DIR/info/exclude. See "Index entry" section from
+ ctime field until "file size".
+
+ - Stat data of core.excludesfile
+
+ - 32-bit dir_flags (see struct dir_struct)
+
+ - 160-bit SHA-1 of $GIT_DIR/info/exclude. Null SHA-1 means the file
+ does not exist.
+
+ - 160-bit SHA-1 of core.excludesfile. Null SHA-1 means the file does
+ not exist.
+
+ - NUL-terminated string of per-dir exclude file name. This usually
+ is ".gitignore".
+
+ - The number of following directory blocks, variable width
+ encoding. If this number is zero, the extension ends here with a
+ following NUL.
+
+ - A number of directory blocks in depth-first-search order, each
+ consists of
+
+ - The number of untracked entries, variable width encoding.
+
+ - The number of sub-directory blocks, variable width encoding.
+
+ - The directory name terminated by NUL.
+
+ - A number of untrached file/dir names terminated by NUL.
+
+The remaining data of each directory block is grouped by type:
+
+ - An ewah bitmap, the n-th bit marks whether the n-th directory has
+ valid untracked cache entries.
+
+ - An ewah bitmap, the n-th bit records "check-only" bit of
+ read_directory_recursive() for the n-th directory.
+
+ - An ewah bitmap, the n-th bit indicates whether SHA-1 and stat data
+ is valid for the n-th directory and exists in the next data.
+
+ - An array of stat data. The n-th data corresponds with the n-th
+ "one" bit in the previous ewah bitmap.
+
+ - An array of SHA-1. The n-th SHA-1 corresponds with the n-th "one" bit
+ in the previous ewah bitmap.
+
+ - One NUL.
diff --git a/Makefile b/Makefile
index 25a453b..323c401 100644
--- a/Makefile
+++ b/Makefile
@@ -574,6 +574,7 @@ TEST_PROGRAMS_NEED_X += test-date
TEST_PROGRAMS_NEED_X += test-delta
TEST_PROGRAMS_NEED_X += test-dump-cache-tree
TEST_PROGRAMS_NEED_X += test-dump-split-index
+TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
TEST_PROGRAMS_NEED_X += test-genrandom
TEST_PROGRAMS_NEED_X += test-hashmap
TEST_PROGRAMS_NEED_X += test-index-version
diff --git a/builtin/commit.c b/builtin/commit.c
index d6515a2..254477f 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -1366,13 +1366,14 @@ int cmd_status(int argc, const char **argv, const char *prefix)
refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL);
fd = hold_locked_index(&index_lock, 0);
- if (0 <= fd)
- update_index_if_able(&the_index, &index_lock);
s.is_initial = get_sha1(s.reference, sha1) ? 1 : 0;
s.ignore_submodule_arg = ignore_submodule_arg;
wt_status_collect(&s);
+ if (0 <= fd)
+ update_index_if_able(&the_index, &index_lock);
+
if (s.relative_paths)
s.prefix = prefix;
diff --git a/builtin/update-index.c b/builtin/update-index.c
index 0665b31..7431938 100644
--- a/builtin/update-index.c
+++ b/builtin/update-index.c
@@ -33,6 +33,7 @@ static int mark_valid_only;
static int mark_skip_worktree_only;
#define MARK_FLAG 1
#define UNMARK_FLAG 2
+static struct strbuf mtime_dir = STRBUF_INIT;
__attribute__((format (printf, 1, 2)))
static void report(const char *fmt, ...)
@@ -48,6 +49,166 @@ static void report(const char *fmt, ...)
va_end(vp);
}
+static void remove_test_directory(void)
+{
+ if (mtime_dir.len)
+ remove_dir_recursively(&mtime_dir, 0);
+}
+
+static const char *get_mtime_path(const char *path)
+{
+ static struct strbuf sb = STRBUF_INIT;
+ strbuf_reset(&sb);
+ strbuf_addf(&sb, "%s/%s", mtime_dir.buf, path);
+ return sb.buf;
+}
+
+static void xmkdir(const char *path)
+{
+ path = get_mtime_path(path);
+ if (mkdir(path, 0700))
+ die_errno(_("failed to create directory %s"), path);
+}
+
+static int xstat_mtime_dir(struct stat *st)
+{
+ if (stat(mtime_dir.buf, st))
+ die_errno(_("failed to stat %s"), mtime_dir.buf);
+ return 0;
+}
+
+static int create_file(const char *path)
+{
+ int fd;
+ path = get_mtime_path(path);
+ fd = open(path, O_CREAT | O_RDWR, 0644);
+ if (fd < 0)
+ die_errno(_("failed to create file %s"), path);
+ return fd;
+}
+
+static void xunlink(const char *path)
+{
+ path = get_mtime_path(path);
+ if (unlink(path))
+ die_errno(_("failed to delete file %s"), path);
+}
+
+static void xrmdir(const char *path)
+{
+ path = get_mtime_path(path);
+ if (rmdir(path))
+ die_errno(_("failed to delete directory %s"), path);
+}
+
+static void avoid_racy(void)
+{
+ /*
+ * not use if we could usleep(10) if USE_NSEC is defined. The
+ * field nsec could be there, but the OS could choose to
+ * ignore it?
+ */
+ sleep(1);
+}
+
+static int test_if_untracked_cache_is_supported(void)
+{
+ struct stat st;
+ struct stat_data base;
+ int fd, ret = 0;
+
+ strbuf_addstr(&mtime_dir, "mtime-test-XXXXXX");
+ if (!mkdtemp(mtime_dir.buf))
+ die_errno("Could not make temporary directory");
+
+ fprintf(stderr, _("Testing "));
+ atexit(remove_test_directory);
+ xstat_mtime_dir(&st);
+ fill_stat_data(&base, &st);
+ fputc('.', stderr);
+
+ avoid_racy();
+ fd = create_file("newfile");
+ xstat_mtime_dir(&st);
+ if (!match_stat_data(&base, &st)) {
+ close(fd);
+ fputc('\n', stderr);
+ fprintf_ln(stderr,_("directory stat info does not "
+ "change after adding a new file"));
+ goto done;
+ }
+ fill_stat_data(&base, &st);
+ fputc('.', stderr);
+
+ avoid_racy();
+ xmkdir("new-dir");
+ xstat_mtime_dir(&st);
+ if (!match_stat_data(&base, &st)) {
+ close(fd);
+ fputc('\n', stderr);
+ fprintf_ln(stderr, _("directory stat info does not change "
+ "after adding a new directory"));
+ goto done;
+ }
+ fill_stat_data(&base, &st);
+ fputc('.', stderr);
+
+ avoid_racy();
+ write_or_die(fd, "data", 4);
+ close(fd);
+ xstat_mtime_dir(&st);
+ if (match_stat_data(&base, &st)) {
+ fputc('\n', stderr);
+ fprintf_ln(stderr, _("directory stat info changes "
+ "after updating a file"));
+ goto done;
+ }
+ fputc('.', stderr);
+
+ avoid_racy();
+ close(create_file("new-dir/new"));
+ xstat_mtime_dir(&st);
+ if (match_stat_data(&base, &st)) {
+ fputc('\n', stderr);
+ fprintf_ln(stderr, _("directory stat info changes after "
+ "adding a file inside subdirectory"));
+ goto done;
+ }
+ fputc('.', stderr);
+
+ avoid_racy();
+ xunlink("newfile");
+ xstat_mtime_dir(&st);
+ if (!match_stat_data(&base, &st)) {
+ fputc('\n', stderr);
+ fprintf_ln(stderr, _("directory stat info does not "
+ "change after deleting a file"));
+ goto done;
+ }
+ fill_stat_data(&base, &st);
+ fputc('.', stderr);
+
+ avoid_racy();
+ xunlink("new-dir/new");
+ xrmdir("new-dir");
+ xstat_mtime_dir(&st);
+ if (!match_stat_data(&base, &st)) {
+ fputc('\n', stderr);
+ fprintf_ln(stderr, _("directory stat info does not "
+ "change after deleting a directory"));
+ goto done;
+ }
+
+ if (rmdir(mtime_dir.buf))
+ die_errno(_("failed to delete directory %s"), mtime_dir.buf);
+ fprintf_ln(stderr, _(" OK"));
+ ret = 1;
+
+done:
+ strbuf_release(&mtime_dir);
+ return ret;
+}
+
static int mark_ce_flags(const char *path, int flag, int mark)
{
int namelen = strlen(path);
@@ -741,6 +902,7 @@ static int reupdate_callback(struct parse_opt_ctx_t *ctx,
int cmd_update_index(int argc, const char **argv, const char *prefix)
{
int newfd, entries, has_errors = 0, line_termination = '\n';
+ int untracked_cache = -1;
int read_from_stdin = 0;
int prefix_length = prefix ? strlen(prefix) : 0;
int preferred_index_format = 0;
@@ -832,6 +994,10 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
N_("write index in this format")),
OPT_BOOL(0, "split-index", &split_index,
N_("enable or disable split index")),
+ OPT_BOOL(0, "untracked-cache", &untracked_cache,
+ N_("enable/disable untracked cache")),
+ OPT_SET_INT(0, "force-untracked-cache", &untracked_cache,
+ N_("enable untracked cache without testing the filesystem"), 2),
OPT_END()
};
@@ -938,6 +1104,28 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
the_index.split_index = NULL;
the_index.cache_changed |= SOMETHING_CHANGED;
}
+ if (untracked_cache > 0) {
+ struct untracked_cache *uc;
+
+ if (untracked_cache < 2) {
+ setup_work_tree();
+ if (!test_if_untracked_cache_is_supported())
+ return 1;
+ }
+ if (!the_index.untracked) {
+ uc = xcalloc(1, sizeof(*uc));
+ strbuf_init(&uc->ident, 100);
+ uc->exclude_per_dir = ".gitignore";
+ /* should be the same flags used by git-status */
+ uc->dir_flags = DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES;
+ the_index.untracked = uc;
+ }
+ add_untracked_ident(the_index.untracked);
+ the_index.cache_changed |= UNTRACKED_CHANGED;
+ } else if (!untracked_cache && the_index.untracked) {
+ the_index.untracked = NULL;
+ the_index.cache_changed |= UNTRACKED_CHANGED;
+ }
if (active_cache_changed) {
if (newfd < 0) {
diff --git a/cache.h b/cache.h
index 1f4226b..9da9784 100644
--- a/cache.h
+++ b/cache.h
@@ -297,8 +297,11 @@ static inline unsigned int canon_mode(unsigned int mode)
#define RESOLVE_UNDO_CHANGED (1 << 4)
#define CACHE_TREE_CHANGED (1 << 5)
#define SPLIT_INDEX_ORDERED (1 << 6)
+#define UNTRACKED_CHANGED (1 << 7)
struct split_index;
+struct untracked_cache;
+
struct index_state {
struct cache_entry **cache;
unsigned int version;
@@ -312,6 +315,7 @@ struct index_state {
struct hashmap name_hash;
struct hashmap dir_hash;
unsigned char sha1[20];
+ struct untracked_cache *untracked;
};
extern struct index_state the_index;
@@ -563,6 +567,8 @@ extern void fill_stat_data(struct stat_data *sd, struct stat *st);
* INODE_CHANGED, and DATA_CHANGED.
*/
extern int match_stat_data(const struct stat_data *sd, struct stat *st);
+extern int match_stat_data_racy(const struct index_state *istate,
+ const struct stat_data *sd, struct stat *st);
extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);
diff --git a/compat/mingw.c b/compat/mingw.c
index 70f3191..496e6f8 100644
--- a/compat/mingw.c
+++ b/compat/mingw.c
@@ -2128,3 +2128,14 @@ void mingw_startup()
/* initialize Unicode console */
winansi_init();
}
+
+int uname(struct utsname *buf)
+{
+ DWORD v = GetVersion();
+ memset(buf, 0, sizeof(*buf));
+ strcpy(buf->sysname, "Windows");
+ sprintf(buf->release, "%u.%u", v & 0xff, (v >> 8) & 0xff);
+ /* assuming NT variants only.. */
+ sprintf(buf->version, "%u", (v >> 16) & 0x7fff);
+ return 0;
+}
diff --git a/compat/mingw.h b/compat/mingw.h
index 98c5e44..738865c 100644
--- a/compat/mingw.h
+++ b/compat/mingw.h
@@ -76,6 +76,14 @@ struct itimerval {
};
#define ITIMER_REAL 0
+struct utsname {
+ char sysname[16];
+ char nodename[1];
+ char release[16];
+ char version[16];
+ char machine[1];
+};
+
/*
* sanitize preprocessor namespace polluted by Windows headers defining
* macros which collide with git local versions
@@ -175,6 +183,7 @@ struct passwd *getpwuid(uid_t uid);
int setitimer(int type, struct itimerval *in, struct itimerval *out);
int sigaction(int sig, struct sigaction *in, struct sigaction *out);
int link(const char *oldpath, const char *newpath);
+int uname(struct utsname *buf);
/*
* replacements of existing functions
diff --git a/dir.c b/dir.c
index 4183acc..d318ffc 100644
--- a/dir.c
+++ b/dir.c
@@ -13,6 +13,8 @@
#include "wildmatch.h"
#include "pathspec.h"
#include "utf8.h"
+#include "varint.h"
+#include "ewah/ewok.h"
struct path_simplify {
int len;
@@ -32,8 +34,22 @@ enum path_treatment {
path_untracked
};
+/*
+ * Support data structure for our opendir/readdir/closedir wrappers
+ */
+struct cached_dir {
+ DIR *fdir;
+ struct untracked_cache_dir *untracked;
+ int nr_files;
+ int nr_dirs;
+
+ struct dirent *de;
+ const char *file;
+ struct untracked_cache_dir *ucd;
+};
+
static enum path_treatment read_directory_recursive(struct dir_struct *dir,
- const char *path, int len,
+ const char *path, int len, struct untracked_cache_dir *untracked,
int check_only, const struct path_simplify *simplify);
static int get_dtype(struct dirent *de, const char *path, int len);
@@ -510,7 +526,8 @@ void add_exclude(const char *string, const char *base,
x->el = el;
}
-static void *read_skip_worktree_file_from_index(const char *path, size_t *size)
+static void *read_skip_worktree_file_from_index(const char *path, size_t *size,
+ struct sha1_stat *sha1_stat)
{
int pos, len;
unsigned long sz;
@@ -529,6 +546,10 @@ static void *read_skip_worktree_file_from_index(const char *path, size_t *size)
return NULL;
}
*size = xsize_t(sz);
+ if (sha1_stat) {
+ memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat));
+ hashcpy(sha1_stat->sha1, active_cache[pos]->sha1);
+ }
return data;
}
@@ -573,11 +594,93 @@ static void trim_trailing_spaces(char *buf)
*last_space = '\0';
}
-int add_excludes_from_file_to_list(const char *fname,
- const char *base,
- int baselen,
- struct exclude_list *el,
- int check_index)
+/*
+ * Given a subdirectory name and "dir" of the current directory,
+ * search the subdir in "dir" and return it, or create a new one if it
+ * does not exist in "dir".
+ *
+ * If "name" has the trailing slash, it'll be excluded in the search.
+ */
+static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc,
+ struct untracked_cache_dir *dir,
+ const char *name, int len)
+{
+ int first, last;
+ struct untracked_cache_dir *d;
+ if (!dir)
+ return NULL;
+ if (len && name[len - 1] == '/')
+ len--;
+ first = 0;
+ last = dir->dirs_nr;
+ while (last > first) {
+ int cmp, next = (last + first) >> 1;
+ d = dir->dirs[next];
+ cmp = strncmp(name, d->name, len);
+ if (!cmp && strlen(d->name) > len)
+ cmp = -1;
+ if (!cmp)
+ return d;
+ if (cmp < 0) {
+ last = next;
+ continue;
+ }
+ first = next+1;
+ }
+
+ uc->dir_created++;
+ d = xmalloc(sizeof(*d) + len + 1);
+ memset(d, 0, sizeof(*d));
+ memcpy(d->name, name, len);
+ d->name[len] = '\0';
+
+ ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc);
+ memmove(dir->dirs + first + 1, dir->dirs + first,
+ (dir->dirs_nr - first) * sizeof(*dir->dirs));
+ dir->dirs_nr++;
+ dir->dirs[first] = d;
+ return d;
+}
+
+static void do_invalidate_gitignore(struct untracked_cache_dir *dir)
+{
+ int i;
+ dir->valid = 0;
+ dir->untracked_nr = 0;
+ for (i = 0; i < dir->dirs_nr; i++)
+ do_invalidate_gitignore(dir->dirs[i]);
+}
+
+static void invalidate_gitignore(struct untracked_cache *uc,
+ struct untracked_cache_dir *dir)
+{
+ uc->gitignore_invalidated++;
+ do_invalidate_gitignore(dir);
+}
+
+static void invalidate_directory(struct untracked_cache *uc,
+ struct untracked_cache_dir *dir)
+{
+ int i;
+ uc->dir_invalidated++;
+ dir->valid = 0;
+ dir->untracked_nr = 0;
+ for (i = 0; i < dir->dirs_nr; i++)
+ dir->dirs[i]->recurse = 0;
+}
+
+/*
+ * Given a file with name "fname", read it (either from disk, or from
+ * the index if "check_index" is non-zero), parse it and store the
+ * exclude rules in "el".
+ *
+ * If "ss" is not NULL, compute SHA-1 of the exclude file and fill
+ * stat data from disk (only valid if add_excludes returns zero). If
+ * ss_valid is non-zero, "ss" must contain good value as input.
+ */
+static int add_excludes(const char *fname, const char *base, int baselen,
+ struct exclude_list *el, int check_index,
+ struct sha1_stat *sha1_stat)
{
struct stat st;
int fd, i, lineno = 1;
@@ -591,7 +694,7 @@ int add_excludes_from_file_to_list(const char *fname,
if (0 <= fd)
close(fd);
if (!check_index ||
- (buf = read_skip_worktree_file_from_index(fname, &size)) == NULL)
+ (buf = read_skip_worktree_file_from_index(fname, &size, sha1_stat)) == NULL)
return -1;
if (size == 0) {
free(buf);
@@ -604,6 +707,11 @@ int add_excludes_from_file_to_list(const char *fname,
} else {
size = xsize_t(st.st_size);
if (size == 0) {
+ if (sha1_stat) {
+ fill_stat_data(&sha1_stat->stat, &st);
+ hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN);
+ sha1_stat->valid = 1;
+ }
close(fd);
return 0;
}
@@ -615,6 +723,22 @@ int add_excludes_from_file_to_list(const char *fname,
}
buf[size++] = '\n';
close(fd);
+ if (sha1_stat) {
+ int pos;
+ if (sha1_stat->valid &&
+ !match_stat_data_racy(&the_index, &sha1_stat->stat, &st))
+ ; /* no content change, ss->sha1 still good */
+ else if (check_index &&
+ (pos = cache_name_pos(fname, strlen(fname))) >= 0 &&
+ !ce_stage(active_cache[pos]) &&
+ ce_uptodate(active_cache[pos]) &&
+ !would_convert_to_git(fname))
+ hashcpy(sha1_stat->sha1, active_cache[pos]->sha1);
+ else
+ hash_sha1_file(buf, size, "blob", sha1_stat->sha1);
+ fill_stat_data(&sha1_stat->stat, &st);
+ sha1_stat->valid = 1;
+ }
}
el->filebuf = buf;
@@ -638,6 +762,13 @@ int add_excludes_from_file_to_list(const char *fname,
return 0;
}
+int add_excludes_from_file_to_list(const char *fname, const char *base,
+ int baselen, struct exclude_list *el,
+ int check_index)
+{
+ return add_excludes(fname, base, baselen, el, check_index, NULL);
+}
+
struct exclude_list *add_exclude_list(struct dir_struct *dir,
int group_type, const char *src)
{
@@ -655,14 +786,28 @@ struct exclude_list *add_exclude_list(struct dir_struct *dir,
/*
* Used to set up core.excludesfile and .git/info/exclude lists.
*/
-void add_excludes_from_file(struct dir_struct *dir, const char *fname)
+static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
+ struct sha1_stat *sha1_stat)
{
struct exclude_list *el;
+ /*
+ * catch setup_standard_excludes() that's called before
+ * dir->untracked is assigned. That function behaves
+ * differently when dir->untracked is non-NULL.
+ */
+ if (!dir->untracked)
+ dir->unmanaged_exclude_files++;
el = add_exclude_list(dir, EXC_FILE, fname);
- if (add_excludes_from_file_to_list(fname, "", 0, el, 0) < 0)
+ if (add_excludes(fname, "", 0, el, 0, sha1_stat) < 0)
die("cannot use %s as an exclude file", fname);
}
+void add_excludes_from_file(struct dir_struct *dir, const char *fname)
+{
+ dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */
+ add_excludes_from_file_1(dir, fname, NULL);
+}
+
int match_basename(const char *basename, int basenamelen,
const char *pattern, int prefix, int patternlen,
int flags)
@@ -837,6 +982,7 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
struct exclude_list_group *group;
struct exclude_list *el;
struct exclude_stack *stk = NULL;
+ struct untracked_cache_dir *untracked;
int current;
group = &dir->exclude_list_group[EXC_DIRS];
@@ -874,8 +1020,14 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
/* Read from the parent directories and push them down. */
current = stk ? stk->baselen : -1;
strbuf_setlen(&dir->basebuf, current < 0 ? 0 : current);
+ if (dir->untracked)
+ untracked = stk ? stk->ucd : dir->untracked->root;
+ else
+ untracked = NULL;
+
while (current < baselen) {
const char *cp;
+ struct sha1_stat sha1_stat;
stk = xcalloc(1, sizeof(*stk));
if (current < 0) {
@@ -886,10 +1038,15 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
if (!cp)
die("oops in prep_exclude");
cp++;
+ untracked =
+ lookup_untracked(dir->untracked, untracked,
+ base + current,
+ cp - base - current);
}
stk->prev = dir->exclude_stack;
stk->baselen = cp - base;
stk->exclude_ix = group->nr;
+ stk->ucd = untracked;
el = add_exclude_list(dir, EXC_DIRS, NULL);
strbuf_add(&dir->basebuf, base + current, stk->baselen - current);
assert(stk->baselen == dir->basebuf.len);
@@ -912,7 +1069,23 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
}
/* Try to read per-directory file */
- if (dir->exclude_per_dir) {
+ hashclr(sha1_stat.sha1);
+ sha1_stat.valid = 0;
+ if (dir->exclude_per_dir &&
+ /*
+ * If we know that no files have been added in
+ * this directory (i.e. valid_cached_dir() has
+ * been executed and set untracked->valid) ..
+ */
+ (!untracked || !untracked->valid ||
+ /*
+ * .. and .gitignore does not exist before
+ * (i.e. null exclude_sha1 and skip_worktree is
+ * not set). Then we can skip loading .gitignore,
+ * which would result in ENOENT anyway.
+ * skip_worktree is taken care in read_directory()
+ */
+ !is_null_sha1(untracked->exclude_sha1))) {
/*
* dir->basebuf gets reused by the traversal, but we
* need fname to remain unchanged to ensure the src
@@ -925,8 +1098,27 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen)
strbuf_addbuf(&sb, &dir->basebuf);
strbuf_addstr(&sb, dir->exclude_per_dir);
el->src = strbuf_detach(&sb, NULL);
- add_excludes_from_file_to_list(el->src, el->src,
- stk->baselen, el, 1);
+ add_excludes(el->src, el->src, stk->baselen, el, 1,
+ untracked ? &sha1_stat : NULL);
+ }
+ /*
+ * NEEDSWORK: when untracked cache is enabled, prep_exclude()
+ * will first be called in valid_cached_dir() then maybe many
+ * times more in last_exclude_matching(). When the cache is
+ * used, last_exclude_matching() will not be called and
+ * reading .gitignore content will be a waste.
+ *
+ * So when it's called by valid_cached_dir() and we can get
+ * .gitignore SHA-1 from the index (i.e. .gitignore is not
+ * modified on work tree), we could delay reading the
+ * .gitignore content until we absolutely need it in
+ * last_exclude_matching(). Be careful about ignore rule
+ * order, though, if you do that.
+ */
+ if (untracked &&
+ hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) {
+ invalidate_gitignore(dir->untracked, untracked);
+ hashcpy(untracked->exclude_sha1, sha1_stat.sha1);
}
dir->exclude_stack = stk;
current = stk->baselen;
@@ -1107,6 +1299,7 @@ static enum exist_status directory_exists_in_index(const char *dirname, int len)
* (c) otherwise, we recurse into it.
*/
static enum path_treatment treat_directory(struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
const char *dirname, int len, int exclude,
const struct path_simplify *simplify)
{
@@ -1134,7 +1327,9 @@ static enum path_treatment treat_directory(struct dir_struct *dir,
if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES))
return exclude ? path_excluded : path_untracked;
- return read_directory_recursive(dir, dirname, len, 1, simplify);
+ untracked = lookup_untracked(dir->untracked, untracked, dirname, len);
+ return read_directory_recursive(dir, dirname, len,
+ untracked, 1, simplify);
}
/*
@@ -1250,6 +1445,7 @@ static int get_dtype(struct dirent *de, const char *path, int len)
}
static enum path_treatment treat_one_path(struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
struct strbuf *path,
const struct path_simplify *simplify,
int dtype, struct dirent *de)
@@ -1302,7 +1498,7 @@ static enum path_treatment treat_one_path(struct dir_struct *dir,
return path_none;
case DT_DIR:
strbuf_addch(path, '/');
- return treat_directory(dir, path->buf, path->len, exclude,
+ return treat_directory(dir, untracked, path->buf, path->len, exclude,
simplify);
case DT_REG:
case DT_LNK:
@@ -1310,14 +1506,52 @@ static enum path_treatment treat_one_path(struct dir_struct *dir,
}
}
+static enum path_treatment treat_path_fast(struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
+ struct cached_dir *cdir,
+ struct strbuf *path,
+ int baselen,
+ const struct path_simplify *simplify)
+{
+ strbuf_setlen(path, baselen);
+ if (!cdir->ucd) {
+ strbuf_addstr(path, cdir->file);
+ return path_untracked;
+ }
+ strbuf_addstr(path, cdir->ucd->name);
+ /* treat_one_path() does this before it calls treat_directory() */
+ if (path->buf[path->len - 1] != '/')
+ strbuf_addch(path, '/');
+ if (cdir->ucd->check_only)
+ /*
+ * check_only is set as a result of treat_directory() getting
+ * to its bottom. Verify again the same set of directories
+ * with check_only set.
+ */
+ return read_directory_recursive(dir, path->buf, path->len,
+ cdir->ucd, 1, simplify);
+ /*
+ * We get path_recurse in the first run when
+ * directory_exists_in_index() returns index_nonexistent. We
+ * are sure that new changes in the index does not impact the
+ * outcome. Return now.
+ */
+ return path_recurse;
+}
+
static enum path_treatment treat_path(struct dir_struct *dir,
- struct dirent *de,
+ struct untracked_cache_dir *untracked,
+ struct cached_dir *cdir,
struct strbuf *path,
int baselen,
const struct path_simplify *simplify)
{
int dtype;
+ struct dirent *de = cdir->de;
+ if (!de)
+ return treat_path_fast(dir, untracked, cdir, path,
+ baselen, simplify);
if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git"))
return path_none;
strbuf_setlen(path, baselen);
@@ -1326,7 +1560,121 @@ static enum path_treatment treat_path(struct dir_struct *dir,
return path_none;
dtype = DTYPE(de);
- return treat_one_path(dir, path, simplify, dtype, de);
+ return treat_one_path(dir, untracked, path, simplify, dtype, de);
+}
+
+static void add_untracked(struct untracked_cache_dir *dir, const char *name)
+{
+ if (!dir)
+ return;
+ ALLOC_GROW(dir->untracked, dir->untracked_nr + 1,
+ dir->untracked_alloc);
+ dir->untracked[dir->untracked_nr++] = xstrdup(name);
+}
+
+static int valid_cached_dir(struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
+ struct strbuf *path,
+ int check_only)
+{
+ struct stat st;
+
+ if (!untracked)
+ return 0;
+
+ if (stat(path->len ? path->buf : ".", &st)) {
+ invalidate_directory(dir->untracked, untracked);
+ memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
+ return 0;
+ }
+ if (!untracked->valid ||
+ match_stat_data_racy(&the_index, &untracked->stat_data, &st)) {
+ if (untracked->valid)
+ invalidate_directory(dir->untracked, untracked);
+ fill_stat_data(&untracked->stat_data, &st);
+ return 0;
+ }
+
+ if (untracked->check_only != !!check_only) {
+ invalidate_directory(dir->untracked, untracked);
+ return 0;
+ }
+
+ /*
+ * prep_exclude will be called eventually on this directory,
+ * but it's called much later in last_exclude_matching(). We
+ * need it now to determine the validity of the cache for this
+ * path. The next calls will be nearly no-op, the way
+ * prep_exclude() is designed.
+ */
+ if (path->len && path->buf[path->len - 1] != '/') {
+ strbuf_addch(path, '/');
+ prep_exclude(dir, path->buf, path->len);
+ strbuf_setlen(path, path->len - 1);
+ } else
+ prep_exclude(dir, path->buf, path->len);
+
+ /* hopefully prep_exclude() haven't invalidated this entry... */
+ return untracked->valid;
+}
+
+static int open_cached_dir(struct cached_dir *cdir,
+ struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
+ struct strbuf *path,
+ int check_only)
+{
+ memset(cdir, 0, sizeof(*cdir));
+ cdir->untracked = untracked;
+ if (valid_cached_dir(dir, untracked, path, check_only))
+ return 0;
+ cdir->fdir = opendir(path->len ? path->buf : ".");
+ if (dir->untracked)
+ dir->untracked->dir_opened++;
+ if (!cdir->fdir)
+ return -1;
+ return 0;
+}
+
+static int read_cached_dir(struct cached_dir *cdir)
+{
+ if (cdir->fdir) {
+ cdir->de = readdir(cdir->fdir);
+ if (!cdir->de)
+ return -1;
+ return 0;
+ }
+ while (cdir->nr_dirs < cdir->untracked->dirs_nr) {
+ struct untracked_cache_dir *d = cdir->untracked->dirs[cdir->nr_dirs];
+ if (!d->recurse) {
+ cdir->nr_dirs++;
+ continue;
+ }
+ cdir->ucd = d;
+ cdir->nr_dirs++;
+ return 0;
+ }
+ cdir->ucd = NULL;
+ if (cdir->nr_files < cdir->untracked->untracked_nr) {
+ struct untracked_cache_dir *d = cdir->untracked;
+ cdir->file = d->untracked[cdir->nr_files++];
+ return 0;
+ }
+ return -1;
+}
+
+static void close_cached_dir(struct cached_dir *cdir)
+{
+ if (cdir->fdir)
+ closedir(cdir->fdir);
+ /*
+ * We have gone through this directory and found no untracked
+ * entries. Mark it valid.
+ */
+ if (cdir->untracked) {
+ cdir->untracked->valid = 1;
+ cdir->untracked->recurse = 1;
+ }
}
/*
@@ -1342,38 +1690,48 @@ static enum path_treatment treat_path(struct dir_struct *dir,
*/
static enum path_treatment read_directory_recursive(struct dir_struct *dir,
const char *base, int baselen,
- int check_only,
+ struct untracked_cache_dir *untracked, int check_only,
const struct path_simplify *simplify)
{
- DIR *fdir;
+ struct cached_dir cdir;
enum path_treatment state, subdir_state, dir_state = path_none;
- struct dirent *de;
struct strbuf path = STRBUF_INIT;
strbuf_add(&path, base, baselen);
- fdir = opendir(path.len ? path.buf : ".");
- if (!fdir)
+ if (open_cached_dir(&cdir, dir, untracked, &path, check_only))
goto out;
- while ((de = readdir(fdir)) != NULL) {
+ if (untracked)
+ untracked->check_only = !!check_only;
+
+ while (!read_cached_dir(&cdir)) {
/* check how the file or directory should be treated */
- state = treat_path(dir, de, &path, baselen, simplify);
+ state = treat_path(dir, untracked, &cdir, &path, baselen, simplify);
+
if (state > dir_state)
dir_state = state;
/* recurse into subdir if instructed by treat_path */
if (state == path_recurse) {
- subdir_state = read_directory_recursive(dir, path.buf,
- path.len, check_only, simplify);
+ struct untracked_cache_dir *ud;
+ ud = lookup_untracked(dir->untracked, untracked,
+ path.buf + baselen,
+ path.len - baselen);
+ subdir_state =
+ read_directory_recursive(dir, path.buf, path.len,
+ ud, check_only, simplify);
if (subdir_state > dir_state)
dir_state = subdir_state;
}
if (check_only) {
/* abort early if maximum state has been reached */
- if (dir_state == path_untracked)
+ if (dir_state == path_untracked) {
+ if (cdir.fdir)
+ add_untracked(untracked, path.buf + baselen);
break;
+ }
/* skip the dir_add_* part */
continue;
}
@@ -1391,15 +1749,18 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir,
break;
case path_untracked:
- if (!(dir->flags & DIR_SHOW_IGNORED))
- dir_add_name(dir, path.buf, path.len);
+ if (dir->flags & DIR_SHOW_IGNORED)
+ break;
+ dir_add_name(dir, path.buf, path.len);
+ if (cdir.fdir)
+ add_untracked(untracked, path.buf + baselen);
break;
default:
break;
}
}
- closedir(fdir);
+ close_cached_dir(&cdir);
out:
strbuf_release(&path);
@@ -1469,7 +1830,7 @@ static int treat_leading_path(struct dir_struct *dir,
break;
if (simplify_away(sb.buf, sb.len, simplify))
break;
- if (treat_one_path(dir, &sb, simplify,
+ if (treat_one_path(dir, NULL, &sb, simplify,
DT_DIR, NULL) == path_none)
break; /* do not recurse into it */
if (len <= baselen) {
@@ -1482,9 +1843,139 @@ static int treat_leading_path(struct dir_struct *dir,
return rc;
}
+static const char *get_ident_string(void)
+{
+ static struct strbuf sb = STRBUF_INIT;
+ struct utsname uts;
+
+ if (sb.len)
+ return sb.buf;
+ if (uname(&uts))
+ die_errno(_("failed to get kernel name and information"));
+ strbuf_addf(&sb, "Location %s, system %s %s %s", get_git_work_tree(),
+ uts.sysname, uts.release, uts.version);
+ return sb.buf;
+}
+
+static int ident_in_untracked(const struct untracked_cache *uc)
+{
+ const char *end = uc->ident.buf + uc->ident.len;
+ const char *p = uc->ident.buf;
+
+ for (p = uc->ident.buf; p < end; p += strlen(p) + 1)
+ if (!strcmp(p, get_ident_string()))
+ return 1;
+ return 0;
+}
+
+void add_untracked_ident(struct untracked_cache *uc)
+{
+ if (ident_in_untracked(uc))
+ return;
+ strbuf_addstr(&uc->ident, get_ident_string());
+ /* this strbuf contains a list of strings, save NUL too */
+ strbuf_addch(&uc->ident, 0);
+}
+
+static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *dir,
+ int base_len,
+ const struct pathspec *pathspec)
+{
+ struct untracked_cache_dir *root;
+ int i;
+
+ if (!dir->untracked || getenv("GIT_DISABLE_UNTRACKED_CACHE"))
+ return NULL;
+
+ /*
+ * We only support $GIT_DIR/info/exclude and core.excludesfile
+ * as the global ignore rule files. Any other additions
+ * (e.g. from command line) invalidate the cache. This
+ * condition also catches running setup_standard_excludes()
+ * before setting dir->untracked!
+ */
+ if (dir->unmanaged_exclude_files)
+ return NULL;
+
+ /*
+ * Optimize for the main use case only: whole-tree git
+ * status. More work involved in treat_leading_path() if we
+ * use cache on just a subset of the worktree. pathspec
+ * support could make the matter even worse.
+ */
+ if (base_len || (pathspec && pathspec->nr))
+ return NULL;
+
+ /* Different set of flags may produce different results */
+ if (dir->flags != dir->untracked->dir_flags ||
+ /*
+ * See treat_directory(), case index_nonexistent. Without
+ * this flag, we may need to also cache .git file content
+ * for the resolve_gitlink_ref() call, which we don't.
+ */
+ !(dir->flags & DIR_SHOW_OTHER_DIRECTORIES) ||
+ /* We don't support collecting ignore files */
+ (dir->flags & (DIR_SHOW_IGNORED | DIR_SHOW_IGNORED_TOO |
+ DIR_COLLECT_IGNORED)))
+ return NULL;
+
+ /*
+ * If we use .gitignore in the cache and now you change it to
+ * .gitexclude, everything will go wrong.
+ */
+ if (dir->exclude_per_dir != dir->untracked->exclude_per_dir &&
+ strcmp(dir->exclude_per_dir, dir->untracked->exclude_per_dir))
+ return NULL;
+
+ /*
+ * EXC_CMDL is not considered in the cache. If people set it,
+ * skip the cache.
+ */
+ if (dir->exclude_list_group[EXC_CMDL].nr)
+ return NULL;
+
+ /*
+ * An optimization in prep_exclude() does not play well with
+ * CE_SKIP_WORKTREE. It's a rare case anyway, if a single
+ * entry has that bit set, disable the whole untracked cache.
+ */
+ for (i = 0; i < active_nr; i++)
+ if (ce_skip_worktree(active_cache[i]))
+ return NULL;
+
+ if (!ident_in_untracked(dir->untracked)) {
+ warning(_("Untracked cache is disabled on this system."));
+ return NULL;
+ }
+
+ if (!dir->untracked->root) {
+ const int len = sizeof(*dir->untracked->root);
+ dir->untracked->root = xmalloc(len);
+ memset(dir->untracked->root, 0, len);
+ }
+
+ /* Validate $GIT_DIR/info/exclude and core.excludesfile */
+ root = dir->untracked->root;
+ if (hashcmp(dir->ss_info_exclude.sha1,
+ dir->untracked->ss_info_exclude.sha1)) {
+ invalidate_gitignore(dir->untracked, root);
+ dir->untracked->ss_info_exclude = dir->ss_info_exclude;
+ }
+ if (hashcmp(dir->ss_excludes_file.sha1,
+ dir->untracked->ss_excludes_file.sha1)) {
+ invalidate_gitignore(dir->untracked, root);
+ dir->untracked->ss_excludes_file = dir->ss_excludes_file;
+ }
+
+ /* Make sure this directory is not dropped out at saving phase */
+ root->recurse = 1;
+ return root;
+}
+
int read_directory(struct dir_struct *dir, const char *path, int len, const struct pathspec *pathspec)
{
struct path_simplify *simplify;
+ struct untracked_cache_dir *untracked;
/*
* Check out create_simplify()
@@ -1508,11 +1999,39 @@ int read_directory(struct dir_struct *dir, const char *path, int len, const stru
* create_simplify().
*/
simplify = create_simplify(pathspec ? pathspec->_raw : NULL);
+ untracked = validate_untracked_cache(dir, len, pathspec);
+ if (!untracked)
+ /*
+ * make sure untracked cache code path is disabled,
+ * e.g. prep_exclude()
+ */
+ dir->untracked = NULL;
if (!len || treat_leading_path(dir, path, len, simplify))
- read_directory_recursive(dir, path, len, 0, simplify);
+ read_directory_recursive(dir, path, len, untracked, 0, simplify);
free_simplify(simplify);
qsort(dir->entries, dir->nr, sizeof(struct dir_entry *), cmp_name);
qsort(dir->ignored, dir->ignored_nr, sizeof(struct dir_entry *), cmp_name);
+ if (dir->untracked) {
+ static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS);
+ trace_printf_key(&trace_untracked_stats,
+ "node creation: %u\n"
+ "gitignore invalidation: %u\n"
+ "directory invalidation: %u\n"
+ "opendir: %u\n",
+ dir->untracked->dir_created,
+ dir->untracked->gitignore_invalidated,
+ dir->untracked->dir_invalidated,
+ dir->untracked->dir_opened);
+ if (dir->untracked == the_index.untracked &&
+ (dir->untracked->dir_opened ||
+ dir->untracked->gitignore_invalidated ||
+ dir->untracked->dir_invalidated))
+ the_index.cache_changed |= UNTRACKED_CHANGED;
+ if (dir->untracked != the_index.untracked) {
+ free(dir->untracked);
+ dir->untracked = NULL;
+ }
+ }
return dir->nr;
}
@@ -1678,12 +2197,14 @@ void setup_standard_excludes(struct dir_struct *dir)
if (!excludes_file)
excludes_file = xdg_config_home("ignore");
if (excludes_file && !access_or_warn(excludes_file, R_OK, 0))
- add_excludes_from_file(dir, excludes_file);
+ add_excludes_from_file_1(dir, excludes_file,
+ dir->untracked ? &dir->ss_excludes_file : NULL);
/* per repository user preference */
path = git_path("info/exclude");
if (!access_or_warn(path, R_OK, 0))
- add_excludes_from_file(dir, path);
+ add_excludes_from_file_1(dir, path,
+ dir->untracked ? &dir->ss_info_exclude : NULL);
}
int remove_path(const char *name)
@@ -1735,3 +2256,404 @@ void clear_directory(struct dir_struct *dir)
}
strbuf_release(&dir->basebuf);
}
+
+struct ondisk_untracked_cache {
+ struct stat_data info_exclude_stat;
+ struct stat_data excludes_file_stat;
+ uint32_t dir_flags;
+ unsigned char info_exclude_sha1[20];
+ unsigned char excludes_file_sha1[20];
+ char exclude_per_dir[FLEX_ARRAY];
+};
+
+#define ouc_size(len) (offsetof(struct ondisk_untracked_cache, exclude_per_dir) + len + 1)
+
+struct write_data {
+ int index; /* number of written untracked_cache_dir */
+ struct ewah_bitmap *check_only; /* from untracked_cache_dir */
+ struct ewah_bitmap *valid; /* from untracked_cache_dir */
+ struct ewah_bitmap *sha1_valid; /* set if exclude_sha1 is not null */
+ struct strbuf out;
+ struct strbuf sb_stat;
+ struct strbuf sb_sha1;
+};
+
+static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from)
+{
+ to->sd_ctime.sec = htonl(from->sd_ctime.sec);
+ to->sd_ctime.nsec = htonl(from->sd_ctime.nsec);
+ to->sd_mtime.sec = htonl(from->sd_mtime.sec);
+ to->sd_mtime.nsec = htonl(from->sd_mtime.nsec);
+ to->sd_dev = htonl(from->sd_dev);
+ to->sd_ino = htonl(from->sd_ino);
+ to->sd_uid = htonl(from->sd_uid);
+ to->sd_gid = htonl(from->sd_gid);
+ to->sd_size = htonl(from->sd_size);
+}
+
+static void write_one_dir(struct untracked_cache_dir *untracked,
+ struct write_data *wd)
+{
+ struct stat_data stat_data;
+ struct strbuf *out = &wd->out;
+ unsigned char intbuf[16];
+ unsigned int intlen, value;
+ int i = wd->index++;
+
+ /*
+ * untracked_nr should be reset whenever valid is clear, but
+ * for safety..
+ */
+ if (!untracked->valid) {
+ untracked->untracked_nr = 0;
+ untracked->check_only = 0;
+ }
+
+ if (untracked->check_only)
+ ewah_set(wd->check_only, i);
+ if (untracked->valid) {
+ ewah_set(wd->valid, i);
+ stat_data_to_disk(&stat_data, &untracked->stat_data);
+ strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data));
+ }
+ if (!is_null_sha1(untracked->exclude_sha1)) {
+ ewah_set(wd->sha1_valid, i);
+ strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20);
+ }
+
+ intlen = encode_varint(untracked->untracked_nr, intbuf);
+ strbuf_add(out, intbuf, intlen);
+
+ /* skip non-recurse directories */
+ for (i = 0, value = 0; i < untracked->dirs_nr; i++)
+ if (untracked->dirs[i]->recurse)
+ value++;
+ intlen = encode_varint(value, intbuf);
+ strbuf_add(out, intbuf, intlen);
+
+ strbuf_add(out, untracked->name, strlen(untracked->name) + 1);
+
+ for (i = 0; i < untracked->untracked_nr; i++)
+ strbuf_add(out, untracked->untracked[i],
+ strlen(untracked->untracked[i]) + 1);
+
+ for (i = 0; i < untracked->dirs_nr; i++)
+ if (untracked->dirs[i]->recurse)
+ write_one_dir(untracked->dirs[i], wd);
+}
+
+void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked)
+{
+ struct ondisk_untracked_cache *ouc;
+ struct write_data wd;
+ unsigned char varbuf[16];
+ int len = 0, varint_len;
+ if (untracked->exclude_per_dir)
+ len = strlen(untracked->exclude_per_dir);
+ ouc = xmalloc(sizeof(*ouc) + len + 1);
+ stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat);
+ stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat);
+ hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1);
+ hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1);
+ ouc->dir_flags = htonl(untracked->dir_flags);
+ memcpy(ouc->exclude_per_dir, untracked->exclude_per_dir, len + 1);
+
+ varint_len = encode_varint(untracked->ident.len, varbuf);
+ strbuf_add(out, varbuf, varint_len);
+ strbuf_add(out, untracked->ident.buf, untracked->ident.len);
+
+ strbuf_add(out, ouc, ouc_size(len));
+ free(ouc);
+ ouc = NULL;
+
+ if (!untracked->root) {
+ varint_len = encode_varint(0, varbuf);
+ strbuf_add(out, varbuf, varint_len);
+ return;
+ }
+
+ wd.index = 0;
+ wd.check_only = ewah_new();
+ wd.valid = ewah_new();
+ wd.sha1_valid = ewah_new();
+ strbuf_init(&wd.out, 1024);
+ strbuf_init(&wd.sb_stat, 1024);
+ strbuf_init(&wd.sb_sha1, 1024);
+ write_one_dir(untracked->root, &wd);
+
+ varint_len = encode_varint(wd.index, varbuf);
+ strbuf_add(out, varbuf, varint_len);
+ strbuf_addbuf(out, &wd.out);
+ ewah_serialize_strbuf(wd.valid, out);
+ ewah_serialize_strbuf(wd.check_only, out);
+ ewah_serialize_strbuf(wd.sha1_valid, out);
+ strbuf_addbuf(out, &wd.sb_stat);
+ strbuf_addbuf(out, &wd.sb_sha1);
+ strbuf_addch(out, '\0'); /* safe guard for string lists */
+
+ ewah_free(wd.valid);
+ ewah_free(wd.check_only);
+ ewah_free(wd.sha1_valid);
+ strbuf_release(&wd.out);
+ strbuf_release(&wd.sb_stat);
+ strbuf_release(&wd.sb_sha1);
+}
+
+static void free_untracked(struct untracked_cache_dir *ucd)
+{
+ int i;
+ if (!ucd)
+ return;
+ for (i = 0; i < ucd->dirs_nr; i++)
+ free_untracked(ucd->dirs[i]);
+ for (i = 0; i < ucd->untracked_nr; i++)
+ free(ucd->untracked[i]);
+ free(ucd->untracked);
+ free(ucd->dirs);
+ free(ucd);
+}
+
+void free_untracked_cache(struct untracked_cache *uc)
+{
+ if (uc)
+ free_untracked(uc->root);
+ free(uc);
+}
+
+struct read_data {
+ int index;
+ struct untracked_cache_dir **ucd;
+ struct ewah_bitmap *check_only;
+ struct ewah_bitmap *valid;
+ struct ewah_bitmap *sha1_valid;
+ const unsigned char *data;
+ const unsigned char *end;
+};
+
+static void stat_data_from_disk(struct stat_data *to, const struct stat_data *from)
+{
+ to->sd_ctime.sec = get_be32(&from->sd_ctime.sec);
+ to->sd_ctime.nsec = get_be32(&from->sd_ctime.nsec);
+ to->sd_mtime.sec = get_be32(&from->sd_mtime.sec);
+ to->sd_mtime.nsec = get_be32(&from->sd_mtime.nsec);
+ to->sd_dev = get_be32(&from->sd_dev);
+ to->sd_ino = get_be32(&from->sd_ino);
+ to->sd_uid = get_be32(&from->sd_uid);
+ to->sd_gid = get_be32(&from->sd_gid);
+ to->sd_size = get_be32(&from->sd_size);
+}
+
+static int read_one_dir(struct untracked_cache_dir **untracked_,
+ struct read_data *rd)
+{
+ struct untracked_cache_dir ud, *untracked;
+ const unsigned char *next, *data = rd->data, *end = rd->end;
+ unsigned int value;
+ int i, len;
+
+ memset(&ud, 0, sizeof(ud));
+
+ next = data;
+ value = decode_varint(&next);
+ if (next > end)
+ return -1;
+ ud.recurse = 1;
+ ud.untracked_alloc = value;
+ ud.untracked_nr = value;
+ if (ud.untracked_nr)
+ ud.untracked = xmalloc(sizeof(*ud.untracked) * ud.untracked_nr);
+ data = next;
+
+ next = data;
+ ud.dirs_alloc = ud.dirs_nr = decode_varint(&next);
+ if (next > end)
+ return -1;
+ ud.dirs = xmalloc(sizeof(*ud.dirs) * ud.dirs_nr);
+ data = next;
+
+ len = strlen((const char *)data);
+ next = data + len + 1;
+ if (next > rd->end)
+ return -1;
+ *untracked_ = untracked = xmalloc(sizeof(*untracked) + len);
+ memcpy(untracked, &ud, sizeof(ud));
+ memcpy(untracked->name, data, len + 1);
+ data = next;
+
+ for (i = 0; i < untracked->untracked_nr; i++) {
+ len = strlen((const char *)data);
+ next = data + len + 1;
+ if (next > rd->end)
+ return -1;
+ untracked->untracked[i] = xstrdup((const char*)data);
+ data = next;
+ }
+
+ rd->ucd[rd->index++] = untracked;
+ rd->data = data;
+
+ for (i = 0; i < untracked->dirs_nr; i++) {
+ len = read_one_dir(untracked->dirs + i, rd);
+ if (len < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void set_check_only(size_t pos, void *cb)
+{
+ struct read_data *rd = cb;
+ struct untracked_cache_dir *ud = rd->ucd[pos];
+ ud->check_only = 1;
+}
+
+static void read_stat(size_t pos, void *cb)
+{
+ struct read_data *rd = cb;
+ struct untracked_cache_dir *ud = rd->ucd[pos];
+ if (rd->data + sizeof(struct stat_data) > rd->end) {
+ rd->data = rd->end + 1;
+ return;
+ }
+ stat_data_from_disk(&ud->stat_data, (struct stat_data *)rd->data);
+ rd->data += sizeof(struct stat_data);
+ ud->valid = 1;
+}
+
+static void read_sha1(size_t pos, void *cb)
+{
+ struct read_data *rd = cb;
+ struct untracked_cache_dir *ud = rd->ucd[pos];
+ if (rd->data + 20 > rd->end) {
+ rd->data = rd->end + 1;
+ return;
+ }
+ hashcpy(ud->exclude_sha1, rd->data);
+ rd->data += 20;
+}
+
+static void load_sha1_stat(struct sha1_stat *sha1_stat,
+ const struct stat_data *stat,
+ const unsigned char *sha1)
+{
+ stat_data_from_disk(&sha1_stat->stat, stat);
+ hashcpy(sha1_stat->sha1, sha1);
+ sha1_stat->valid = 1;
+}
+
+struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
+{
+ const struct ondisk_untracked_cache *ouc;
+ struct untracked_cache *uc;
+ struct read_data rd;
+ const unsigned char *next = data, *end = (const unsigned char *)data + sz;
+ const char *ident;
+ int ident_len, len;
+
+ if (sz <= 1 || end[-1] != '\0')
+ return NULL;
+ end--;
+
+ ident_len = decode_varint(&next);
+ if (next + ident_len > end)
+ return NULL;
+ ident = (const char *)next;
+ next += ident_len;
+
+ ouc = (const struct ondisk_untracked_cache *)next;
+ if (next + ouc_size(0) > end)
+ return NULL;
+
+ uc = xcalloc(1, sizeof(*uc));
+ strbuf_init(&uc->ident, ident_len);
+ strbuf_add(&uc->ident, ident, ident_len);
+ load_sha1_stat(&uc->ss_info_exclude, &ouc->info_exclude_stat,
+ ouc->info_exclude_sha1);
+ load_sha1_stat(&uc->ss_excludes_file, &ouc->excludes_file_stat,
+ ouc->excludes_file_sha1);
+ uc->dir_flags = get_be32(&ouc->dir_flags);
+ uc->exclude_per_dir = xstrdup(ouc->exclude_per_dir);
+ /* NUL after exclude_per_dir is covered by sizeof(*ouc) */
+ next += ouc_size(strlen(ouc->exclude_per_dir));
+ if (next >= end)
+ goto done2;
+
+ len = decode_varint(&next);
+ if (next > end || len == 0)
+ goto done2;
+
+ rd.valid = ewah_new();
+ rd.check_only = ewah_new();
+ rd.sha1_valid = ewah_new();
+ rd.data = next;
+ rd.end = end;
+ rd.index = 0;
+ rd.ucd = xmalloc(sizeof(*rd.ucd) * len);
+
+ if (read_one_dir(&uc->root, &rd) || rd.index != len)
+ goto done;
+
+ next = rd.data;
+ len = ewah_read_mmap(rd.valid, next, end - next);
+ if (len < 0)
+ goto done;
+
+ next += len;
+ len = ewah_read_mmap(rd.check_only, next, end - next);
+ if (len < 0)
+ goto done;
+
+ next += len;
+ len = ewah_read_mmap(rd.sha1_valid, next, end - next);
+ if (len < 0)
+ goto done;
+
+ ewah_each_bit(rd.check_only, set_check_only, &rd);
+ rd.data = next + len;
+ ewah_each_bit(rd.valid, read_stat, &rd);
+ ewah_each_bit(rd.sha1_valid, read_sha1, &rd);
+ next = rd.data;
+
+done:
+ free(rd.ucd);
+ ewah_free(rd.valid);
+ ewah_free(rd.check_only);
+ ewah_free(rd.sha1_valid);
+done2:
+ if (next != end) {
+ free_untracked_cache(uc);
+ uc = NULL;
+ }
+ return uc;
+}
+
+void untracked_cache_invalidate_path(struct index_state *istate,
+ const char *path)
+{
+ const char *sep;
+ struct untracked_cache_dir *d;
+ if (!istate->untracked || !istate->untracked->root)
+ return;
+ sep = strrchr(path, '/');
+ if (sep)
+ d = lookup_untracked(istate->untracked,
+ istate->untracked->root,
+ path, sep - path);
+ else
+ d = istate->untracked->root;
+ istate->untracked->dir_invalidated++;
+ d->valid = 0;
+ d->untracked_nr = 0;
+}
+
+void untracked_cache_remove_from_index(struct index_state *istate,
+ const char *path)
+{
+ untracked_cache_invalidate_path(istate, path);
+}
+
+void untracked_cache_add_to_index(struct index_state *istate,
+ const char *path)
+{
+ untracked_cache_invalidate_path(istate, path);
+}
diff --git a/dir.h b/dir.h
index 72b73c6..7b5855d 100644
--- a/dir.h
+++ b/dir.h
@@ -66,6 +66,7 @@ struct exclude_stack {
struct exclude_stack *prev; /* the struct exclude_stack for the parent directory */
int baselen;
int exclude_ix; /* index of exclude_list within EXC_DIRS exclude_list_group */
+ struct untracked_cache_dir *ucd;
};
struct exclude_list_group {
@@ -73,6 +74,73 @@ struct exclude_list_group {
struct exclude_list *el;
};
+struct sha1_stat {
+ struct stat_data stat;
+ unsigned char sha1[20];
+ int valid;
+};
+
+/*
+ * Untracked cache
+ *
+ * The following inputs are sufficient to determine what files in a
+ * directory are excluded:
+ *
+ * - The list of files and directories of the directory in question
+ * - The $GIT_DIR/index
+ * - dir_struct flags
+ * - The content of $GIT_DIR/info/exclude
+ * - The content of core.excludesfile
+ * - The content (or the lack) of .gitignore of all parent directories
+ * from $GIT_WORK_TREE
+ * - The check_only flag in read_directory_recursive (for
+ * DIR_HIDE_EMPTY_DIRECTORIES)
+ *
+ * The first input can be checked using directory mtime. In many
+ * filesystems, directory mtime (stat_data field) is updated when its
+ * files or direct subdirs are added or removed.
+ *
+ * The second one can be hooked from cache_tree_invalidate_path().
+ * Whenever a file (or a submodule) is added or removed from a
+ * directory, we invalidate that directory.
+ *
+ * The remaining inputs are easy, their SHA-1 could be used to verify
+ * their contents (exclude_sha1[], info_exclude_sha1[] and
+ * excludes_file_sha1[])
+ */
+struct untracked_cache_dir {
+ struct untracked_cache_dir **dirs;
+ char **untracked;
+ struct stat_data stat_data;
+ unsigned int untracked_alloc, dirs_nr, dirs_alloc;
+ unsigned int untracked_nr;
+ unsigned int check_only : 1;
+ /* all data except 'dirs' in this struct are good */
+ unsigned int valid : 1;
+ unsigned int recurse : 1;
+ /* null SHA-1 means this directory does not have .gitignore */
+ unsigned char exclude_sha1[20];
+ char name[FLEX_ARRAY];
+};
+
+struct untracked_cache {
+ struct sha1_stat ss_info_exclude;
+ struct sha1_stat ss_excludes_file;
+ const char *exclude_per_dir;
+ struct strbuf ident;
+ /*
+ * dir_struct#flags must match dir_flags or the untracked
+ * cache is ignored.
+ */
+ unsigned dir_flags;
+ struct untracked_cache_dir *root;
+ /* Statistics */
+ int dir_created;
+ int gitignore_invalidated;
+ int dir_invalidated;
+ int dir_opened;
+};
+
struct dir_struct {
int nr, alloc;
int ignored_nr, ignored_alloc;
@@ -120,6 +188,12 @@ struct dir_struct {
struct exclude_stack *exclude_stack;
struct exclude *exclude;
struct strbuf basebuf;
+
+ /* Enable untracked file cache if set */
+ struct untracked_cache *untracked;
+ struct sha1_stat ss_info_exclude;
+ struct sha1_stat ss_excludes_file;
+ unsigned unmanaged_exclude_files;
};
/*
@@ -226,4 +300,12 @@ static inline int dir_path_match(const struct dir_entry *ent,
has_trailing_dir);
}
+void untracked_cache_invalidate_path(struct index_state *, const char *);
+void untracked_cache_remove_from_index(struct index_state *, const char *);
+void untracked_cache_add_to_index(struct index_state *, const char *);
+
+void free_untracked_cache(struct untracked_cache *);
+struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz);
+void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked);
+void add_untracked_ident(struct untracked_cache *);
#endif
diff --git a/ewah/ewah_io.c b/ewah/ewah_io.c
index 1c2d7af..43481b9 100644
--- a/ewah/ewah_io.c
+++ b/ewah/ewah_io.c
@@ -19,6 +19,7 @@
*/
#include "git-compat-util.h"
#include "ewok.h"
+#include "strbuf.h"
int ewah_serialize_native(struct ewah_bitmap *self, int fd)
{
@@ -110,6 +111,18 @@ int ewah_serialize(struct ewah_bitmap *self, int fd)
return ewah_serialize_to(self, write_helper, (void *)(intptr_t)fd);
}
+static int write_strbuf(void *user_data, const void *data, size_t len)
+{
+ struct strbuf *sb = user_data;
+ strbuf_add(sb, data, len);
+ return len;
+}
+
+int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *sb)
+{
+ return ewah_serialize_to(self, write_strbuf, sb);
+}
+
int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len)
{
const uint8_t *ptr = map;
diff --git a/ewah/ewok.h b/ewah/ewok.h
index 13c6e20..e732525 100644
--- a/ewah/ewok.h
+++ b/ewah/ewok.h
@@ -30,6 +30,7 @@
# define ewah_calloc xcalloc
#endif
+struct strbuf;
typedef uint64_t eword_t;
#define BITS_IN_WORD (sizeof(eword_t) * 8)
@@ -98,6 +99,7 @@ int ewah_serialize_to(struct ewah_bitmap *self,
void *out);
int ewah_serialize(struct ewah_bitmap *self, int fd);
int ewah_serialize_native(struct ewah_bitmap *self, int fd);
+int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *);
int ewah_deserialize(struct ewah_bitmap *self, int fd);
int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len);
diff --git a/git-compat-util.h b/git-compat-util.h
index b7a97fb..17584ad 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -189,6 +189,7 @@
#elif defined(_MSC_VER)
#include "compat/msvc.h"
#else
+#include <sys/utsname.h>
#include <sys/wait.h>
#include <sys/resource.h>
#include <sys/socket.h>
diff --git a/read-cache.c b/read-cache.c
index 36ff89f..723d48d 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -39,11 +39,12 @@ static struct cache_entry *refresh_cache_entry(struct cache_entry *ce,
#define CACHE_EXT_TREE 0x54524545 /* "TREE" */
#define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
#define CACHE_EXT_LINK 0x6c696e6b /* "link" */
+#define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */
/* changes that can be kept in $GIT_DIR/index (basically all extensions) */
#define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \
CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \
- SPLIT_INDEX_ORDERED)
+ SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED)
struct index_state the_index;
static const char *alternate_index_output;
@@ -79,6 +80,7 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n
memcpy(new->name, new_name, namelen + 1);
cache_tree_invalidate_path(istate, old->name);
+ untracked_cache_remove_from_index(istate, old->name);
remove_index_entry_at(istate, nr);
add_index_entry(istate, new, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE);
}
@@ -270,20 +272,34 @@ static int ce_match_stat_basic(const struct cache_entry *ce, struct stat *st)
return changed;
}
-static int is_racy_timestamp(const struct index_state *istate,
- const struct cache_entry *ce)
+static int is_racy_stat(const struct index_state *istate,
+ const struct stat_data *sd)
{
- return (!S_ISGITLINK(ce->ce_mode) &&
- istate->timestamp.sec &&
+ return (istate->timestamp.sec &&
#ifdef USE_NSEC
/* nanosecond timestamped files can also be racy! */
- (istate->timestamp.sec < ce->ce_stat_data.sd_mtime.sec ||
- (istate->timestamp.sec == ce->ce_stat_data.sd_mtime.sec &&
- istate->timestamp.nsec <= ce->ce_stat_data.sd_mtime.nsec))
+ (istate->timestamp.sec < sd->sd_mtime.sec ||
+ (istate->timestamp.sec == sd->sd_mtime.sec &&
+ istate->timestamp.nsec <= sd->sd_mtime.nsec))
#else
- istate->timestamp.sec <= ce->ce_stat_data.sd_mtime.sec
+ istate->timestamp.sec <= sd->sd_mtime.sec
#endif
- );
+ );
+}
+
+static int is_racy_timestamp(const struct index_state *istate,
+ const struct cache_entry *ce)
+{
+ return (!S_ISGITLINK(ce->ce_mode) &&
+ is_racy_stat(istate, &ce->ce_stat_data));
+}
+
+int match_stat_data_racy(const struct index_state *istate,
+ const struct stat_data *sd, struct stat *st)
+{
+ if (is_racy_stat(istate, sd))
+ return MTIME_CHANGED;
+ return match_stat_data(sd, st);
}
int ie_match_stat(const struct index_state *istate,
@@ -538,6 +554,7 @@ int remove_file_from_index(struct index_state *istate, const char *path)
if (pos < 0)
pos = -pos-1;
cache_tree_invalidate_path(istate, path);
+ untracked_cache_remove_from_index(istate, path);
while (pos < istate->cache_nr && !strcmp(istate->cache[pos]->name, path))
remove_index_entry_at(istate, pos);
return 0;
@@ -982,6 +999,8 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e
}
pos = -pos-1;
+ untracked_cache_add_to_index(istate, ce->name);
+
/*
* Inserting a merged entry ("stage 0") into the index
* will always replace all non-merged entries..
@@ -1372,6 +1391,9 @@ static int read_index_extension(struct index_state *istate,
if (read_link_extension(istate, data, sz))
return -1;
break;
+ case CACHE_EXT_UNTRACKED:
+ istate->untracked = read_untracked_extension(data, sz);
+ break;
default:
if (*ext < 'A' || 'Z' < *ext)
return error("index uses %.4s extension, which we do not understand",
@@ -1667,6 +1689,8 @@ int discard_index(struct index_state *istate)
istate->cache = NULL;
istate->cache_alloc = 0;
discard_split_index(istate);
+ free_untracked_cache(istate->untracked);
+ istate->untracked = NULL;
return 0;
}
@@ -2053,6 +2077,17 @@ static int do_write_index(struct index_state *istate, int newfd,
if (err)
return -1;
}
+ if (!strip_extensions && istate->untracked) {
+ struct strbuf sb = STRBUF_INIT;
+
+ write_untracked_extension(&sb, istate->untracked);
+ err = write_index_ext_header(&c, newfd, CACHE_EXT_UNTRACKED,
+ sb.len) < 0 ||
+ ce_write(&c, newfd, sb.buf, sb.len) < 0;
+ strbuf_release(&sb);
+ if (err)
+ return -1;
+ }
if (ce_flush(&c, newfd, istate->sha1) || fstat(newfd, &st))
return -1;
diff --git a/split-index.c b/split-index.c
index 21485e2..968b780 100644
--- a/split-index.c
+++ b/split-index.c
@@ -41,13 +41,6 @@ int read_link_extension(struct index_state *istate,
return 0;
}
-static int write_strbuf(void *user_data, const void *data, size_t len)
-{
- struct strbuf *sb = user_data;
- strbuf_add(sb, data, len);
- return len;
-}
-
int write_link_extension(struct strbuf *sb,
struct index_state *istate)
{
@@ -55,8 +48,8 @@ int write_link_extension(struct strbuf *sb,
strbuf_add(sb, si->base_sha1, 20);
if (!si->delete_bitmap && !si->replace_bitmap)
return 0;
- ewah_serialize_to(si->delete_bitmap, write_strbuf, sb);
- ewah_serialize_to(si->replace_bitmap, write_strbuf, sb);
+ ewah_serialize_strbuf(si->delete_bitmap, sb);
+ ewah_serialize_strbuf(si->replace_bitmap, sb);
return 0;
}
diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh
new file mode 100755
index 0000000..2b2ffd7
--- /dev/null
+++ b/t/t7063-status-untracked-cache.sh
@@ -0,0 +1,353 @@
+#!/bin/sh
+
+test_description='test untracked cache'
+
+. ./test-lib.sh
+
+avoid_racy() {
+ sleep 1
+}
+
+git update-index --untracked-cache
+# It's fine if git update-index returns an error code other than one,
+# it'll be caught in the first test.
+if test $? -eq 1; then
+ skip_all='This system does not support untracked cache'
+ test_done
+fi
+
+test_expect_success 'setup' '
+ git init worktree &&
+ cd worktree &&
+ mkdir done dtwo dthree &&
+ touch one two three done/one dtwo/two dthree/three &&
+ git add one two done/one &&
+ : >.git/info/exclude &&
+ git update-index --untracked-cache
+'
+
+test_expect_success 'untracked cache is empty' '
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude 0000000000000000000000000000000000000000
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+EOF
+ test_cmp ../expect ../actual
+'
+
+cat >../status.expect <<EOF &&
+A done/one
+A one
+A two
+?? dthree/
+?? dtwo/
+?? three
+EOF
+
+cat >../dump.expect <<EOF &&
+info/exclude e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ 0000000000000000000000000000000000000000 recurse valid
+dthree/
+dtwo/
+three
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+three
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+
+test_expect_success 'status first time (empty cache)' '
+ avoid_racy &&
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 3
+gitignore invalidation: 1
+directory invalidation: 0
+opendir: 4
+EOF
+ test_cmp ../trace.expect ../trace
+'
+
+test_expect_success 'untracked cache after first status' '
+ test-dump-untracked-cache >../actual &&
+ test_cmp ../dump.expect ../actual
+'
+
+test_expect_success 'status second time (fully populated cache)' '
+ avoid_racy &&
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 0
+gitignore invalidation: 0
+directory invalidation: 0
+opendir: 0
+EOF
+ test_cmp ../trace.expect ../trace
+'
+
+test_expect_success 'untracked cache after second status' '
+ test-dump-untracked-cache >../actual &&
+ test_cmp ../dump.expect ../actual
+'
+
+test_expect_success 'modify in root directory, one dir invalidation' '
+ avoid_racy &&
+ : >four &&
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ cat >../status.expect <<EOF &&
+A done/one
+A one
+A two
+?? dthree/
+?? dtwo/
+?? four
+?? three
+EOF
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 0
+gitignore invalidation: 0
+directory invalidation: 1
+opendir: 1
+EOF
+ test_cmp ../trace.expect ../trace
+
+'
+
+test_expect_success 'verify untracked cache dump' '
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ 0000000000000000000000000000000000000000 recurse valid
+dthree/
+dtwo/
+four
+three
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+three
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_expect_success 'new .gitignore invalidates recursively' '
+ avoid_racy &&
+ echo four >.gitignore &&
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ cat >../status.expect <<EOF &&
+A done/one
+A one
+A two
+?? .gitignore
+?? dthree/
+?? dtwo/
+?? three
+EOF
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 0
+gitignore invalidation: 1
+directory invalidation: 1
+opendir: 4
+EOF
+ test_cmp ../trace.expect ../trace
+
+'
+
+test_expect_success 'verify untracked cache dump' '
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid
+.gitignore
+dthree/
+dtwo/
+three
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+three
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_expect_success 'new info/exclude invalidates everything' '
+ avoid_racy &&
+ echo three >>.git/info/exclude &&
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ cat >../status.expect <<EOF &&
+A done/one
+A one
+A two
+?? .gitignore
+?? dtwo/
+EOF
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 0
+gitignore invalidation: 1
+directory invalidation: 0
+opendir: 4
+EOF
+ test_cmp ../trace.expect ../trace
+'
+
+test_expect_success 'verify untracked cache dump' '
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid
+.gitignore
+dtwo/
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_expect_success 'move two from tracked to untracked' '
+ git rm --cached two &&
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_expect_success 'status after the move' '
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ cat >../status.expect <<EOF &&
+A done/one
+A one
+?? .gitignore
+?? dtwo/
+?? two
+EOF
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 0
+gitignore invalidation: 0
+directory invalidation: 0
+opendir: 1
+EOF
+ test_cmp ../trace.expect ../trace
+'
+
+test_expect_success 'verify untracked cache dump' '
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid
+.gitignore
+dtwo/
+two
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_expect_success 'move two from untracked to tracked' '
+ git add two &&
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_expect_success 'status after the move' '
+ : >../trace &&
+ GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \
+ git status --porcelain >../actual &&
+ cat >../status.expect <<EOF &&
+A done/one
+A one
+A two
+?? .gitignore
+?? dtwo/
+EOF
+ test_cmp ../status.expect ../actual &&
+ cat >../trace.expect <<EOF &&
+node creation: 0
+gitignore invalidation: 0
+directory invalidation: 0
+opendir: 1
+EOF
+ test_cmp ../trace.expect ../trace
+'
+
+test_expect_success 'verify untracked cache dump' '
+ test-dump-untracked-cache >../actual &&
+ cat >../expect <<EOF &&
+info/exclude 13263c0978fb9fad16b2d580fb800b6d811c3ff0
+core.excludesfile 0000000000000000000000000000000000000000
+exclude_per_dir .gitignore
+flags 00000006
+/ e6fcc8f2ee31bae321d66afd183fcb7237afae6e recurse valid
+.gitignore
+dtwo/
+/done/ 0000000000000000000000000000000000000000 recurse valid
+/dthree/ 0000000000000000000000000000000000000000 recurse check_only valid
+/dtwo/ 0000000000000000000000000000000000000000 recurse check_only valid
+two
+EOF
+ test_cmp ../expect ../actual
+'
+
+test_done
diff --git a/test-dump-untracked-cache.c b/test-dump-untracked-cache.c
new file mode 100644
index 0000000..25d855d
--- /dev/null
+++ b/test-dump-untracked-cache.c
@@ -0,0 +1,62 @@
+#include "cache.h"
+#include "dir.h"
+
+static int compare_untracked(const void *a_, const void *b_)
+{
+ const char *const *a = a_;
+ const char *const *b = b_;
+ return strcmp(*a, *b);
+}
+
+static int compare_dir(const void *a_, const void *b_)
+{
+ const struct untracked_cache_dir *const *a = a_;
+ const struct untracked_cache_dir *const *b = b_;
+ return strcmp((*a)->name, (*b)->name);
+}
+
+static void dump(struct untracked_cache_dir *ucd, struct strbuf *base)
+{
+ int i, len;
+ qsort(ucd->untracked, ucd->untracked_nr, sizeof(*ucd->untracked),
+ compare_untracked);
+ qsort(ucd->dirs, ucd->dirs_nr, sizeof(*ucd->dirs),
+ compare_dir);
+ len = base->len;
+ strbuf_addf(base, "%s/", ucd->name);
+ printf("%s %s", base->buf,
+ sha1_to_hex(ucd->exclude_sha1));
+ if (ucd->recurse)
+ fputs(" recurse", stdout);
+ if (ucd->check_only)
+ fputs(" check_only", stdout);
+ if (ucd->valid)
+ fputs(" valid", stdout);
+ printf("\n");
+ for (i = 0; i < ucd->untracked_nr; i++)
+ printf("%s\n", ucd->untracked[i]);
+ for (i = 0; i < ucd->dirs_nr; i++)
+ dump(ucd->dirs[i], base);
+ strbuf_setlen(base, len);
+}
+
+int main(int ac, char **av)
+{
+ struct untracked_cache *uc;
+ struct strbuf base = STRBUF_INIT;
+ setup_git_directory();
+ if (read_cache() < 0)
+ die("unable to read index file");
+ uc = the_index.untracked;
+ if (!uc) {
+ printf("no untracked cache\n");
+ return 0;
+ }
+ printf("info/exclude %s\n", sha1_to_hex(uc->ss_info_exclude.sha1));
+ printf("core.excludesfile %s\n", sha1_to_hex(uc->ss_excludes_file.sha1));
+ printf("exclude_per_dir %s\n", uc->exclude_per_dir);
+ printf("flags %08x\n", uc->dir_flags);
+ if (uc->root)
+ dump(uc->root, &base);
+ return 0;
+}
diff --git a/unpack-trees.c b/unpack-trees.c
index be84ba2..2927660 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -9,6 +9,7 @@
#include "refs.h"
#include "attr.h"
#include "split-index.h"
+#include "dir.h"
/*
* Error messages expected by scripts out of plumbing commands such as
@@ -1259,8 +1260,10 @@ static int verify_uptodate_sparse(const struct cache_entry *ce,
static void invalidate_ce_path(const struct cache_entry *ce,
struct unpack_trees_options *o)
{
- if (ce)
- cache_tree_invalidate_path(o->src_index, ce->name);
+ if (!ce)
+ return;
+ cache_tree_invalidate_path(o->src_index, ce->name);
+ untracked_cache_invalidate_path(o->src_index, ce->name);
}
/*
diff --git a/wt-status.c b/wt-status.c
index 38cb165..33452f1 100644
--- a/wt-status.c
+++ b/wt-status.c
@@ -585,6 +585,8 @@ static void wt_status_collect_untracked(struct wt_status *s)
DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES;
if (s->show_ignored_files)
dir.flags |= DIR_SHOW_IGNORED_TOO;
+ else
+ dir.untracked = the_index.untracked;
setup_standard_excludes(&dir);
fill_directory(&dir, &s->pathspec);