1 files changed, 233 insertions, 112 deletions

diff --git a/convert.c b/convert.c
index ab80b72..f524b8d 100644
--- a/convert.c
+++ b/convert.c
@@ -13,18 +13,25 @@
  * translation when the "text" attribute or "auto_crlf" option is set.
  */
 
+/* Stat bits: When BIN is set, the txt bits are unset */
+#define CONVERT_STAT_BITS_TXT_LF    0x1
+#define CONVERT_STAT_BITS_TXT_CRLF  0x2
+#define CONVERT_STAT_BITS_BIN       0x4
+
 enum crlf_action {
-	CRLF_GUESS = -1,
-	CRLF_BINARY = 0,
+	CRLF_UNDEFINED,
+	CRLF_BINARY,
 	CRLF_TEXT,
-	CRLF_INPUT,
-	CRLF_CRLF,
-	CRLF_AUTO
+	CRLF_TEXT_INPUT,
+	CRLF_TEXT_CRLF,
+	CRLF_AUTO,
+	CRLF_AUTO_INPUT,
+	CRLF_AUTO_CRLF
 };
 
 struct text_stat {
 	/* NUL, CR, LF and CRLF counts */
-	unsigned nul, cr, lf, crlf;
+	unsigned nul, lonecr, lonelf, crlf;
 
 	/* These are just approximations! */
 	unsigned printable, nonprintable;
@@ -39,13 +46,15 @@ static void gather_stats(const char *buf, unsigned long size, struct text_stat *
 	for (i = 0; i < size; i++) {
 		unsigned char c = buf[i];
 		if (c == '\r') {
-			stats->cr++;
-			if (i+1 < size && buf[i+1] == '\n')
+			if (i+1 < size && buf[i+1] == '\n') {
 				stats->crlf++;
+				i++;
+			} else
+				stats->lonecr++;
 			continue;
 		}
 		if (c == '\n') {
-			stats->lf++;
+			stats->lonelf++;
 			continue;
 		}
 		if (c == 127)
@@ -75,23 +84,84 @@ static void gather_stats(const char *buf, unsigned long size, struct text_stat *
 
 /*
  * The same heuristics as diff.c::mmfile_is_binary()
+ * We treat files with bare CR as binary
  */
-static int is_binary(unsigned long size, struct text_stat *stats)
+static int convert_is_binary(unsigned long size, const struct text_stat *stats)
 {
-
+	if (stats->lonecr)
+		return 1;
 	if (stats->nul)
 		return 1;
 	if ((stats->printable >> 7) < stats->nonprintable)
 		return 1;
-	/*
-	 * Other heuristics? Average line length might be relevant,
-	 * as might LF vs CR vs CRLF counts..
-	 *
-	 * NOTE! It might be normal to have a low ratio of CRLF to LF
-	 * (somebody starts with a LF-only file and edits it with an editor
-	 * that adds CRLF only to lines that are added..). But do  we
-	 * want to support CR-only? Probably not.
-	 */
+	return 0;
+}
+
+static unsigned int gather_convert_stats(const char *data, unsigned long size)
+{
+	struct text_stat stats;
+	int ret = 0;
+	if (!data || !size)
+		return 0;
+	gather_stats(data, size, &stats);
+	if (convert_is_binary(size, &stats))
+		ret |= CONVERT_STAT_BITS_BIN;
+	if (stats.crlf)
+		ret |= CONVERT_STAT_BITS_TXT_CRLF;
+	if (stats.lonelf)
+		ret |=  CONVERT_STAT_BITS_TXT_LF;
+
+	return ret;
+}
+
+static const char *gather_convert_stats_ascii(const char *data, unsigned long size)
+{
+	unsigned int convert_stats = gather_convert_stats(data, size);
+
+	if (convert_stats & CONVERT_STAT_BITS_BIN)
+		return "-text";
+	switch (convert_stats) {
+	case CONVERT_STAT_BITS_TXT_LF:
+		return "lf";
+	case CONVERT_STAT_BITS_TXT_CRLF:
+		return "crlf";
+	case CONVERT_STAT_BITS_TXT_LF | CONVERT_STAT_BITS_TXT_CRLF:
+		return "mixed";
+	default:
+		return "none";
+	}
+}
+
+const char *get_cached_convert_stats_ascii(const char *path)
+{
+	const char *ret;
+	unsigned long sz;
+	void *data = read_blob_data_from_cache(path, &sz);
+	ret = gather_convert_stats_ascii(data, sz);
+	free(data);
+	return ret;
+}
+
+const char *get_wt_convert_stats_ascii(const char *path)
+{
+	const char *ret = "";
+	struct strbuf sb = STRBUF_INIT;
+	if (strbuf_read_file(&sb, path, 0) >= 0)
+		ret = gather_convert_stats_ascii(sb.buf, sb.len);
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int text_eol_is_crlf(void)
+{
+	if (auto_crlf == AUTO_CRLF_TRUE)
+		return 1;
+	else if (auto_crlf == AUTO_CRLF_INPUT)
+		return 0;
+	if (core_eol == EOL_CRLF)
+		return 1;
+	if (core_eol == EOL_UNSET && EOL_NATIVE == EOL_CRLF)
+		return 1;
 	return 0;
 }
 
@@ -100,23 +170,19 @@ static enum eol output_eol(enum crlf_action crlf_action)
 	switch (crlf_action) {
 	case CRLF_BINARY:
 		return EOL_UNSET;
-	case CRLF_CRLF:
+	case CRLF_TEXT_CRLF:
 		return EOL_CRLF;
-	case CRLF_INPUT:
+	case CRLF_TEXT_INPUT:
 		return EOL_LF;
-	case CRLF_GUESS:
-		if (!auto_crlf)
-			return EOL_UNSET;
-		/* fall through */
+	case CRLF_UNDEFINED:
+	case CRLF_AUTO_CRLF:
+	case CRLF_AUTO_INPUT:
 	case CRLF_TEXT:
 	case CRLF_AUTO:
-		if (auto_crlf == AUTO_CRLF_TRUE)
-			return EOL_CRLF;
-		else if (auto_crlf == AUTO_CRLF_INPUT)
-			return EOL_LF;
-		else if (core_eol == EOL_UNSET)
-			return EOL_NATIVE;
+		/* fall through */
+		return text_eol_is_crlf() ? EOL_CRLF : EOL_LF;
 	}
+	warning("Illegal crlf_action %d\n", (int)crlf_action);
 	return core_eol;
 }
 
@@ -142,7 +208,7 @@ static void check_safe_crlf(const char *path, enum crlf_action crlf_action,
 		 * CRLFs would be added by checkout:
 		 * check if we have "naked" LFs
 		 */
-		if (stats->lf != stats->crlf) {
+		if (stats->lonelf) {
 			if (checksafe == SAFE_CRLF_WARN)
 				warning("LF will be replaced by CRLF in %s.\nThe file will have its original line endings in your working directory.", path);
 			else /* i.e. SAFE_CRLF_FAIL */
@@ -173,7 +239,6 @@ static int crlf_to_git(const char *path, const char *src, size_t len,
 	char *dst;
 
 	if (crlf_action == CRLF_BINARY ||
-	    (crlf_action == CRLF_GUESS && auto_crlf == AUTO_CRLF_FALSE) ||
 	    (src && !len))
 		return 0;
 
@@ -186,22 +251,11 @@ static int crlf_to_git(const char *path, const char *src, size_t len,
 
 	gather_stats(src, len, &stats);
 
-	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_GUESS) {
-		/*
-		 * We're currently not going to even try to convert stuff
-		 * that has bare CR characters. Does anybody do that crazy
-		 * stuff?
-		 */
-		if (stats.cr != stats.crlf)
-			return 0;
-
-		/*
-		 * And add some heuristics for binary vs text, of course...
-		 */
-		if (is_binary(len, &stats))
+	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
+		if (convert_is_binary(len, &stats))
 			return 0;
 
-		if (crlf_action == CRLF_GUESS) {
+		if (crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
 			/*
 			 * If the file in the index has any CR in it, do not convert.
 			 * This is the new safer autocrlf handling.
@@ -213,8 +267,8 @@ static int crlf_to_git(const char *path, const char *src, size_t len,
 
 	check_safe_crlf(path, crlf_action, &stats, checksafe);
 
-	/* Optimization: No CR? Nothing to convert, regardless. */
-	if (!stats.cr)
+	/* Optimization: No CRLF? Nothing to convert, regardless. */
+	if (!stats.crlf)
 		return 0;
 
 	/*
@@ -228,7 +282,7 @@ static int crlf_to_git(const char *path, const char *src, size_t len,
 	if (strbuf_avail(buf) + buf->len < len)
 		strbuf_grow(buf, len - buf->len);
 	dst = buf->buf;
-	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_GUESS) {
+	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
 		/*
 		 * If we guessed, we already know we rejected a file with
 		 * lone CR, and we can strip a CR without looking at what
@@ -261,27 +315,19 @@ static int crlf_to_worktree(const char *path, const char *src, size_t len,
 
 	gather_stats(src, len, &stats);
 
-	/* No LF? Nothing to convert, regardless. */
-	if (!stats.lf)
+	/* No "naked" LF? Nothing to convert, regardless. */
+	if (!stats.lonelf)
 		return 0;
 
-	/* Was it already in CRLF format? */
-	if (stats.lf == stats.crlf)
-		return 0;
-
-	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_GUESS) {
-		if (crlf_action == CRLF_GUESS) {
+	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
+		if (crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
 			/* If we have any CR or CRLF line endings, we do not touch it */
 			/* This is the new safer autocrlf-handling */
-			if (stats.cr > 0 || stats.crlf > 0)
+			if (stats.lonecr || stats.crlf )
 				return 0;
 		}
 
-		/* If we have any bare CR characters, we're not going to touch it */
-		if (stats.cr != stats.crlf)
-			return 0;
-
-		if (is_binary(len, &stats))
+		if (convert_is_binary(len, &stats))
 			return 0;
 	}
 
@@ -289,7 +335,7 @@ static int crlf_to_worktree(const char *path, const char *src, size_t len,
 	if (src == buf->buf)
 		to_free = strbuf_detach(buf, NULL);
 
-	strbuf_grow(buf, len + stats.lf - stats.crlf);
+	strbuf_grow(buf, len + stats.lonelf);
 	for (;;) {
 		const char *nl = memchr(src, '\n', len);
 		if (!nl)
@@ -312,16 +358,17 @@ static int crlf_to_worktree(const char *path, const char *src, size_t len,
 struct filter_params {
 	const char *src;
 	unsigned long size;
+	int fd;
 	const char *cmd;
 	const char *path;
 };
 
-static int filter_buffer(int in, int out, void *data)
+static int filter_buffer_or_fd(int in, int out, void *data)
 {
 	/*
 	 * Spawn cmd and feed the buffer contents through its stdin.
 	 */
-	struct child_process child_process;
+	struct child_process child_process = CHILD_PROCESS_INIT;
 	struct filter_params *params = (struct filter_params *)data;
 	int write_err, status;
 	const char *argv[] = { NULL, NULL };
@@ -344,7 +391,6 @@ static int filter_buffer(int in, int out, void *data)
 
 	argv[0] = cmd.buf;
 
-	memset(&child_process, 0, sizeof(child_process));
 	child_process.argv = argv;
 	child_process.use_shell = 1;
 	child_process.in = -1;
@@ -355,7 +401,17 @@ static int filter_buffer(int in, int out, void *data)
 
 	sigchain_push(SIGPIPE, SIG_IGN);
 
-	write_err = (write_in_full(child_process.in, params->src, params->size) < 0);
+	if (params->src) {
+		write_err = (write_in_full(child_process.in,
+					   params->src, params->size) < 0);
+		if (errno == EPIPE)
+			write_err = 0;
+	} else {
+		write_err = copy_fd(params->fd, child_process.in);
+		if (write_err == COPY_WRITE_ERROR && errno == EPIPE)
+			write_err = 0;
+	}
+
 	if (close(child_process.in))
 		write_err = 1;
 	if (write_err)
@@ -371,7 +427,7 @@ static int filter_buffer(int in, int out, void *data)
 	return (write_err || status);
 }
 
-static int apply_filter(const char *path, const char *src, size_t len,
+static int apply_filter(const char *path, const char *src, size_t len, int fd,
                         struct strbuf *dst, const char *cmd)
 {
 	/*
@@ -385,18 +441,19 @@ static int apply_filter(const char *path, const char *src, size_t len,
 	struct async async;
 	struct filter_params params;
 
-	if (!cmd)
+	if (!cmd || !*cmd)
 		return 0;
 
 	if (!dst)
 		return 1;
 
 	memset(&async, 0, sizeof(async));
-	async.proc = filter_buffer;
+	async.proc = filter_buffer_or_fd;
 	async.data = &params;
 	async.out = -1;
 	params.src = src;
 	params.size = len;
+	params.fd = fd;
 	params.cmd = cmd;
 	params.path = path;
 
@@ -646,7 +703,7 @@ static int ident_to_worktree(const char *path, const char *src, size_t len,
 	return 1;
 }
 
-static enum crlf_action git_path_check_crlf(const char *path, struct git_attr_check *check)
+static enum crlf_action git_path_check_crlf(struct git_attr_check *check)
 {
 	const char *value = check->value;
 
@@ -657,13 +714,13 @@ static enum crlf_action git_path_check_crlf(const char *path, struct git_attr_ch
 	else if (ATTR_UNSET(value))
 		;
 	else if (!strcmp(value, "input"))
-		return CRLF_INPUT;
+		return CRLF_TEXT_INPUT;
 	else if (!strcmp(value, "auto"))
 		return CRLF_AUTO;
-	return CRLF_GUESS;
+	return CRLF_UNDEFINED;
 }
 
-static enum eol git_path_check_eol(const char *path, struct git_attr_check *check)
+static enum eol git_path_check_eol(struct git_attr_check *check)
 {
 	const char *value = check->value;
 
@@ -676,8 +733,7 @@ static enum eol git_path_check_eol(const char *path, struct git_attr_check *chec
 	return EOL_UNSET;
 }
 
-static struct convert_driver *git_path_check_convert(const char *path,
-					     struct git_attr_check *check)
+static struct convert_driver *git_path_check_convert(struct git_attr_check *check)
 {
 	const char *value = check->value;
 	struct convert_driver *drv;
@@ -690,28 +746,17 @@ static struct convert_driver *git_path_check_convert(const char *path,
 	return NULL;
 }
 
-static int git_path_check_ident(const char *path, struct git_attr_check *check)
+static int git_path_check_ident(struct git_attr_check *check)
 {
 	const char *value = check->value;
 
 	return !!ATTR_TRUE(value);
 }
 
-static enum crlf_action input_crlf_action(enum crlf_action text_attr, enum eol eol_attr)
-{
-	if (text_attr == CRLF_BINARY)
-		return CRLF_BINARY;
-	if (eol_attr == EOL_LF)
-		return CRLF_INPUT;
-	if (eol_attr == EOL_CRLF)
-		return CRLF_CRLF;
-	return text_attr;
-}
-
 struct conv_attrs {
 	struct convert_driver *drv;
-	enum crlf_action crlf_action;
-	enum eol eol_attr;
+	enum crlf_action attr_action; /* What attr says */
+	enum crlf_action crlf_action; /* When no attr is set, use core.autocrlf */
 	int ident;
 };
 
@@ -733,18 +778,78 @@ static void convert_attrs(struct conv_attrs *ca, const char *path)
 	}
 
 	if (!git_check_attr(path, NUM_CONV_ATTRS, ccheck)) {
-		ca->crlf_action = git_path_check_crlf(path, ccheck + 4);
-		if (ca->crlf_action == CRLF_GUESS)
-			ca->crlf_action = git_path_check_crlf(path, ccheck + 0);
-		ca->ident = git_path_check_ident(path, ccheck + 1);
-		ca->drv = git_path_check_convert(path, ccheck + 2);
-		ca->eol_attr = git_path_check_eol(path, ccheck + 3);
+		ca->crlf_action = git_path_check_crlf(ccheck + 4);
+		if (ca->crlf_action == CRLF_UNDEFINED)
+			ca->crlf_action = git_path_check_crlf(ccheck + 0);
+		ca->attr_action = ca->crlf_action;
+		ca->ident = git_path_check_ident(ccheck + 1);
+		ca->drv = git_path_check_convert(ccheck + 2);
+		if (ca->crlf_action != CRLF_BINARY) {
+			enum eol eol_attr = git_path_check_eol(ccheck + 3);
+			if (eol_attr == EOL_LF)
+				ca->crlf_action = CRLF_TEXT_INPUT;
+			else if (eol_attr == EOL_CRLF)
+				ca->crlf_action = CRLF_TEXT_CRLF;
+		}
+		ca->attr_action = ca->crlf_action;
 	} else {
 		ca->drv = NULL;
-		ca->crlf_action = CRLF_GUESS;
-		ca->eol_attr = EOL_UNSET;
+		ca->crlf_action = CRLF_UNDEFINED;
 		ca->ident = 0;
 	}
+	if (ca->crlf_action == CRLF_TEXT)
+		ca->crlf_action = text_eol_is_crlf() ? CRLF_TEXT_CRLF : CRLF_TEXT_INPUT;
+	if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_FALSE)
+		ca->crlf_action = CRLF_BINARY;
+	if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_TRUE)
+		ca->crlf_action = CRLF_AUTO_CRLF;
+	if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_INPUT)
+		ca->crlf_action = CRLF_AUTO_INPUT;
+}
+
+int would_convert_to_git_filter_fd(const char *path)
+{
+	struct conv_attrs ca;
+
+	convert_attrs(&ca, path);
+	if (!ca.drv)
+		return 0;
+
+	/*
+	 * Apply a filter to an fd only if the filter is required to succeed.
+	 * We must die if the filter fails, because the original data before
+	 * filtering is not available.
+	 */
+	if (!ca.drv->required)
+		return 0;
+
+	return apply_filter(path, NULL, 0, -1, NULL, ca.drv->clean);
+}
+
+const char *get_convert_attr_ascii(const char *path)
+{
+	struct conv_attrs ca;
+
+	convert_attrs(&ca, path);
+	switch (ca.attr_action) {
+	case CRLF_UNDEFINED:
+		return "";
+	case CRLF_BINARY:
+		return "-text";
+	case CRLF_TEXT:
+		return "text";
+	case CRLF_TEXT_INPUT:
+		return "text eol=lf";
+	case CRLF_TEXT_CRLF:
+		return "text eol=crlf";
+	case CRLF_AUTO:
+		return "text=auto";
+	case CRLF_AUTO_CRLF:
+		return "text=auto eol=crlf"; /* This is not supported yet */
+	case CRLF_AUTO_INPUT:
+		return "text=auto eol=lf"; /* This is not supported yet */
+	}
+	return "";
 }
 
 int convert_to_git(const char *path, const char *src, size_t len,
@@ -761,7 +866,7 @@ int convert_to_git(const char *path, const char *src, size_t len,
 		required = ca.drv->required;
 	}
 
-	ret |= apply_filter(path, src, len, dst, filter);
+	ret |= apply_filter(path, src, len, -1, dst, filter);
 	if (!ret && required)
 		die("%s: clean filter '%s' failed", path, ca.drv->name);
 
@@ -769,7 +874,6 @@ int convert_to_git(const char *path, const char *src, size_t len,
 		src = dst->buf;
 		len = dst->len;
 	}
-	ca.crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
 	ret |= crlf_to_git(path, src, len, dst, ca.crlf_action, checksafe);
 	if (ret && dst) {
 		src = dst->buf;
@@ -778,6 +882,22 @@ int convert_to_git(const char *path, const char *src, size_t len,
 	return ret | ident_to_git(path, src, len, dst, ca.ident);
 }
 
+void convert_to_git_filter_fd(const char *path, int fd, struct strbuf *dst,
+			      enum safe_crlf checksafe)
+{
+	struct conv_attrs ca;
+	convert_attrs(&ca, path);
+
+	assert(ca.drv);
+	assert(ca.drv->clean);
+
+	if (!apply_filter(path, NULL, 0, fd, dst, ca.drv->clean))
+		die("%s: clean filter '%s' failed", path, ca.drv->name);
+
+	crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
+	ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
+}
+
 static int convert_to_working_tree_internal(const char *path, const char *src,
 					    size_t len, struct strbuf *dst,
 					    int normalizing)
@@ -803,7 +923,6 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
 	 * is a smudge filter.  The filter might expect CRLFs.
 	 */
 	if (filter || !normalizing) {
-		ca.crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
 		ret |= crlf_to_worktree(path, src, len, dst, ca.crlf_action);
 		if (ret) {
 			src = dst->buf;
@@ -811,7 +930,7 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
 		}
 	}
 
-	ret_filter = apply_filter(path, src, len, dst, filter);
+	ret_filter = apply_filter(path, src, len, -1, dst, filter);
 	if (!ret_filter && required)
 		die("%s: smudge filter %s failed", path, ca.drv->name);
 
@@ -1121,9 +1240,9 @@ static int is_foreign_ident(const char *str)
 {
 	int i;
 
-	if (!starts_with(str, "$Id: "))
+	if (!skip_prefix(str, "$Id: ", &str))
 		return 0;
-	for (i = 5; str[i]; i++) {
+	for (i = 0; str[i]; i++) {
 		if (isspace(str[i]) && str[i+1] != '$')
 			return 1;
 	}
@@ -1242,7 +1361,8 @@ static struct stream_filter *ident_filter(const unsigned char *sha1)
 {
 	struct ident_filter *ident = xmalloc(sizeof(*ident));
 
-	sprintf(ident->ident, ": %s $", sha1_to_hex(sha1));
+	xsnprintf(ident->ident, sizeof(ident->ident),
+		  ": %s $", sha1_to_hex(sha1));
 	strbuf_init(&ident->left, 0);
 	ident->filter.vtbl = &ident_vtbl;
 	ident->state = 0;
@@ -1271,14 +1391,15 @@ struct stream_filter *get_stream_filter(const char *path, const unsigned char *s
 	if (ca.ident)
 		filter = ident_filter(sha1);
 
-	crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
+	crlf_action = ca.crlf_action;
 
-	if ((crlf_action == CRLF_BINARY) || (crlf_action == CRLF_INPUT) ||
-	    (crlf_action == CRLF_GUESS && auto_crlf == AUTO_CRLF_FALSE))
+	if ((crlf_action == CRLF_BINARY) ||
+			crlf_action == CRLF_AUTO_INPUT ||
+			(crlf_action == CRLF_TEXT_INPUT))
 		filter = cascade_filter(filter, &null_filter_singleton);
 
 	else if (output_eol(crlf_action) == EOL_CRLF &&
-		 !(crlf_action == CRLF_AUTO || crlf_action == CRLF_GUESS))
+		 !(crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_CRLF))
 		filter = cascade_filter(filter, lf_to_crlf_filter());
 
 	return filter;