From 9430147ca0aab0189d7e52df97b95a0985fc0c8a Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:05 -0700 Subject: list-objects-filter: encapsulate filter components Encapsulate filter_fn, filter_free_fn, and filter_data into their own opaque struct. Due to opaqueness, filter_fn and filter_free_fn can no longer be accessed directly by users. Currently, all usages of filter_fn are guarded by a necessary check: (obj->flags & NOT_USER_GIVEN) && filter_fn Take the opportunity to include this check into the new function list_objects_filter__filter_object(), so that we no longer need to write this check at every caller of the filter function. Also, the init functions in list-objects-filter.c no longer need to confusingly return the filter constituents in various places (filter_fn and filter_free_fn as out parameters, and filter_data as the function's return value); they can just initialize the "struct filter" passed in. Helped-by: Jeff Hostetler Helped-by: Jonathan Tan Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/list-objects-filter.c b/list-objects-filter.c index 36e1f77..e06b82d 100644 --- a/list-objects-filter.c +++ b/list-objects-filter.c @@ -26,6 +26,20 @@ */ #define FILTER_SHOWN_BUT_REVISIT (1<<21) +struct filter { + enum list_objects_filter_result (*filter_object_fn)( + struct repository *r, + enum list_objects_filter_situation filter_situation, + struct object *obj, + const char *pathname, + const char *filename, + void *filter_data); + + void (*free_fn)(void *filter_data); + + void *filter_data; +}; + /* * A filter for list-objects to omit ALL blobs from the traversal. * And to OPTIONALLY collect a list of the omitted OIDs. @@ -67,18 +81,17 @@ static enum list_objects_filter_result filter_blobs_none( } } -static void *filter_blobs_none__init( +static void filter_blobs_none__init( struct oidset *omitted, struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn) + struct filter *filter) { struct filter_blobs_none_data *d = xcalloc(1, sizeof(*d)); d->omits = omitted; - *filter_fn = filter_blobs_none; - *filter_free_fn = free; - return d; + filter->filter_data = d; + filter->filter_object_fn = filter_blobs_none; + filter->free_fn = free; } /* @@ -201,11 +214,10 @@ static void filter_trees_free(void *filter_data) { free(d); } -static void *filter_trees_depth__init( +static void filter_trees_depth__init( struct oidset *omitted, struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn) + struct filter *filter) { struct filter_trees_depth_data *d = xcalloc(1, sizeof(*d)); d->omits = omitted; @@ -213,9 +225,9 @@ static void *filter_trees_depth__init( d->exclude_depth = filter_options->tree_exclude_depth; d->current_depth = 0; - *filter_fn = filter_trees_depth; - *filter_free_fn = filter_trees_free; - return d; + filter->filter_data = d; + filter->filter_object_fn = filter_trees_depth; + filter->free_fn = filter_trees_free; } /* @@ -281,19 +293,18 @@ include_it: return LOFR_MARK_SEEN | LOFR_DO_SHOW; } -static void *filter_blobs_limit__init( +static void filter_blobs_limit__init( struct oidset *omitted, struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn) + struct filter *filter) { struct filter_blobs_limit_data *d = xcalloc(1, sizeof(*d)); d->omits = omitted; d->max_bytes = filter_options->blob_limit_value; - *filter_fn = filter_blobs_limit; - *filter_free_fn = free; - return d; + filter->filter_data = d; + filter->filter_object_fn = filter_blobs_limit; + filter->free_fn = free; } /* @@ -456,11 +467,10 @@ static void filter_sparse_free(void *filter_data) free(d); } -static void *filter_sparse_oid__init( +static void filter_sparse_oid__init( struct oidset *omitted, struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn) + struct filter *filter) { struct filter_sparse_data *d = xcalloc(1, sizeof(*d)); d->omits = omitted; @@ -473,16 +483,15 @@ static void *filter_sparse_oid__init( d->array_frame[d->nr].child_prov_omit = 0; d->nr++; - *filter_fn = filter_sparse; - *filter_free_fn = filter_sparse_free; - return d; + filter->filter_data = d; + filter->filter_object_fn = filter_sparse; + filter->free_fn = filter_sparse_free; } -typedef void *(*filter_init_fn)( +typedef void (*filter_init_fn)( struct oidset *omitted, struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn); + struct filter *filter); /* * Must match "enum list_objects_filter_choice". @@ -495,12 +504,11 @@ static filter_init_fn s_filters[] = { filter_sparse_oid__init, }; -void *list_objects_filter__init( +struct filter *list_objects_filter__init( struct oidset *omitted, - struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn) + struct list_objects_filter_options *filter_options) { + struct filter *filter; filter_init_fn init_fn; assert((sizeof(s_filters) / sizeof(s_filters[0])) == LOFC__COUNT); @@ -510,10 +518,40 @@ void *list_objects_filter__init( filter_options->choice); init_fn = s_filters[filter_options->choice]; - if (init_fn) - return init_fn(omitted, filter_options, - filter_fn, filter_free_fn); - *filter_fn = NULL; - *filter_free_fn = NULL; - return NULL; + if (!init_fn) + return NULL; + + filter = xcalloc(1, sizeof(*filter)); + init_fn(omitted, filter_options, filter); + return filter; +} + +enum list_objects_filter_result list_objects_filter__filter_object( + struct repository *r, + enum list_objects_filter_situation filter_situation, + struct object *obj, + const char *pathname, + const char *filename, + struct filter *filter) +{ + if (filter && (obj->flags & NOT_USER_GIVEN)) + return filter->filter_object_fn(r, filter_situation, obj, + pathname, filename, + filter->filter_data); + /* + * No filter is active or user gave object explicitly. In this case, + * always show the object (except when LOFS_END_TREE, since this tree + * had already been shown when LOFS_BEGIN_TREE). + */ + if (filter_situation == LOFS_END_TREE) + return 0; + return LOFR_MARK_SEEN | LOFR_DO_SHOW; +} + +void list_objects_filter__free(struct filter *filter) +{ + if (!filter) + return; + filter->free_fn(filter->filter_data); + free(filter); } diff --git a/list-objects-filter.h b/list-objects-filter.h index 1d45a4a..6908954 100644 --- a/list-objects-filter.h +++ b/list-objects-filter.h @@ -60,30 +60,27 @@ enum list_objects_filter_situation { LOFS_BLOB }; -typedef enum list_objects_filter_result (*filter_object_fn)( +struct filter; + +/* Constructor for the set of defined list-objects filters. */ +struct filter *list_objects_filter__init( + struct oidset *omitted, + struct list_objects_filter_options *filter_options); + +/* + * Lets `filter` decide how to handle the `obj`. If `filter` is NULL, this + * function behaves as expected if no filter is configured: all objects are + * included. + */ +enum list_objects_filter_result list_objects_filter__filter_object( struct repository *r, enum list_objects_filter_situation filter_situation, struct object *obj, const char *pathname, const char *filename, - void *filter_data); - -typedef void (*filter_free_fn)(void *filter_data); + struct filter *filter); -/* - * Constructor for the set of defined list-objects filters. - * Returns a generic "void *filter_data". - * - * The returned "filter_fn" will be used by traverse_commit_list() - * to filter the results. - * - * The returned "filter_free_fn" is a destructor for the - * filter_data. - */ -void *list_objects_filter__init( - struct oidset *omitted, - struct list_objects_filter_options *filter_options, - filter_object_fn *filter_fn, - filter_free_fn *filter_free_fn); +/* Destroys `filter`. Does nothing if `filter` is null. */ +void list_objects_filter__free(struct filter *filter); #endif /* LIST_OBJECTS_FILTER_H */ diff --git a/list-objects.c b/list-objects.c index b5651dd..9307d91 100644 --- a/list-objects.c +++ b/list-objects.c @@ -18,8 +18,7 @@ struct traversal_context { show_object_fn show_object; show_commit_fn show_commit; void *show_data; - filter_object_fn filter_fn; - void *filter_data; + struct filter *filter; }; static void process_blob(struct traversal_context *ctx, @@ -29,7 +28,7 @@ static void process_blob(struct traversal_context *ctx, { struct object *obj = &blob->object; size_t pathlen; - enum list_objects_filter_result r = LOFR_MARK_SEEN | LOFR_DO_SHOW; + enum list_objects_filter_result r; if (!ctx->revs->blob_objects) return; @@ -54,11 +53,10 @@ static void process_blob(struct traversal_context *ctx, pathlen = path->len; strbuf_addstr(path, name); - if ((obj->flags & NOT_USER_GIVEN) && ctx->filter_fn) - r = ctx->filter_fn(ctx->revs->repo, - LOFS_BLOB, obj, - path->buf, &path->buf[pathlen], - ctx->filter_data); + r = list_objects_filter__filter_object(ctx->revs->repo, + LOFS_BLOB, obj, + path->buf, &path->buf[pathlen], + ctx->filter); if (r & LOFR_MARK_SEEN) obj->flags |= SEEN; if (r & LOFR_DO_SHOW) @@ -157,7 +155,7 @@ static void process_tree(struct traversal_context *ctx, struct object *obj = &tree->object; struct rev_info *revs = ctx->revs; int baselen = base->len; - enum list_objects_filter_result r = LOFR_MARK_SEEN | LOFR_DO_SHOW; + enum list_objects_filter_result r; int failed_parse; if (!revs->tree_objects) @@ -186,11 +184,10 @@ static void process_tree(struct traversal_context *ctx, } strbuf_addstr(base, name); - if ((obj->flags & NOT_USER_GIVEN) && ctx->filter_fn) - r = ctx->filter_fn(ctx->revs->repo, - LOFS_BEGIN_TREE, obj, - base->buf, &base->buf[baselen], - ctx->filter_data); + r = list_objects_filter__filter_object(ctx->revs->repo, + LOFS_BEGIN_TREE, obj, + base->buf, &base->buf[baselen], + ctx->filter); if (r & LOFR_MARK_SEEN) obj->flags |= SEEN; if (r & LOFR_DO_SHOW) @@ -203,16 +200,14 @@ static void process_tree(struct traversal_context *ctx, else if (!failed_parse) process_tree_contents(ctx, tree, base); - if ((obj->flags & NOT_USER_GIVEN) && ctx->filter_fn) { - r = ctx->filter_fn(ctx->revs->repo, - LOFS_END_TREE, obj, - base->buf, &base->buf[baselen], - ctx->filter_data); - if (r & LOFR_MARK_SEEN) - obj->flags |= SEEN; - if (r & LOFR_DO_SHOW) - ctx->show_object(obj, base->buf, ctx->show_data); - } + r = list_objects_filter__filter_object(ctx->revs->repo, + LOFS_END_TREE, obj, + base->buf, &base->buf[baselen], + ctx->filter); + if (r & LOFR_MARK_SEEN) + obj->flags |= SEEN; + if (r & LOFR_DO_SHOW) + ctx->show_object(obj, base->buf, ctx->show_data); strbuf_setlen(base, baselen); free_tree_buffer(tree); @@ -402,8 +397,7 @@ void traverse_commit_list(struct rev_info *revs, ctx.show_commit = show_commit; ctx.show_object = show_object; ctx.show_data = show_data; - ctx.filter_fn = NULL; - ctx.filter_data = NULL; + ctx.filter = NULL; do_traverse(&ctx); } @@ -416,17 +410,12 @@ void traverse_commit_list_filtered( struct oidset *omitted) { struct traversal_context ctx; - filter_free_fn filter_free_fn = NULL; ctx.revs = revs; ctx.show_object = show_object; ctx.show_commit = show_commit; ctx.show_data = show_data; - ctx.filter_fn = NULL; - - ctx.filter_data = list_objects_filter__init(omitted, filter_options, - &ctx.filter_fn, &filter_free_fn); + ctx.filter = list_objects_filter__init(omitted, filter_options); do_traverse(&ctx); - if (ctx.filter_data && filter_free_fn) - filter_free_fn(ctx.filter_data); + list_objects_filter__free(ctx.filter); } -- cgit v0.10.2-6-g49f6 From 7a7c7f4a6d22477e3548021eb3571384651c00be Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:06 -0700 Subject: list-objects-filter: put omits set in filter struct The oidset *omits pointer must be accessed by the combine filter in a type-agnostic way once the graph traversal is over. Store that pointer in the general `filter` struct. This will be used in a follow-up patch to implement the combine filter. Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/list-objects-filter.c b/list-objects-filter.c index e06b82d..3b4b676 100644 --- a/list-objects-filter.c +++ b/list-objects-filter.c @@ -33,18 +33,14 @@ struct filter { struct object *obj, const char *pathname, const char *filename, + struct oidset *omits, void *filter_data); void (*free_fn)(void *filter_data); void *filter_data; -}; -/* - * A filter for list-objects to omit ALL blobs from the traversal. - * And to OPTIONALLY collect a list of the omitted OIDs. - */ -struct filter_blobs_none_data { + /* If non-NULL, the filter collects a list of the omitted OIDs here. */ struct oidset *omits; }; @@ -54,10 +50,9 @@ static enum list_objects_filter_result filter_blobs_none( struct object *obj, const char *pathname, const char *filename, + struct oidset *omits, void *filter_data_) { - struct filter_blobs_none_data *filter_data = filter_data_; - switch (filter_situation) { default: BUG("unknown filter_situation: %d", filter_situation); @@ -75,21 +70,16 @@ static enum list_objects_filter_result filter_blobs_none( assert(obj->type == OBJ_BLOB); assert((obj->flags & SEEN) == 0); - if (filter_data->omits) - oidset_insert(filter_data->omits, &obj->oid); + if (omits) + oidset_insert(omits, &obj->oid); return LOFR_MARK_SEEN; /* but not LOFR_DO_SHOW (hard omit) */ } } static void filter_blobs_none__init( - struct oidset *omitted, struct list_objects_filter_options *filter_options, struct filter *filter) { - struct filter_blobs_none_data *d = xcalloc(1, sizeof(*d)); - d->omits = omitted; - - filter->filter_data = d; filter->filter_object_fn = filter_blobs_none; filter->free_fn = free; } @@ -99,8 +89,6 @@ static void filter_blobs_none__init( * Can OPTIONALLY collect a list of the omitted OIDs. */ struct filter_trees_depth_data { - struct oidset *omits; - /* * Maps trees to the minimum depth at which they were seen. It is not * necessary to re-traverse a tree at deeper or equal depths than it has @@ -123,16 +111,16 @@ struct seen_map_entry { /* Returns 1 if the oid was in the omits set before it was invoked. */ static int filter_trees_update_omits( struct object *obj, - struct filter_trees_depth_data *filter_data, + struct oidset *omits, int include_it) { - if (!filter_data->omits) + if (!omits) return 0; if (include_it) - return oidset_remove(filter_data->omits, &obj->oid); + return oidset_remove(omits, &obj->oid); else - return oidset_insert(filter_data->omits, &obj->oid); + return oidset_insert(omits, &obj->oid); } static enum list_objects_filter_result filter_trees_depth( @@ -141,6 +129,7 @@ static enum list_objects_filter_result filter_trees_depth( struct object *obj, const char *pathname, const char *filename, + struct oidset *omits, void *filter_data_) { struct filter_trees_depth_data *filter_data = filter_data_; @@ -165,7 +154,7 @@ static enum list_objects_filter_result filter_trees_depth( return LOFR_ZERO; case LOFS_BLOB: - filter_trees_update_omits(obj, filter_data, include_it); + filter_trees_update_omits(obj, omits, include_it); return include_it ? LOFR_MARK_SEEN | LOFR_DO_SHOW : LOFR_ZERO; case LOFS_BEGIN_TREE: @@ -186,12 +175,12 @@ static enum list_objects_filter_result filter_trees_depth( filter_res = LOFR_SKIP_TREE; } else { int been_omitted = filter_trees_update_omits( - obj, filter_data, include_it); + obj, omits, include_it); seen_info->depth = filter_data->current_depth; if (include_it) filter_res = LOFR_DO_SHOW; - else if (filter_data->omits && !been_omitted) + else if (omits && !been_omitted) /* * Must update omit information of children * recursively; they have not been omitted yet. @@ -215,12 +204,10 @@ static void filter_trees_free(void *filter_data) { } static void filter_trees_depth__init( - struct oidset *omitted, struct list_objects_filter_options *filter_options, struct filter *filter) { struct filter_trees_depth_data *d = xcalloc(1, sizeof(*d)); - d->omits = omitted; oidmap_init(&d->seen_at_depth, 0); d->exclude_depth = filter_options->tree_exclude_depth; d->current_depth = 0; @@ -235,7 +222,6 @@ static void filter_trees_depth__init( * And to OPTIONALLY collect a list of the omitted OIDs. */ struct filter_blobs_limit_data { - struct oidset *omits; unsigned long max_bytes; }; @@ -245,6 +231,7 @@ static enum list_objects_filter_result filter_blobs_limit( struct object *obj, const char *pathname, const char *filename, + struct oidset *omits, void *filter_data_) { struct filter_blobs_limit_data *filter_data = filter_data_; @@ -282,24 +269,22 @@ static enum list_objects_filter_result filter_blobs_limit( if (object_length < filter_data->max_bytes) goto include_it; - if (filter_data->omits) - oidset_insert(filter_data->omits, &obj->oid); + if (omits) + oidset_insert(omits, &obj->oid); return LOFR_MARK_SEEN; /* but not LOFR_DO_SHOW (hard omit) */ } include_it: - if (filter_data->omits) - oidset_remove(filter_data->omits, &obj->oid); + if (omits) + oidset_remove(omits, &obj->oid); return LOFR_MARK_SEEN | LOFR_DO_SHOW; } static void filter_blobs_limit__init( - struct oidset *omitted, struct list_objects_filter_options *filter_options, struct filter *filter) { struct filter_blobs_limit_data *d = xcalloc(1, sizeof(*d)); - d->omits = omitted; d->max_bytes = filter_options->blob_limit_value; filter->filter_data = d; @@ -337,7 +322,6 @@ struct frame { }; struct filter_sparse_data { - struct oidset *omits; struct exclude_list el; size_t nr, alloc; @@ -350,6 +334,7 @@ static enum list_objects_filter_result filter_sparse( struct object *obj, const char *pathname, const char *filename, + struct oidset *omits, void *filter_data_) { struct filter_sparse_data *filter_data = filter_data_; @@ -431,8 +416,8 @@ static enum list_objects_filter_result filter_sparse( if (val < 0) val = frame->defval; if (val > 0) { - if (filter_data->omits) - oidset_remove(filter_data->omits, &obj->oid); + if (omits) + oidset_remove(omits, &obj->oid); return LOFR_MARK_SEEN | LOFR_DO_SHOW; } @@ -446,8 +431,8 @@ static enum list_objects_filter_result filter_sparse( * Leave the LOFR_ bits unset so that if the blob appears * again in the traversal, we will be asked again. */ - if (filter_data->omits) - oidset_insert(filter_data->omits, &obj->oid); + if (omits) + oidset_insert(omits, &obj->oid); /* * Remember that at least 1 blob in this tree was @@ -468,12 +453,10 @@ static void filter_sparse_free(void *filter_data) } static void filter_sparse_oid__init( - struct oidset *omitted, struct list_objects_filter_options *filter_options, struct filter *filter) { struct filter_sparse_data *d = xcalloc(1, sizeof(*d)); - d->omits = omitted; if (add_excludes_from_blob_to_list(filter_options->sparse_oid_value, NULL, 0, &d->el) < 0) die("could not load filter specification"); @@ -489,7 +472,6 @@ static void filter_sparse_oid__init( } typedef void (*filter_init_fn)( - struct oidset *omitted, struct list_objects_filter_options *filter_options, struct filter *filter); @@ -522,7 +504,8 @@ struct filter *list_objects_filter__init( return NULL; filter = xcalloc(1, sizeof(*filter)); - init_fn(omitted, filter_options, filter); + filter->omits = omitted; + init_fn(filter_options, filter); return filter; } @@ -537,6 +520,7 @@ enum list_objects_filter_result list_objects_filter__filter_object( if (filter && (obj->flags & NOT_USER_GIVEN)) return filter->filter_object_fn(r, filter_situation, obj, pathname, filename, + filter->omits, filter->filter_data); /* * No filter is active or user gave object explicitly. In this case, -- cgit v0.10.2-6-g49f6 From 842b00516aebee06fc99c51a663b6587f642d36d Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:07 -0700 Subject: list-objects-filter-options: always supply *errbuf Making errbuf an optional argument complicates error reporting. Fix this by making all callers supply an errbuf, even if they may ignore it. This will be important in follow-up patches where the filter-spec parsing has more pitfalls and possible errors. Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 1cb20c6..7c3e397 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -30,11 +30,8 @@ static int gently_parse_list_objects_filter( const char *v0; if (filter_options->choice) { - if (errbuf) { - strbuf_addstr( - errbuf, - _("multiple filter-specs cannot be combined")); - } + strbuf_addstr( + errbuf, _("multiple filter-specs cannot be combined")); return 1; } @@ -52,11 +49,7 @@ static int gently_parse_list_objects_filter( } else if (skip_prefix(arg, "tree:", &v0)) { if (!git_parse_ulong(v0, &filter_options->tree_exclude_depth)) { - if (errbuf) { - strbuf_addstr( - errbuf, - _("expected 'tree:'")); - } + strbuf_addstr(errbuf, _("expected 'tree:'")); return 1; } filter_options->choice = LOFC_TREE_DEPTH; @@ -90,8 +83,7 @@ static int gently_parse_list_objects_filter( * add new filters */ - if (errbuf) - strbuf_addf(errbuf, _("invalid filter-spec '%s'"), arg); + strbuf_addf(errbuf, _("invalid filter-spec '%s'"), arg); memset(filter_options, 0, sizeof(*filter_options)); return 1; @@ -175,6 +167,8 @@ void partial_clone_register( void partial_clone_get_default_filter_spec( struct list_objects_filter_options *filter_options) { + struct strbuf errbuf = STRBUF_INIT; + /* * Parse default value, but silently ignore it if it is invalid. */ @@ -182,5 +176,6 @@ void partial_clone_get_default_filter_spec( return; gently_parse_list_objects_filter(filter_options, core_partial_clone_filter_default, - NULL); + &errbuf); + strbuf_release(&errbuf); } -- cgit v0.10.2-6-g49f6 From e987df5fe62b8b29be4cdcdeb3704681ada2b29e Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:08 -0700 Subject: list-objects-filter: implement composite filters Allow combining filters such that only objects accepted by all filters are shown. The motivation for this is to allow getting directory listings without also fetching blobs. This can be done by combining blob:none with tree:. There are massive repositories that have larger-than-expected trees - even if you include only a single commit. A combined filter supports any number of subfilters, and is written in the following form: combine:++ Certain non-alphanumeric characters in each filter must be URL-encoded. For now, combined filters must be specified in this form. In a subsequent commit, rev-list will support multiple --filter arguments which will have the same effect as specifying one filter argument starting with "combine:". The documentation will be updated in that commit, as the URL-encoding scheme is in general not meant to be used directly by the user, and it is better to describe the URL-encoding feature in terms of the repeated flag. Helped-by: Emily Shaffer Helped-by: Jeff Hostetler Helped-by: Johannes Schindelin Helped-by: Jonathan Tan Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 7c3e397..75d0236 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -6,6 +6,12 @@ #include "list-objects.h" #include "list-objects-filter.h" #include "list-objects-filter-options.h" +#include "url.h" + +static int parse_combine_filter( + struct list_objects_filter_options *filter_options, + const char *arg, + struct strbuf *errbuf); /* * Parse value of the argument to the "filter" keyword. @@ -35,8 +41,6 @@ static int gently_parse_list_objects_filter( return 1; } - filter_options->filter_spec = strdup(arg); - if (!strcmp(arg, "blob:none")) { filter_options->choice = LOFC_BLOB_NONE; return 0; @@ -77,6 +81,10 @@ static int gently_parse_list_objects_filter( _("sparse:path filters support has been dropped")); } return 1; + + } else if (skip_prefix(arg, "combine:", &v0)) { + return parse_combine_filter(filter_options, v0, errbuf); + } /* * Please update _git_fetch() in git-completion.bash when you @@ -89,10 +97,95 @@ static int gently_parse_list_objects_filter( return 1; } +static const char *RESERVED_NON_WS = "~`!@#$^&*()[]{}\\;'\",<>?"; + +static int has_reserved_character( + struct strbuf *sub_spec, struct strbuf *errbuf) +{ + const char *c = sub_spec->buf; + while (*c) { + if (*c <= ' ' || strchr(RESERVED_NON_WS, *c)) { + strbuf_addf( + errbuf, + _("must escape char in sub-filter-spec: '%c'"), + *c); + return 1; + } + c++; + } + + return 0; +} + +static int parse_combine_subfilter( + struct list_objects_filter_options *filter_options, + struct strbuf *subspec, + struct strbuf *errbuf) +{ + size_t new_index = filter_options->sub_nr++; + char *decoded; + int result; + + ALLOC_GROW(filter_options->sub, filter_options->sub_nr, + filter_options->sub_alloc); + memset(&filter_options->sub[new_index], 0, + sizeof(*filter_options->sub)); + + decoded = url_percent_decode(subspec->buf); + + result = has_reserved_character(subspec, errbuf) || + gently_parse_list_objects_filter( + &filter_options->sub[new_index], decoded, errbuf); + + free(decoded); + return result; +} + +static int parse_combine_filter( + struct list_objects_filter_options *filter_options, + const char *arg, + struct strbuf *errbuf) +{ + struct strbuf **subspecs = strbuf_split_str(arg, '+', 0); + size_t sub; + int result = 0; + + if (!subspecs[0]) { + strbuf_addstr(errbuf, _("expected something after combine:")); + result = 1; + goto cleanup; + } + + for (sub = 0; subspecs[sub] && !result; sub++) { + if (subspecs[sub + 1]) { + /* + * This is not the last subspec. Remove trailing "+" so + * we can parse it. + */ + size_t last = subspecs[sub]->len - 1; + assert(subspecs[sub]->buf[last] == '+'); + strbuf_remove(subspecs[sub], last, 1); + } + result = parse_combine_subfilter( + filter_options, subspecs[sub], errbuf); + } + + filter_options->choice = LOFC_COMBINE; + +cleanup: + strbuf_list_free(subspecs); + if (result) { + list_objects_filter_release(filter_options); + memset(filter_options, 0, sizeof(*filter_options)); + } + return result; +} + int parse_list_objects_filter(struct list_objects_filter_options *filter_options, const char *arg) { struct strbuf buf = STRBUF_INIT; + filter_options->filter_spec = strdup(arg); if (gently_parse_list_objects_filter(filter_options, arg, &buf)) die("%s", buf.buf); return 0; @@ -129,8 +222,15 @@ void expand_list_objects_filter_spec( void list_objects_filter_release( struct list_objects_filter_options *filter_options) { + size_t sub; + + if (!filter_options) + return; free(filter_options->filter_spec); free(filter_options->sparse_oid_value); + for (sub = 0; sub < filter_options->sub_nr; sub++) + list_objects_filter_release(&filter_options->sub[sub]); + free(filter_options->sub); memset(filter_options, 0, sizeof(*filter_options)); } @@ -174,6 +274,8 @@ void partial_clone_get_default_filter_spec( */ if (!core_partial_clone_filter_default) return; + + filter_options->filter_spec = strdup(core_partial_clone_filter_default); gently_parse_list_objects_filter(filter_options, core_partial_clone_filter_default, &errbuf); diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index c54f000..789faef 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -13,6 +13,7 @@ enum list_objects_filter_choice { LOFC_BLOB_LIMIT, LOFC_TREE_DEPTH, LOFC_SPARSE_OID, + LOFC_COMBINE, LOFC__COUNT /* must be last */ }; @@ -38,13 +39,23 @@ struct list_objects_filter_options { unsigned int no_filter : 1; /* - * Parsed values (fields) from within the filter-spec. These are - * choice-specific; not all values will be defined for any given - * choice. + * BEGIN choice-specific parsed values from within the filter-spec. Only + * some values will be defined for any given choice. */ + struct object_id *sparse_oid_value; unsigned long blob_limit_value; unsigned long tree_exclude_depth; + + /* LOFC_COMBINE values */ + + /* This array contains all the subfilters which this filter combines. */ + size_t sub_nr, sub_alloc; + struct list_objects_filter_options *sub; + + /* + * END choice-specific parsed values. + */ }; /* Normalized command line arguments */ diff --git a/list-objects-filter.c b/list-objects-filter.c index 3b4b676..d664264 100644 --- a/list-objects-filter.c +++ b/list-objects-filter.c @@ -26,6 +26,14 @@ */ #define FILTER_SHOWN_BUT_REVISIT (1<<21) +struct subfilter { + struct filter *filter; + struct oidset seen; + struct oidset omits; + struct object_id skip_tree; + unsigned is_skipping_tree : 1; +}; + struct filter { enum list_objects_filter_result (*filter_object_fn)( struct repository *r, @@ -36,6 +44,23 @@ struct filter { struct oidset *omits, void *filter_data); + /* + * Optional. If this function is supplied and the filter needs + * to collect omits, then this function is called once before + * free_fn is called. + * + * This is required because the following two conditions hold: + * + * a. A tree filter can add and remove objects as an object + * graph is traversed. + * b. A combine filter's omit set is the union of all its + * subfilters, which may include tree: filters. + * + * As such, the omits sets must be separate sets, and can only + * be unioned after the traversal is completed. + */ + void (*finalize_omits_fn)(struct oidset *omits, void *filter_data); + void (*free_fn)(void *filter_data); void *filter_data; @@ -471,6 +496,139 @@ static void filter_sparse_oid__init( filter->free_fn = filter_sparse_free; } +/* A filter which only shows objects shown by all sub-filters. */ +struct combine_filter_data { + struct subfilter *sub; + size_t nr; +}; + +static enum list_objects_filter_result process_subfilter( + struct repository *r, + enum list_objects_filter_situation filter_situation, + struct object *obj, + const char *pathname, + const char *filename, + struct subfilter *sub) +{ + enum list_objects_filter_result result; + + /* + * Check and update is_skipping_tree before oidset_contains so + * that is_skipping_tree gets unset even when the object is + * marked as seen. As of this writing, no filter uses + * LOFR_MARK_SEEN on trees that also uses LOFR_SKIP_TREE, so the + * ordering is only theoretically important. Be cautious if you + * change the order of the below checks and more filters have + * been added! + */ + if (sub->is_skipping_tree) { + if (filter_situation == LOFS_END_TREE && + oideq(&obj->oid, &sub->skip_tree)) + sub->is_skipping_tree = 0; + else + return LOFR_ZERO; + } + if (oidset_contains(&sub->seen, &obj->oid)) + return LOFR_ZERO; + + result = list_objects_filter__filter_object( + r, filter_situation, obj, pathname, filename, sub->filter); + + if (result & LOFR_MARK_SEEN) + oidset_insert(&sub->seen, &obj->oid); + + if (result & LOFR_SKIP_TREE) { + sub->is_skipping_tree = 1; + sub->skip_tree = obj->oid; + } + + return result; +} + +static enum list_objects_filter_result filter_combine( + struct repository *r, + enum list_objects_filter_situation filter_situation, + struct object *obj, + const char *pathname, + const char *filename, + struct oidset *omits, + void *filter_data) +{ + struct combine_filter_data *d = filter_data; + enum list_objects_filter_result combined_result = + LOFR_DO_SHOW | LOFR_MARK_SEEN | LOFR_SKIP_TREE; + size_t sub; + + for (sub = 0; sub < d->nr; sub++) { + enum list_objects_filter_result sub_result = process_subfilter( + r, filter_situation, obj, pathname, filename, + &d->sub[sub]); + if (!(sub_result & LOFR_DO_SHOW)) + combined_result &= ~LOFR_DO_SHOW; + if (!(sub_result & LOFR_MARK_SEEN)) + combined_result &= ~LOFR_MARK_SEEN; + if (!d->sub[sub].is_skipping_tree) + combined_result &= ~LOFR_SKIP_TREE; + } + + return combined_result; +} + +static void filter_combine__free(void *filter_data) +{ + struct combine_filter_data *d = filter_data; + size_t sub; + for (sub = 0; sub < d->nr; sub++) { + list_objects_filter__free(d->sub[sub].filter); + oidset_clear(&d->sub[sub].seen); + if (d->sub[sub].omits.set.size) + BUG("expected oidset to be cleared already"); + } + free(d->sub); +} + +static void add_all(struct oidset *dest, struct oidset *src) { + struct oidset_iter iter; + struct object_id *src_oid; + + oidset_iter_init(src, &iter); + while ((src_oid = oidset_iter_next(&iter)) != NULL) + oidset_insert(dest, src_oid); +} + +static void filter_combine__finalize_omits( + struct oidset *omits, + void *filter_data) +{ + struct combine_filter_data *d = filter_data; + size_t sub; + + for (sub = 0; sub < d->nr; sub++) { + add_all(omits, &d->sub[sub].omits); + oidset_clear(&d->sub[sub].omits); + } +} + +static void filter_combine__init( + struct list_objects_filter_options *filter_options, + struct filter* filter) +{ + struct combine_filter_data *d = xcalloc(1, sizeof(*d)); + size_t sub; + + d->nr = filter_options->sub_nr; + d->sub = xcalloc(d->nr, sizeof(*d->sub)); + for (sub = 0; sub < d->nr; sub++) + d->sub[sub].filter = list_objects_filter__init( + filter->omits ? &d->sub[sub].omits : NULL, + &filter_options->sub[sub]); + + filter->filter_data = d; + filter->filter_object_fn = filter_combine; + filter->free_fn = filter_combine__free; + filter->finalize_omits_fn = filter_combine__finalize_omits; +} + typedef void (*filter_init_fn)( struct list_objects_filter_options *filter_options, struct filter *filter); @@ -484,6 +642,7 @@ static filter_init_fn s_filters[] = { filter_blobs_limit__init, filter_trees_depth__init, filter_sparse_oid__init, + filter_combine__init, }; struct filter *list_objects_filter__init( @@ -536,6 +695,8 @@ void list_objects_filter__free(struct filter *filter) { if (!filter) return; + if (filter->finalize_omits_fn && filter->omits) + filter->finalize_omits_fn(filter->omits, filter->filter_data); filter->free_fn(filter->filter_data); free(filter); } diff --git a/list-objects-filter.h b/list-objects-filter.h index 6908954..cfd784e 100644 --- a/list-objects-filter.h +++ b/list-objects-filter.h @@ -62,7 +62,13 @@ enum list_objects_filter_situation { struct filter; -/* Constructor for the set of defined list-objects filters. */ +/* + * Constructor for the set of defined list-objects filters. + * The `omitted` set is optional. It is populated with objects that the + * filter excludes. This set should not be considered finalized until + * after list_objects_filter__free is called on the returned `struct + * filter *`. + */ struct filter *list_objects_filter__init( struct oidset *omitted, struct list_objects_filter_options *filter_options); @@ -80,7 +86,10 @@ enum list_objects_filter_result list_objects_filter__filter_object( const char *filename, struct filter *filter); -/* Destroys `filter`. Does nothing if `filter` is null. */ +/* + * Destroys `filter` and finalizes the `omitted` set, if present. Does + * nothing if `filter` is null. + */ void list_objects_filter__free(struct filter *filter); #endif /* LIST_OBJECTS_FILTER_H */ diff --git a/t/t6112-rev-list-filters-objects.sh b/t/t6112-rev-list-filters-objects.sh index acd7f5a..05d4f2e 100755 --- a/t/t6112-rev-list-filters-objects.sh +++ b/t/t6112-rev-list-filters-objects.sh @@ -278,7 +278,19 @@ test_expect_success 'verify skipping tree iteration when not collecting omits' ' test_line_count = 2 actual && # Make sure no other trees were considered besides the root. - ! grep "Skipping contents of tree [^.]" filter_trace + ! grep "Skipping contents of tree [^.]" filter_trace && + + # Try this again with "combine:". If both sub-filters are skipping + # trees, the composite filter should also skip trees. This is not + # important unless the user does combine:tree:X+tree:Y or another filter + # besides "tree:" is implemented in the future which can skip trees. + GIT_TRACE=1 git -C r3 rev-list \ + --objects --filter=combine:tree:1+tree:3 HEAD 2>filter_trace && + + # Only skip the dir1/ tree, which is shared between the two commits. + grep "Skipping contents of tree " filter_trace >actual && + test_write_lines "Skipping contents of tree dir1/..." >expected && + test_cmp expected actual ' # Test tree:# filters. @@ -330,6 +342,112 @@ test_expect_success 'verify tree:3 includes everything expected' ' test_line_count = 10 actual ' +test_expect_success 'combine:... for a simple combination' ' + git -C r3 rev-list --objects --filter=combine:tree:2+blob:none HEAD \ + >actual && + + expect_has HEAD "" && + expect_has HEAD~1 "" && + expect_has HEAD dir1 && + + # There are also 2 commit objects + test_line_count = 5 actual +' + +test_expect_success 'combine:... with URL encoding' ' + git -C r3 rev-list --objects \ + --filter=combine:tree%3a2+blob:%6Eon%65 HEAD >actual && + + expect_has HEAD "" && + expect_has HEAD~1 "" && + expect_has HEAD dir1 && + + # There are also 2 commit objects + test_line_count = 5 actual +' + +expect_invalid_filter_spec () { + spec="$1" && + err="$2" && + + test_must_fail git -C r3 rev-list --objects --filter="$spec" HEAD \ + >actual 2>actual_stderr && + test_must_be_empty actual && + test_i18ngrep "$err" actual_stderr +} + +test_expect_success 'combine:... while URL-encoding things that should not be' ' + expect_invalid_filter_spec combine%3Atree:2+blob:none \ + "invalid filter-spec" +' + +test_expect_success 'combine: with nothing after the :' ' + expect_invalid_filter_spec combine: "expected something after combine:" +' + +test_expect_success 'parse error in first sub-filter in combine:' ' + expect_invalid_filter_spec combine:tree:asdf+blob:none \ + "expected .tree:." +' + +test_expect_success 'combine:... with non-encoded reserved chars' ' + expect_invalid_filter_spec combine:tree:2+sparse:@xyz \ + "must escape char in sub-filter-spec: .@." && + expect_invalid_filter_spec combine:tree:2+sparse:\` \ + "must escape char in sub-filter-spec: .\`." && + expect_invalid_filter_spec combine:tree:2+sparse:~abc \ + "must escape char in sub-filter-spec: .\~." +' + +test_expect_success 'validate err msg for "combine:+"' ' + expect_invalid_filter_spec combine:tree:2+ "expected .tree:." +' + +test_expect_success 'combine:... with edge-case hex digits: Ff Aa 0 9' ' + git -C r3 rev-list --objects --filter="combine:tree:2+bl%6Fb:n%6fne" \ + HEAD >actual && + test_line_count = 5 actual && + git -C r3 rev-list --objects --filter="combine:tree%3A2+blob%3anone" \ + HEAD >actual && + test_line_count = 5 actual && + git -C r3 rev-list --objects --filter="combine:tree:%30" HEAD >actual && + test_line_count = 2 actual && + git -C r3 rev-list --objects --filter="combine:tree:%39+blob:none" \ + HEAD >actual && + test_line_count = 5 actual +' + +test_expect_success 'add a sparse pattern blob whose path has reserved chars' ' + cp r3/pattern r3/pattern1+renamed% && + git -C r3 add pattern1+renamed% && + git -C r3 commit -m "add sparse pattern file with reserved chars" +' + +test_expect_success 'combine:... with more than two sub-filters' ' + git -C r3 rev-list --objects \ + --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern \ + HEAD >actual && + + expect_has HEAD "" && + expect_has HEAD~1 "" && + expect_has HEAD~2 "" && + expect_has HEAD dir1 && + expect_has HEAD dir1/sparse1 && + expect_has HEAD dir1/sparse2 && + + # Should also have 3 commits + test_line_count = 9 actual && + + # Try again, this time making sure the last sub-filter is only + # URL-decoded once. + cp actual expect && + + git -C r3 rev-list --objects \ + --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern1%2brenamed%25 \ + HEAD >actual && + test_cmp expect actual +' + # Test provisional omit collection logic with a repo that has objects appearing # at multiple depths - first deeper than the filter's threshold, then shallow. @@ -373,6 +491,37 @@ test_expect_success 'verify skipping tree iteration when collecting omits' ' test_cmp expect actual ' +test_expect_success 'setup r5' ' + git init r5 && + mkdir -p r5/subdir && + + echo 1 >r5/short-root && + echo 12345 >r5/long-root && + echo a >r5/subdir/short-subdir && + echo abcde >r5/subdir/long-subdir && + + git -C r5 add short-root long-root subdir && + git -C r5 commit -m "commit msg" +' + +test_expect_success 'verify collecting omits in combined: filter' ' + # Note that this test guards against the naive implementation of simply + # giving both filters the same "omits" set and expecting it to + # automatically merge them. + git -C r5 rev-list --objects --quiet --filter-print-omitted \ + --filter=combine:tree:2+blob:limit=3 HEAD >actual && + + # Expect 0 trees/commits, 3 blobs omitted (all blobs except short-root) + omitted_1=$(echo 12345 | git hash-object --stdin) && + omitted_2=$(echo a | git hash-object --stdin) && + omitted_3=$(echo abcde | git hash-object --stdin) && + + grep ~$omitted_1 actual && + grep ~$omitted_2 actual && + grep ~$omitted_3 actual && + test_line_count = 3 actual +' + # Test tree: where a tree is iterated to twice - once where a subentry is # too deep to be included, and again where the blob inside it is shallow enough # to be included. This makes sure we don't use LOFR_MARK_SEEN incorrectly (we diff --git a/url.c b/url.c index 1b8ef78..e34e5e7 100644 --- a/url.c +++ b/url.c @@ -86,6 +86,12 @@ char *url_decode_mem(const char *url, int len) return url_decode_internal(&url, len, NULL, &out, 0); } +char *url_percent_decode(const char *encoded) +{ + struct strbuf out = STRBUF_INIT; + return url_decode_internal(&encoded, strlen(encoded), NULL, &out, 0); +} + char *url_decode_parameter_name(const char **query) { struct strbuf out = STRBUF_INIT; diff --git a/url.h b/url.h index 00b7d58..2a27c34 100644 --- a/url.h +++ b/url.h @@ -7,6 +7,14 @@ int is_url(const char *url); int is_urlschemechar(int first_flag, int ch); char *url_decode(const char *url); char *url_decode_mem(const char *url, int len); + +/* + * Similar to the url_decode_{,mem} methods above, but doesn't assume there + * is a scheme followed by a : at the start of the string. Instead, %-sequences + * before any : are also parsed. + */ +char *url_percent_decode(const char *encoded); + char *url_decode_parameter_name(const char **query); char *url_decode_parameter_value(const char **query); -- cgit v0.10.2-6-g49f6 From f56f764279be2433ecf3cb3a484210bcaffea70c Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:09 -0700 Subject: list-objects-filter-options: move error check up Move the check that filter_options->choice is set to higher in the call stack. This can only be set when the gentle parse function is called from one of the two call sites. This is important because in an upcoming patch this may or may not be an error, and whether it is an error is only known to the parse_list_objects_filter function. Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 75d0236..5fe2814 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -35,11 +35,8 @@ static int gently_parse_list_objects_filter( { const char *v0; - if (filter_options->choice) { - strbuf_addstr( - errbuf, _("multiple filter-specs cannot be combined")); - return 1; - } + if (filter_options->choice) + BUG("filter_options already populated"); if (!strcmp(arg, "blob:none")) { filter_options->choice = LOFC_BLOB_NONE; @@ -185,6 +182,8 @@ int parse_list_objects_filter(struct list_objects_filter_options *filter_options const char *arg) { struct strbuf buf = STRBUF_INIT; + if (filter_options->choice) + die(_("multiple filter-specs cannot be combined")); filter_options->filter_spec = strdup(arg); if (gently_parse_list_objects_filter(filter_options, arg, &buf)) die("%s", buf.buf); -- cgit v0.10.2-6-g49f6 From cf9ceb5a12cad9c9153d227a0f497d1b522ce085 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:10 -0700 Subject: list-objects-filter-options: make filter_spec a string_list Make the filter_spec string a string_list rather than a raw C string. The list of strings must be concatted together to make a complete filter_spec. A future patch will use this capability to build "combine:" filter specs gradually. A strbuf would seem to be a more natural choice for this object, but it unfortunately requires initialization besides just zero'ing out the memory. This results in all container structs, and all containers of those structs, etc., to also require initialization. Initializing them all would be more cumbersome that simply using a string_list, which behaves properly when its contents are zero'd. For the purposes of code simplification, change behavior in how filter specs are conveyed over the protocol: do not normalize the tree: filter specs since there should be no server in existence that supports tree:# but not tree:#k etc. Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/builtin/clone.c b/builtin/clone.c index 5b9ebe9..a693e6c 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1149,13 +1149,11 @@ int cmd_clone(int argc, const char **argv, const char *prefix) transport->server_options = &server_options; if (filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); + const char *spec = + expand_list_objects_filter_spec(&filter_options); transport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, - expanded_filter_spec.buf); + spec); transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - strbuf_release(&expanded_filter_spec); } if (transport->smart_options && !deepen && !filter_options.choice) diff --git a/builtin/fetch.c b/builtin/fetch.c index 4ba63d5..dee89e1 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1188,13 +1188,10 @@ static struct transport *prepare_transport(struct remote *remote, int deepen) if (update_shallow) set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes"); if (filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); - set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, - expanded_filter_spec.buf); + const char *spec = + expand_list_objects_filter_spec(&filter_options); + set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, spec); set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - strbuf_release(&expanded_filter_spec); } if (negotiation_tip.nr) { if (transport->smart_options) diff --git a/builtin/rev-list.c b/builtin/rev-list.c index 660172b..68acbe8 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -466,8 +466,10 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix) die(_("object filtering requires --objects")); if (filter_options.choice == LOFC_SPARSE_OID && !filter_options.sparse_oid_value) - die(_("invalid sparse value '%s'"), - filter_options.filter_spec); + die( + _("invalid sparse value '%s'"), + list_objects_filter_spec( + &filter_options)); continue; } if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) { diff --git a/fetch-pack.c b/fetch-pack.c index 1c10f54..72e13b0 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -339,12 +339,9 @@ static int find_common(struct fetch_negotiator *negotiator, } } if (server_supports_filtering && args->filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&args->filter_options, - &expanded_filter_spec); - packet_buf_write(&req_buf, "filter %s", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); + packet_buf_write(&req_buf, "filter %s", spec); } packet_buf_flush(&req_buf); state_len = req_buf.len; @@ -1099,7 +1096,7 @@ static int add_haves(struct fetch_negotiator *negotiator, } static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, - const struct fetch_pack_args *args, + struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain, int sideband_all) @@ -1140,13 +1137,10 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, /* Add filter */ if (server_supports_feature("fetch", "filter", 0) && args->filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); print_verbose(args, _("Server supports filter")); - expand_list_objects_filter_spec(&args->filter_options, - &expanded_filter_spec); - packet_buf_write(&req_buf, "filter %s", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + packet_buf_write(&req_buf, "filter %s", spec); } else if (args->filter_options.choice) { warning("filtering not recognized by server, ignoring"); } diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 5fe2814..01c0f13 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -184,7 +184,7 @@ int parse_list_objects_filter(struct list_objects_filter_options *filter_options struct strbuf buf = STRBUF_INIT; if (filter_options->choice) die(_("multiple filter-specs cannot be combined")); - filter_options->filter_spec = strdup(arg); + string_list_append(&filter_options->filter_spec, xstrdup(arg)); if (gently_parse_list_objects_filter(filter_options, arg, &buf)) die("%s", buf.buf); return 0; @@ -203,19 +203,36 @@ int opt_parse_list_objects_filter(const struct option *opt, return parse_list_objects_filter(filter_options, arg); } -void expand_list_objects_filter_spec( - const struct list_objects_filter_options *filter, - struct strbuf *expanded_spec) +const char *list_objects_filter_spec(struct list_objects_filter_options *filter) { - strbuf_init(expanded_spec, strlen(filter->filter_spec)); - if (filter->choice == LOFC_BLOB_LIMIT) - strbuf_addf(expanded_spec, "blob:limit=%lu", + if (!filter->filter_spec.nr) + BUG("no filter_spec available for this filter"); + if (filter->filter_spec.nr != 1) { + struct strbuf concatted = STRBUF_INIT; + strbuf_add_separated_string_list( + &concatted, "", &filter->filter_spec); + string_list_clear(&filter->filter_spec, /*free_util=*/0); + string_list_append( + &filter->filter_spec, strbuf_detach(&concatted, NULL)); + } + + return filter->filter_spec.items[0].string; +} + +const char *expand_list_objects_filter_spec( + struct list_objects_filter_options *filter) +{ + if (filter->choice == LOFC_BLOB_LIMIT) { + struct strbuf expanded_spec = STRBUF_INIT; + strbuf_addf(&expanded_spec, "blob:limit=%lu", filter->blob_limit_value); - else if (filter->choice == LOFC_TREE_DEPTH) - strbuf_addf(expanded_spec, "tree:%lu", - filter->tree_exclude_depth); - else - strbuf_addstr(expanded_spec, filter->filter_spec); + string_list_clear(&filter->filter_spec, /*free_util=*/0); + string_list_append( + &filter->filter_spec, + strbuf_detach(&expanded_spec, NULL)); + } + + return list_objects_filter_spec(filter); } void list_objects_filter_release( @@ -225,7 +242,7 @@ void list_objects_filter_release( if (!filter_options) return; - free(filter_options->filter_spec); + string_list_clear(&filter_options->filter_spec, /*free_util=*/0); free(filter_options->sparse_oid_value); for (sub = 0; sub < filter_options->sub_nr; sub++) list_objects_filter_release(&filter_options->sub[sub]); @@ -235,7 +252,7 @@ void list_objects_filter_release( void partial_clone_register( const char *remote, - const struct list_objects_filter_options *filter_options) + struct list_objects_filter_options *filter_options) { /* * Record the name of the partial clone remote in the @@ -258,7 +275,7 @@ void partial_clone_register( * the default for subsequent fetches from this remote. */ core_partial_clone_filter_default = - xstrdup(filter_options->filter_spec); + xstrdup(expand_list_objects_filter_spec(filter_options)); git_config_set("core.partialclonefilter", core_partial_clone_filter_default); } @@ -274,7 +291,8 @@ void partial_clone_get_default_filter_spec( if (!core_partial_clone_filter_default) return; - filter_options->filter_spec = strdup(core_partial_clone_filter_default); + string_list_append(&filter_options->filter_spec, + core_partial_clone_filter_default); gently_parse_list_objects_filter(filter_options, core_partial_clone_filter_default, &errbuf); diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index 789faef..bb33303 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -2,7 +2,7 @@ #define LIST_OBJECTS_FILTER_OPTIONS_H #include "parse-options.h" -#include "strbuf.h" +#include "string-list.h" /* * The list of defined filters for list-objects. @@ -24,8 +24,10 @@ struct list_objects_filter_options { * commands that launch filtering sub-processes, or for communication * over the network, don't use this value; use the result of * expand_list_objects_filter_spec() instead. + * To get the raw filter spec given by the user, use the result of + * list_objects_filter_spec(). */ - char *filter_spec; + struct string_list filter_spec; /* * 'choice' is determined by parsing the filter-spec. This indicates @@ -76,13 +78,22 @@ int opt_parse_list_objects_filter(const struct option *opt, /* * Translates abbreviated numbers in the filter's filter_spec into their * fully-expanded forms (e.g., "limit:blob=1k" becomes "limit:blob=1024"). + * Returns a string owned by the list_objects_filter_options object. * - * This form should be used instead of the raw filter_spec field when - * communicating with a remote process or subprocess. + * This form should be used instead of the raw list_objects_filter_spec() + * value when communicating with a remote process or subprocess. */ -void expand_list_objects_filter_spec( - const struct list_objects_filter_options *filter, - struct strbuf *expanded_spec); +const char *expand_list_objects_filter_spec( + struct list_objects_filter_options *filter); + +/* + * Returns the filter spec string more or less in the form as the user + * entered it. This form of the filter_spec can be used in user-facing + * messages. Returns a string owned by the list_objects_filter_options + * object. + */ +const char *list_objects_filter_spec( + struct list_objects_filter_options *filter); void list_objects_filter_release( struct list_objects_filter_options *filter_options); @@ -96,7 +107,7 @@ static inline void list_objects_filter_set_no_filter( void partial_clone_register( const char *remote, - const struct list_objects_filter_options *filter_options); + struct list_objects_filter_options *filter_options); void partial_clone_get_default_filter_spec( struct list_objects_filter_options *filter_options); diff --git a/t/t6112-rev-list-filters-objects.sh b/t/t6112-rev-list-filters-objects.sh index 05d4f2e..27ba157 100755 --- a/t/t6112-rev-list-filters-objects.sh +++ b/t/t6112-rev-list-filters-objects.sh @@ -590,11 +590,4 @@ test_expect_success 'expand blob limit in protocol' ' grep "blob:limit=1024" trace ' -test_expect_success 'expand tree depth limit in protocol' ' - GIT_TRACE_PACKET="$(pwd)/tree_trace" git -c protocol.version=2 clone \ - --filter=tree:0k "file://$(pwd)/r2" tree && - ! grep "tree:0k" tree_trace && - grep "tree:0" tree_trace -' - test_done diff --git a/transport-helper.c b/transport-helper.c index c7e17ec..0a34544 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -682,13 +682,9 @@ static int fetch(struct transport *transport, set_helper_option(transport, "update-shallow", "true"); if (data->transport_options.filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec( - &data->transport_options.filter_options, - &expanded_filter_spec); - set_helper_option(transport, "filter", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + const char *spec = expand_list_objects_filter_spec( + &data->transport_options.filter_options); + set_helper_option(transport, "filter", spec); } if (data->transport_options.negotiation_tips) diff --git a/upload-pack.c b/upload-pack.c index 4d2129e..d404d88 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -140,18 +140,17 @@ static void create_pack_file(const struct object_array *have_obj, argv_array_push(&pack_objects.args, "--delta-base-offset"); if (use_include_tag) argv_array_push(&pack_objects.args, "--include-tag"); - if (filter_options.filter_spec) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); + if (filter_options.choice) { + const char *spec = + expand_list_objects_filter_spec(&filter_options); if (pack_objects.use_shell) { struct strbuf buf = STRBUF_INIT; - sq_quote_buf(&buf, expanded_filter_spec.buf); + sq_quote_buf(&buf, spec); argv_array_pushf(&pack_objects.args, "--filter=%s", buf.buf); strbuf_release(&buf); } else { argv_array_pushf(&pack_objects.args, "--filter=%s", - expanded_filter_spec.buf); + spec); } } -- cgit v0.10.2-6-g49f6 From c2694952e33764818983fa247dcee72113c6ac6a Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:11 -0700 Subject: strbuf: give URL-encoding API a char predicate fn Allow callers to specify exactly what characters need to be URL-encoded and which do not. This new API will be taken advantage of in a patch later in this set. Helped-by: Jeff King Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/credential-store.c b/credential-store.c index ac29542..c010497 100644 --- a/credential-store.c +++ b/credential-store.c @@ -72,15 +72,16 @@ static void store_credential_file(const char *fn, struct credential *c) struct strbuf buf = STRBUF_INIT; strbuf_addf(&buf, "%s://", c->protocol); - strbuf_addstr_urlencode(&buf, c->username, 1); + strbuf_addstr_urlencode(&buf, c->username, is_rfc3986_unreserved); strbuf_addch(&buf, ':'); - strbuf_addstr_urlencode(&buf, c->password, 1); + strbuf_addstr_urlencode(&buf, c->password, is_rfc3986_unreserved); strbuf_addch(&buf, '@'); if (c->host) - strbuf_addstr_urlencode(&buf, c->host, 1); + strbuf_addstr_urlencode(&buf, c->host, is_rfc3986_unreserved); if (c->path) { strbuf_addch(&buf, '/'); - strbuf_addstr_urlencode(&buf, c->path, 0); + strbuf_addstr_urlencode(&buf, c->path, + is_rfc3986_reserved_or_unreserved); } rewrite_credential_file(fn, c, &buf); diff --git a/http.c b/http.c index 27aa0a3..938b9e5 100644 --- a/http.c +++ b/http.c @@ -513,9 +513,11 @@ static void set_proxyauth_name_password(CURL *result) #else struct strbuf s = STRBUF_INIT; - strbuf_addstr_urlencode(&s, proxy_auth.username, 1); + strbuf_addstr_urlencode(&s, proxy_auth.username, + is_rfc3986_unreserved); strbuf_addch(&s, ':'); - strbuf_addstr_urlencode(&s, proxy_auth.password, 1); + strbuf_addstr_urlencode(&s, proxy_auth.password, + is_rfc3986_unreserved); curl_proxyuserpwd = strbuf_detach(&s, NULL); curl_easy_setopt(result, CURLOPT_PROXYUSERPWD, curl_proxyuserpwd); #endif diff --git a/strbuf.c b/strbuf.c index 0e18b25..60ab514 100644 --- a/strbuf.c +++ b/strbuf.c @@ -774,8 +774,10 @@ void strbuf_addstr_xml_quoted(struct strbuf *buf, const char *s) } } -static int is_rfc3986_reserved(char ch) +int is_rfc3986_reserved_or_unreserved(char ch) { + if (is_rfc3986_unreserved(ch)) + return 1; switch (ch) { case '!': case '*': case '\'': case '(': case ')': case ';': case ':': case '@': case '&': case '=': case '+': case '$': @@ -785,20 +787,19 @@ static int is_rfc3986_reserved(char ch) return 0; } -static int is_rfc3986_unreserved(char ch) +int is_rfc3986_unreserved(char ch) { return isalnum(ch) || ch == '-' || ch == '_' || ch == '.' || ch == '~'; } static void strbuf_add_urlencode(struct strbuf *sb, const char *s, size_t len, - int reserved) + char_predicate allow_unencoded_fn) { strbuf_grow(sb, len); while (len--) { char ch = *s++; - if (is_rfc3986_unreserved(ch) || - (!reserved && is_rfc3986_reserved(ch))) + if (allow_unencoded_fn(ch)) strbuf_addch(sb, ch); else strbuf_addf(sb, "%%%02x", (unsigned char)ch); @@ -806,9 +807,9 @@ static void strbuf_add_urlencode(struct strbuf *sb, const char *s, size_t len, } void strbuf_addstr_urlencode(struct strbuf *sb, const char *s, - int reserved) + char_predicate allow_unencoded_fn) { - strbuf_add_urlencode(sb, s, strlen(s), reserved); + strbuf_add_urlencode(sb, s, strlen(s), allow_unencoded_fn); } void strbuf_humanise_bytes(struct strbuf *buf, off_t bytes) diff --git a/strbuf.h b/strbuf.h index c8d98df..346d722 100644 --- a/strbuf.h +++ b/strbuf.h @@ -666,8 +666,13 @@ void strbuf_branchname(struct strbuf *sb, const char *name, */ int strbuf_check_branch_ref(struct strbuf *sb, const char *name); +typedef int (*char_predicate)(char ch); + +int is_rfc3986_unreserved(char ch); +int is_rfc3986_reserved_or_unreserved(char ch); + void strbuf_addstr_urlencode(struct strbuf *sb, const char *name, - int reserved); + char_predicate allow_unencoded_fn); __attribute__((format (printf,1,2))) int printf_ln(const char *fmt, ...); -- cgit v0.10.2-6-g49f6 From 489fc9ee718b7c8594f17b55f090ac5292c655e1 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:12 -0700 Subject: list-objects-filter-options: allow mult. --filter Allow combining of multiple filters by simply repeating the --filter flag. Before this patch, the user had to combine them in a single flag somewhat awkwardly (e.g. --filter=combine:FOO+BAR), including URL-encoding the individual filters. To make this work, in the --filter flag parsing callback, rather than error out when we detect that the filter_options struct is already populated, we modify it in-place to contain the added sub-filter. The existing sub-filter becomes the lhs of the combined filter, and the next sub-filter becomes the rhs. We also have to URL-encode the LHS and RHS sub-filters. We can simplify the operation if the LHS is already a combine: filter. In that case, we just append the URL-encoded RHS sub-filter to the LHS spec to get the new spec. Helped-by: Emily Shaffer Helped-by: Jeff Hostetler Helped-by: Jeff King Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt index 71a1fcc..d1f080b 100644 --- a/Documentation/rev-list-options.txt +++ b/Documentation/rev-list-options.txt @@ -738,6 +738,22 @@ explicitly-given commit or tree. Note that the form '--filter=sparse:path=' that wants to read from an arbitrary path on the filesystem has been dropped for security reasons. ++ +Multiple '--filter=' flags can be specified to combine filters. Only +objects which are accepted by every filter are included. ++ +The form '--filter=combine:++...' can also be +used to combined several filters, but this is harder than just repeating +the '--filter' flag and is usually not necessary. Filters are joined by +'{plus}' and individual filters are %-encoded (i.e. URL-encoded). +Besides the '{plus}' and '%' characters, the following characters are +reserved and also must be encoded: `~!@#$^&*()[]{}\;",<>?`+'`+ +as well as all characters with ASCII code <= `0x20`, which includes +space and newline. ++ +Other arbitrary characters can also be encoded. For instance, +'combine:tree:3+blob:none' and 'combine:tree%3A3+blob%3Anone' are +equivalent. --no-filter:: Turn off any previous `--filter=` argument. diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 01c0f13..2506dc8 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -6,6 +6,7 @@ #include "list-objects.h" #include "list-objects-filter.h" #include "list-objects-filter-options.h" +#include "trace.h" #include "url.h" static int parse_combine_filter( @@ -178,15 +179,92 @@ cleanup: return result; } -int parse_list_objects_filter(struct list_objects_filter_options *filter_options, - const char *arg) +static int allow_unencoded(char ch) +{ + if (ch <= ' ' || ch == '%' || ch == '+') + return 0; + return !strchr(RESERVED_NON_WS, ch); +} + +static void filter_spec_append_urlencode( + struct list_objects_filter_options *filter, const char *raw) { struct strbuf buf = STRBUF_INIT; + strbuf_addstr_urlencode(&buf, raw, allow_unencoded); + trace_printf("Add to combine filter-spec: %s\n", buf.buf); + string_list_append(&filter->filter_spec, strbuf_detach(&buf, NULL)); +} + +/* + * Changes filter_options into an equivalent LOFC_COMBINE filter options + * instance. Does not do anything if filter_options is already LOFC_COMBINE. + */ +static void transform_to_combine_type( + struct list_objects_filter_options *filter_options) +{ + assert(filter_options->choice); + if (filter_options->choice == LOFC_COMBINE) + return; + { + const int initial_sub_alloc = 2; + struct list_objects_filter_options *sub_array = + xcalloc(initial_sub_alloc, sizeof(*sub_array)); + sub_array[0] = *filter_options; + memset(filter_options, 0, sizeof(*filter_options)); + filter_options->sub = sub_array; + filter_options->sub_alloc = initial_sub_alloc; + } + filter_options->sub_nr = 1; + filter_options->choice = LOFC_COMBINE; + string_list_append(&filter_options->filter_spec, xstrdup("combine:")); + filter_spec_append_urlencode( + filter_options, + list_objects_filter_spec(&filter_options->sub[0])); + /* + * We don't need the filter_spec strings for subfilter specs, only the + * top level. + */ + string_list_clear(&filter_options->sub[0].filter_spec, /*free_util=*/0); +} + +void list_objects_filter_die_if_populated( + struct list_objects_filter_options *filter_options) +{ if (filter_options->choice) die(_("multiple filter-specs cannot be combined")); - string_list_append(&filter_options->filter_spec, xstrdup(arg)); - if (gently_parse_list_objects_filter(filter_options, arg, &buf)) - die("%s", buf.buf); +} + +int parse_list_objects_filter( + struct list_objects_filter_options *filter_options, + const char *arg) +{ + struct strbuf errbuf = STRBUF_INIT; + int parse_error; + + if (!filter_options->choice) { + string_list_append(&filter_options->filter_spec, xstrdup(arg)); + + parse_error = gently_parse_list_objects_filter( + filter_options, arg, &errbuf); + } else { + /* + * Make filter_options an LOFC_COMBINE spec so we can trivially + * add subspecs to it. + */ + transform_to_combine_type(filter_options); + + string_list_append(&filter_options->filter_spec, xstrdup("+")); + filter_spec_append_urlencode(filter_options, arg); + ALLOC_GROW(filter_options->sub, filter_options->sub_nr + 1, + filter_options->sub_alloc); + filter_options = &filter_options->sub[filter_options->sub_nr++]; + memset(filter_options, 0, sizeof(*filter_options)); + + parse_error = gently_parse_list_objects_filter( + filter_options, arg, &errbuf); + } + if (parse_error) + die("%s", errbuf.buf); return 0; } diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index bb33303..d8bc7e9 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -63,6 +63,17 @@ struct list_objects_filter_options { /* Normalized command line arguments */ #define CL_ARG__FILTER "filter" +void list_objects_filter_die_if_populated( + struct list_objects_filter_options *filter_options); + +/* + * Parses the filter spec string given by arg and either (1) simply places the + * result in filter_options if it is not yet populated or (2) combines it with + * the filter already in filter_options if it is already populated. In the case + * of (2), the filter specs are combined as if specified with 'combine:'. + * + * Dies and prints a user-facing message if an error occurs. + */ int parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg); diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index b91ef54..32b7d72 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -208,6 +208,25 @@ test_expect_success 'use fsck before and after manually fetching a missing subtr test_cmp unique_types.expected unique_types.observed ' +test_expect_success 'implicitly construct combine: filter with repeated flags' ' + GIT_TRACE=$(pwd)/trace git clone --bare \ + --filter=blob:none --filter=tree:1 \ + "file://$(pwd)/srv.bare" pc2 && + grep "trace:.* git pack-objects .*--filter=combine:blob:none+tree:1" \ + trace && + git -C pc2 rev-list --objects --missing=allow-any HEAD >objects && + + # We should have gotten some root trees. + grep " $" objects && + # Should not have gotten any non-root trees or blobs. + ! grep " ." objects && + + xargs -n 1 git -C pc2 cat-file -t types && + sort -u types >unique_types.actual && + test_write_lines commit tree >unique_types.expected && + test_cmp unique_types.expected unique_types.actual +' + test_expect_success 'partial clone fetches blobs pointed to by refs even if normally filtered out' ' rm -rf src dst && git init src && diff --git a/t/t6112-rev-list-filters-objects.sh b/t/t6112-rev-list-filters-objects.sh index 27ba157..de0e5a5 100755 --- a/t/t6112-rev-list-filters-objects.sh +++ b/t/t6112-rev-list-filters-objects.sh @@ -351,7 +351,16 @@ test_expect_success 'combine:... for a simple combination' ' expect_has HEAD dir1 && # There are also 2 commit objects - test_line_count = 5 actual + test_line_count = 5 actual && + + cp actual expected && + + # Try again using repeated --filter - this is equivalent to a manual + # combine with "combine:...+..." + git -C r3 rev-list --objects --filter=combine:tree:2 \ + --filter=blob:none HEAD >actual && + + test_cmp expected actual ' test_expect_success 'combine:... with URL encoding' ' @@ -417,10 +426,12 @@ test_expect_success 'combine:... with edge-case hex digits: Ff Aa 0 9' ' test_line_count = 5 actual ' -test_expect_success 'add a sparse pattern blob whose path has reserved chars' ' +test_expect_success 'add sparse pattern blobs whose paths have reserved chars' ' cp r3/pattern r3/pattern1+renamed% && - git -C r3 add pattern1+renamed% && - git -C r3 commit -m "add sparse pattern file with reserved chars" + cp r3/pattern "r3/p;at%ter+n" && + cp r3/pattern r3/^~pattern && + git -C r3 add pattern1+renamed% "p;at%ter+n" ^~pattern && + git -C r3 commit -m "add sparse pattern files with reserved chars" ' test_expect_success 'combine:... with more than two sub-filters' ' @@ -445,7 +456,32 @@ test_expect_success 'combine:... with more than two sub-filters' ' git -C r3 rev-list --objects \ --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern1%2brenamed%25 \ HEAD >actual && - test_cmp expect actual + test_cmp expect actual && + + # Use the same composite filter again, but with a pattern file name that + # requires encoding multiple characters, and use implicit filter + # combining. + test_when_finished "rm -f trace1" && + GIT_TRACE=$(pwd)/trace1 git -C r3 rev-list --objects \ + --filter=tree:3 --filter=blob:limit=40 \ + --filter=sparse:oid="master:p;at%ter+n" \ + HEAD >actual && + + test_cmp expect actual && + grep "Add to combine filter-spec: sparse:oid=master:p%3bat%25ter%2bn" \ + trace1 && + + # Repeat the above test, but this time, the characters to encode are in + # the LHS of the combined filter. + test_when_finished "rm -f trace2" && + GIT_TRACE=$(pwd)/trace2 git -C r3 rev-list --objects \ + --filter=sparse:oid=master:^~pattern \ + --filter=tree:3 --filter=blob:limit=40 \ + HEAD >actual && + + test_cmp expect actual && + grep "Add to combine filter-spec: sparse:oid=master:%5e%7epattern" \ + trace2 ' # Test provisional omit collection logic with a repo that has objects appearing diff --git a/transport.c b/transport.c index f1fcd2c..ee7dd1c 100644 --- a/transport.c +++ b/transport.c @@ -224,6 +224,7 @@ static int set_git_option(struct git_transport_options *opts, opts->no_dependents = !!value; return 0; } else if (!strcmp(name, TRANS_OPT_LIST_OBJECTS_FILTER)) { + list_objects_filter_die_if_populated(&opts->filter_options); parse_list_objects_filter(&opts->filter_options, value); return 0; } diff --git a/upload-pack.c b/upload-pack.c index d404d88..f8a76eb 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -883,6 +883,7 @@ static void receive_needs(struct packet_reader *reader, struct object_array *wan if (skip_prefix(reader->line, "filter ", &arg)) { if (!filter_capability_requested) die("git upload-pack: filtering capability not negotiated"); + list_objects_filter_die_if_populated(&filter_options); parse_list_objects_filter(&filter_options, arg); continue; } @@ -1304,6 +1305,7 @@ static void process_args(struct packet_reader *request, } if (allow_filter && skip_prefix(arg, "filter ", &p)) { + list_objects_filter_die_if_populated(&filter_options); parse_list_objects_filter(&filter_options, p); continue; } -- cgit v0.10.2-6-g49f6 From 5a133e8a7f7c28c57f7a0a85e57692b8b781d896 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:13 -0700 Subject: list-objects-filter-options: clean up use of ALLOC_GROW Introduce a new macro ALLOC_GROW_BY which automatically zeros the added array elements and takes care of updating the nr value. Use the macro in code introduced earlier in this patchset. Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/cache.h b/cache.h index bf20337..cf5d70c 100644 --- a/cache.h +++ b/cache.h @@ -660,6 +660,9 @@ int daemonize(void); * at least 'nr' entries; the number of entries currently allocated * is 'alloc', using the standard growing factor alloc_nr() macro. * + * Consider using ALLOC_GROW_BY instead of ALLOC_GROW as it has some + * added niceties. + * * DO NOT USE any expression with side-effect for 'x', 'nr', or 'alloc'. */ #define ALLOC_GROW(x, nr, alloc) \ @@ -673,6 +676,25 @@ int daemonize(void); } \ } while (0) +/* + * Similar to ALLOC_GROW but handles updating of the nr value and + * zeroing the bytes of the newly-grown array elements. + * + * DO NOT USE any expression with side-effect for any of the + * arguments. + */ +#define ALLOC_GROW_BY(x, nr, increase, alloc) \ + do { \ + if (increase) { \ + size_t new_nr = nr + (increase); \ + if (new_nr < nr) \ + BUG("negative growth in ALLOC_GROW_BY"); \ + ALLOC_GROW(x, new_nr, alloc); \ + memset((x) + nr, 0, sizeof(*(x)) * (increase)); \ + nr = new_nr; \ + } \ + } while (0) + /* Initialize and use the cache information */ struct lock_file; void preload_index(struct index_state *index, diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 2506dc8..44bc115 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -120,14 +120,12 @@ static int parse_combine_subfilter( struct strbuf *subspec, struct strbuf *errbuf) { - size_t new_index = filter_options->sub_nr++; + size_t new_index = filter_options->sub_nr; char *decoded; int result; - ALLOC_GROW(filter_options->sub, filter_options->sub_nr, - filter_options->sub_alloc); - memset(&filter_options->sub[new_index], 0, - sizeof(*filter_options->sub)); + ALLOC_GROW_BY(filter_options->sub, filter_options->sub_nr, 1, + filter_options->sub_alloc); decoded = url_percent_decode(subspec->buf); @@ -255,13 +253,12 @@ int parse_list_objects_filter( string_list_append(&filter_options->filter_spec, xstrdup("+")); filter_spec_append_urlencode(filter_options, arg); - ALLOC_GROW(filter_options->sub, filter_options->sub_nr + 1, - filter_options->sub_alloc); - filter_options = &filter_options->sub[filter_options->sub_nr++]; - memset(filter_options, 0, sizeof(*filter_options)); + ALLOC_GROW_BY(filter_options->sub, filter_options->sub_nr, 1, + filter_options->sub_alloc); parse_error = gently_parse_list_objects_filter( - filter_options, arg, &errbuf); + &filter_options->sub[filter_options->sub_nr - 1], arg, + &errbuf); } if (parse_error) die("%s", errbuf.buf); -- cgit v0.10.2-6-g49f6 From 90d21f9ebf6906f0ebb4fb1b20ec9536072e2916 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:14 -0700 Subject: list-objects-filter-options: make parser void This function always returns 0, so make it return void instead. Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 44bc115..ba1425c 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -232,7 +232,7 @@ void list_objects_filter_die_if_populated( die(_("multiple filter-specs cannot be combined")); } -int parse_list_objects_filter( +void parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg) { @@ -262,7 +262,6 @@ int parse_list_objects_filter( } if (parse_error) die("%s", errbuf.buf); - return 0; } int opt_parse_list_objects_filter(const struct option *opt, @@ -270,12 +269,11 @@ int opt_parse_list_objects_filter(const struct option *opt, { struct list_objects_filter_options *filter_options = opt->value; - if (unset || !arg) { + if (unset || !arg) list_objects_filter_set_no_filter(filter_options); - return 0; - } - - return parse_list_objects_filter(filter_options, arg); + else + parse_list_objects_filter(filter_options, arg); + return 0; } const char *list_objects_filter_spec(struct list_objects_filter_options *filter) diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index d8bc7e9..db37dfb 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -74,7 +74,7 @@ void list_objects_filter_die_if_populated( * * Dies and prints a user-facing message if an error occurs. */ -int parse_list_objects_filter( +void parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg); -- cgit v0.10.2-6-g49f6