diff options
44 files changed, 3318 insertions, 368 deletions
@@ -140,6 +140,7 @@ /git-rm /git-send-email /git-send-pack +/git-serve /git-sh-i18n /git-sh-i18n--envsubst /git-sh-setup diff --git a/Documentation/Makefile b/Documentation/Makefile index 6232143..fa9e5c0 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -78,6 +78,7 @@ TECH_DOCS += technical/pack-heuristics TECH_DOCS += technical/pack-protocol TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common +TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow diff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt index 4b8c93e..9d1459a 100644 --- a/Documentation/gitremote-helpers.txt +++ b/Documentation/gitremote-helpers.txt @@ -102,6 +102,14 @@ Capabilities for Pushing + Supported commands: 'connect'. +'stateless-connect':: + Experimental; for internal use only. + Can attempt to connect to a remote server for communication + using git's wire-protocol version 2. See the documentation + for the stateless-connect command for more information. ++ +Supported commands: 'stateless-connect'. + 'push':: Can discover remote refs and push local commits and the history leading up to them to new or existing remote refs. @@ -136,6 +144,14 @@ Capabilities for Fetching + Supported commands: 'connect'. +'stateless-connect':: + Experimental; for internal use only. + Can attempt to connect to a remote server for communication + using git's wire-protocol version 2. See the documentation + for the stateless-connect command for more information. ++ +Supported commands: 'stateless-connect'. + 'fetch':: Can discover remote refs and transfer objects reachable from them to the local object store. @@ -375,6 +391,22 @@ Supported if the helper has the "export" capability. + Supported if the helper has the "connect" capability. +'stateless-connect' <service>:: + Experimental; for internal use only. + Connects to the given remote service for communication using + git's wire-protocol version 2. Valid replies to this command + are empty line (connection established), 'fallback' (no smart + transport support, fall back to dumb transports) and just + exiting with error message printed (can't connect, don't bother + trying to fall back). After line feed terminating the positive + (empty) response, the output of the service starts. Messages + (both request and response) must consist of zero or more + PKT-LINEs, terminating in a flush packet. The client must not + expect the server to store any state in between request-response + pairs. After the connection ends, the remote helper exits. ++ +Supported if the helper has the "stateless-connect" capability. + If a fatal error occurs, the program writes the error message to stderr and exits. The caller should expect that a suitable error message has been printed if the child closes the connection without diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt new file mode 100644 index 0000000..136179d --- /dev/null +++ b/Documentation/technical/protocol-v2.txt @@ -0,0 +1,395 @@ + Git Wire Protocol, Version 2 +============================== + +This document presents a specification for a version 2 of Git's wire +protocol. Protocol v2 will improve upon v1 in the following ways: + + * Instead of multiple service names, multiple commands will be + supported by a single service + * Easily extendable as capabilities are moved into their own section + of the protocol, no longer being hidden behind a NUL byte and + limited by the size of a pkt-line + * Separate out other information hidden behind NUL bytes (e.g. agent + string as a capability and symrefs can be requested using 'ls-refs') + * Reference advertisement will be omitted unless explicitly requested + * ls-refs command to explicitly request some refs + * Designed with http and stateless-rpc in mind. With clear flush + semantics the http remote helper can simply act as a proxy + +In protocol v2 communication is command oriented. When first contacting a +server a list of capabilities will advertised. Some of these capabilities +will be commands which a client can request be executed. Once a command +has completed, a client can reuse the connection and request that other +commands be executed. + + Packet-Line Framing +--------------------- + +All communication is done using packet-line framing, just as in v1. See +`Documentation/technical/pack-protocol.txt` and +`Documentation/technical/protocol-common.txt` for more information. + +In protocol v2 these special packets will have the following semantics: + + * '0000' Flush Packet (flush-pkt) - indicates the end of a message + * '0001' Delimiter Packet (delim-pkt) - separates sections of a message + + Initial Client Request +------------------------ + +In general a client can request to speak protocol v2 by sending +`version=2` through the respective side-channel for the transport being +used which inevitably sets `GIT_PROTOCOL`. More information can be +found in `pack-protocol.txt` and `http-protocol.txt`. In all cases the +response from the server is the capability advertisement. + + Git Transport +~~~~~~~~~~~~~~~ + +When using the git:// transport, you can request to use protocol v2 by +sending "version=2" as an extra parameter: + + 003egit-upload-pack /project.git\0host=myserver.com\0\0version=2\0 + + SSH and File Transport +~~~~~~~~~~~~~~~~~~~~~~~~ + +When using either the ssh:// or file:// transport, the GIT_PROTOCOL +environment variable must be set explicitly to include "version=2". + + HTTP Transport +~~~~~~~~~~~~~~~~ + +When using the http:// or https:// transport a client makes a "smart" +info/refs request as described in `http-protocol.txt` and requests that +v2 be used by supplying "version=2" in the `Git-Protocol` header. + + C: Git-Protocol: version=2 + C: + C: GET $GIT_URL/info/refs?service=git-upload-pack HTTP/1.0 + +A v2 server would reply: + + S: 200 OK + S: <Some headers> + S: ... + S: + S: 000eversion 2\n + S: <capability-advertisement> + +Subsequent requests are then made directly to the service +`$GIT_URL/git-upload-pack`. (This works the same for git-receive-pack). + + Capability Advertisement +-------------------------- + +A server which decides to communicate (based on a request from a client) +using protocol version 2, notifies the client by sending a version string +in its initial response followed by an advertisement of its capabilities. +Each capability is a key with an optional value. Clients must ignore all +unknown keys. Semantics of unknown values are left to the definition of +each key. Some capabilities will describe commands which can be requested +to be executed by the client. + + capability-advertisement = protocol-version + capability-list + flush-pkt + + protocol-version = PKT-LINE("version 2" LF) + capability-list = *capability + capability = PKT-LINE(key[=value] LF) + + key = 1*(ALPHA | DIGIT | "-_") + value = 1*(ALPHA | DIGIT | " -_.,?\/{}[]()<>!@#$%^&*+=:;") + + Command Request +----------------- + +After receiving the capability advertisement, a client can then issue a +request to select the command it wants with any particular capabilities +or arguments. There is then an optional section where the client can +provide any command specific parameters or queries. Only a single +command can be requested at a time. + + request = empty-request | command-request + empty-request = flush-pkt + command-request = command + capability-list + [command-args] + flush-pkt + command = PKT-LINE("command=" key LF) + command-args = delim-pkt + *command-specific-arg + + command-specific-args are packet line framed arguments defined by + each individual command. + +The server will then check to ensure that the client's request is +comprised of a valid command as well as valid capabilities which were +advertised. If the request is valid the server will then execute the +command. A server MUST wait till it has received the client's entire +request before issuing a response. The format of the response is +determined by the command being executed, but in all cases a flush-pkt +indicates the end of the response. + +When a command has finished, and the client has received the entire +response from the server, a client can either request that another +command be executed or can terminate the connection. A client may +optionally send an empty request consisting of just a flush-pkt to +indicate that no more requests will be made. + + Capabilities +-------------- + +There are two different types of capabilities: normal capabilities, +which can be used to to convey information or alter the behavior of a +request, and commands, which are the core actions that a client wants to +perform (fetch, push, etc). + +Protocol version 2 is stateless by default. This means that all commands +must only last a single round and be stateless from the perspective of the +server side, unless the client has requested a capability indicating that +state should be maintained by the server. Clients MUST NOT require state +management on the server side in order to function correctly. This +permits simple round-robin load-balancing on the server side, without +needing to worry about state management. + + agent +~~~~~~~ + +The server can advertise the `agent` capability with a value `X` (in the +form `agent=X`) to notify the client that the server is running version +`X`. The client may optionally send its own agent string by including +the `agent` capability with a value `Y` (in the form `agent=Y`) in its +request to the server (but it MUST NOT do so if the server did not +advertise the agent capability). The `X` and `Y` strings may contain any +printable ASCII characters except space (i.e., the byte range 32 < x < +127), and are typically of the form "package/version" (e.g., +"git/1.8.3.1"). The agent strings are purely informative for statistics +and debugging purposes, and MUST NOT be used to programmatically assume +the presence or absence of particular features. + + ls-refs +~~~~~~~~~ + +`ls-refs` is the command used to request a reference advertisement in v2. +Unlike the current reference advertisement, ls-refs takes in arguments +which can be used to limit the refs sent from the server. + +Additional features not supported in the base command will be advertised +as the value of the command in the capability advertisement in the form +of a space separated list of features: "<command>=<feature 1> <feature 2>" + +ls-refs takes in the following arguments: + + symrefs + In addition to the object pointed by it, show the underlying ref + pointed by it when showing a symbolic ref. + peel + Show peeled tags. + ref-prefix <prefix> + When specified, only references having a prefix matching one of + the provided prefixes are displayed. + +The output of ls-refs is as follows: + + output = *ref + flush-pkt + ref = PKT-LINE(obj-id SP refname *(SP ref-attribute) LF) + ref-attribute = (symref | peeled) + symref = "symref-target:" symref-target + peeled = "peeled:" obj-id + + fetch +~~~~~~~ + +`fetch` is the command used to fetch a packfile in v2. It can be looked +at as a modified version of the v1 fetch where the ref-advertisement is +stripped out (since the `ls-refs` command fills that role) and the +message format is tweaked to eliminate redundancies and permit easy +addition of future extensions. + +Additional features not supported in the base command will be advertised +as the value of the command in the capability advertisement in the form +of a space separated list of features: "<command>=<feature 1> <feature 2>" + +A `fetch` request can take the following arguments: + + want <oid> + Indicates to the server an object which the client wants to + retrieve. Wants can be anything and are not limited to + advertised objects. + + have <oid> + Indicates to the server an object which the client has locally. + This allows the server to make a packfile which only contains + the objects that the client needs. Multiple 'have' lines can be + supplied. + + done + Indicates to the server that negotiation should terminate (or + not even begin if performing a clone) and that the server should + use the information supplied in the request to construct the + packfile. + + thin-pack + Request that a thin pack be sent, which is a pack with deltas + which reference base objects not contained within the pack (but + are known to exist at the receiving end). This can reduce the + network traffic significantly, but it requires the receiving end + to know how to "thicken" these packs by adding the missing bases + to the pack. + + no-progress + Request that progress information that would normally be sent on + side-band channel 2, during the packfile transfer, should not be + sent. However, the side-band channel 3 is still used for error + responses. + + include-tag + Request that annotated tags should be sent if the objects they + point to are being sent. + + ofs-delta + Indicate that the client understands PACKv2 with delta referring + to its base by position in pack rather than by an oid. That is, + they can read OBJ_OFS_DELTA (ake type 6) in a packfile. + +If the 'shallow' feature is advertised the following arguments can be +included in the clients request as well as the potential addition of the +'shallow-info' section in the server's response as explained below. + + shallow <oid> + A client must notify the server of all commits for which it only + has shallow copies (meaning that it doesn't have the parents of + a commit) by supplying a 'shallow <oid>' line for each such + object so that the server is aware of the limitations of the + client's history. This is so that the server is aware that the + client may not have all objects reachable from such commits. + + deepen <depth> + Requests that the fetch/clone should be shallow having a commit + depth of <depth> relative to the remote side. + + deepen-relative + Requests that the semantics of the "deepen" command be changed + to indicate that the depth requested is relative to the client's + current shallow boundary, instead of relative to the requested + commits. + + deepen-since <timestamp> + Requests that the shallow clone/fetch should be cut at a + specific time, instead of depth. Internally it's equivalent to + doing "git rev-list --max-age=<timestamp>". Cannot be used with + "deepen". + + deepen-not <rev> + Requests that the shallow clone/fetch should be cut at a + specific revision specified by '<rev>', instead of a depth. + Internally it's equivalent of doing "git rev-list --not <rev>". + Cannot be used with "deepen", but can be used with + "deepen-since". + +The response of `fetch` is broken into a number of sections separated by +delimiter packets (0001), with each section beginning with its section +header. + + output = *section + section = (acknowledgments | shallow-info | packfile) + (flush-pkt | delim-pkt) + + acknowledgments = PKT-LINE("acknowledgments" LF) + (nak | *ack) + (ready) + ready = PKT-LINE("ready" LF) + nak = PKT-LINE("NAK" LF) + ack = PKT-LINE("ACK" SP obj-id LF) + + shallow-info = PKT-LINE("shallow-info" LF) + *PKT-LINE((shallow | unshallow) LF) + shallow = "shallow" SP obj-id + unshallow = "unshallow" SP obj-id + + packfile = PKT-LINE("packfile" LF) + *PKT-LINE(%x01-03 *%x00-ff) + + acknowledgments section + * If the client determines that it is finished with negotiations + by sending a "done" line, the acknowledgments sections MUST be + omitted from the server's response. + + * Always begins with the section header "acknowledgments" + + * The server will respond with "NAK" if none of the object ids sent + as have lines were common. + + * The server will respond with "ACK obj-id" for all of the + object ids sent as have lines which are common. + + * A response cannot have both "ACK" lines as well as a "NAK" + line. + + * The server will respond with a "ready" line indicating that + the server has found an acceptable common base and is ready to + make and send a packfile (which will be found in the packfile + section of the same response) + + * If the server has found a suitable cut point and has decided + to send a "ready" line, then the server can decide to (as an + optimization) omit any "ACK" lines it would have sent during + its response. This is because the server will have already + determined the objects it plans to send to the client and no + further negotiation is needed. + + shallow-info section + * If the client has requested a shallow fetch/clone, a shallow + client requests a fetch or the server is shallow then the + server's response may include a shallow-info section. The + shallow-info section will be included if (due to one of the + above conditions) the server needs to inform the client of any + shallow boundaries or adjustments to the clients already + existing shallow boundaries. + + * Always begins with the section header "shallow-info" + + * If a positive depth is requested, the server will compute the + set of commits which are no deeper than the desired depth. + + * The server sends a "shallow obj-id" line for each commit whose + parents will not be sent in the following packfile. + + * The server sends an "unshallow obj-id" line for each commit + which the client has indicated is shallow, but is no longer + shallow as a result of the fetch (due to its parents being + sent in the following packfile). + + * The server MUST NOT send any "unshallow" lines for anything + which the client has not indicated was shallow as a part of + its request. + + * This section is only included if a packfile section is also + included in the response. + + packfile section + * This section is only included if the client has sent 'want' + lines in its request and either requested that no more + negotiation be done by sending 'done' or if the server has + decided it has found a sufficient cut point to produce a + packfile. + + * Always begins with the section header "packfile" + + * The transmission of the packfile begins immediately after the + section header + + * The data transfer of the packfile is always multiplexed, using + the same semantics of the 'side-band-64k' capability from + protocol version 1. This means that each packet, during the + packfile data stream, is made up of a leading 4-byte pkt-line + length (typical of the pkt-line format), followed by a 1-byte + stream code, followed by the actual data. + + The stream code can be one of: + 1 - pack data + 2 - progress messages + 3 - fatal error message just before stream aborts @@ -652,7 +652,6 @@ PROGRAM_OBJS += imap-send.o PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o PROGRAM_OBJS += show-index.o -PROGRAM_OBJS += upload-pack.o PROGRAM_OBJS += remote-testsvn.o # Binary suffix, set to .exe for Windows builds @@ -701,6 +700,7 @@ TEST_PROGRAMS_NEED_X += test-dump-untracked-cache TEST_PROGRAMS_NEED_X += test-fake-ssh TEST_PROGRAMS_NEED_X += test-line-buffer TEST_PROGRAMS_NEED_X += test-parse-options +TEST_PROGRAMS_NEED_X += test-pkt-line TEST_PROGRAMS_NEED_X += test-svn-fe TEST_PROGRAMS_NEED_X += test-tool @@ -842,6 +842,7 @@ LIB_OBJS += list-objects-filter-options.o LIB_OBJS += ll-merge.o LIB_OBJS += lockfile.o LIB_OBJS += log-tree.o +LIB_OBJS += ls-refs.o LIB_OBJS += mailinfo.o LIB_OBJS += mailmap.o LIB_OBJS += match-trees.o @@ -898,6 +899,7 @@ LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o LIB_OBJS += sequencer.o +LIB_OBJS += serve.o LIB_OBJS += server-info.o LIB_OBJS += setup.o LIB_OBJS += sha1-array.o @@ -926,6 +928,7 @@ LIB_OBJS += tree-diff.o LIB_OBJS += tree.o LIB_OBJS += tree-walk.o LIB_OBJS += unpack-trees.o +LIB_OBJS += upload-pack.o LIB_OBJS += url.o LIB_OBJS += urlmatch.o LIB_OBJS += usage.o @@ -1030,6 +1033,7 @@ BUILTIN_OBJS += builtin/rev-parse.o BUILTIN_OBJS += builtin/revert.o BUILTIN_OBJS += builtin/rm.o BUILTIN_OBJS += builtin/send-pack.o +BUILTIN_OBJS += builtin/serve.o BUILTIN_OBJS += builtin/shortlog.o BUILTIN_OBJS += builtin/show-branch.o BUILTIN_OBJS += builtin/show-ref.o @@ -1043,6 +1047,7 @@ BUILTIN_OBJS += builtin/update-index.o BUILTIN_OBJS += builtin/update-ref.o BUILTIN_OBJS += builtin/update-server-info.o BUILTIN_OBJS += builtin/upload-archive.o +BUILTIN_OBJS += builtin/upload-pack.o BUILTIN_OBJS += builtin/var.o BUILTIN_OBJS += builtin/verify-commit.o BUILTIN_OBJS += builtin/verify-pack.o @@ -215,6 +215,7 @@ extern int cmd_rev_parse(int argc, const char **argv, const char *prefix); extern int cmd_revert(int argc, const char **argv, const char *prefix); extern int cmd_rm(int argc, const char **argv, const char *prefix); extern int cmd_send_pack(int argc, const char **argv, const char *prefix); +extern int cmd_serve(int argc, const char **argv, const char *prefix); extern int cmd_shortlog(int argc, const char **argv, const char *prefix); extern int cmd_show(int argc, const char **argv, const char *prefix); extern int cmd_show_branch(int argc, const char **argv, const char *prefix); @@ -231,6 +232,7 @@ extern int cmd_update_ref(int argc, const char **argv, const char *prefix); extern int cmd_update_server_info(int argc, const char **argv, const char *prefix); extern int cmd_upload_archive(int argc, const char **argv, const char *prefix); extern int cmd_upload_archive_writer(int argc, const char **argv, const char *prefix); +extern int cmd_upload_pack(int argc, const char **argv, const char *prefix); extern int cmd_var(int argc, const char **argv, const char *prefix); extern int cmd_verify_commit(int argc, const char **argv, const char *prefix); extern int cmd_verify_tag(int argc, const char **argv, const char *prefix); diff --git a/builtin/clone.c b/builtin/clone.c index 7df5932..84f1473 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1135,7 +1135,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (transport->smart_options && !deepen && !filter_options.choice) transport->smart_options->check_self_contained_and_connected = 1; - refs = transport_get_remote_refs(transport); + refs = transport_get_remote_refs(transport, NULL); if (refs) { mapped_refs = wanted_peer_refs(refs, refspec); diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index a7bc136..1a1bc63 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -4,6 +4,7 @@ #include "remote.h" #include "connect.h" #include "sha1-array.h" +#include "protocol.h" static const char fetch_pack_usage[] = "git fetch-pack [--all] [--stdin] [--quiet | -q] [--keep | -k] [--thin] " @@ -52,6 +53,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct fetch_pack_args args; struct oid_array shallow = OID_ARRAY_INIT; struct string_list deepen_not = STRING_LIST_INIT_DUP; + struct packet_reader reader; fetch_if_missing = 0; @@ -211,10 +213,24 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) if (!conn) return args.diag_url ? 0 : 1; } - get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, &shallow); + + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &ref, 0, NULL, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought, - &shallow, pack_lockfile_ptr); + &shallow, pack_lockfile_ptr, protocol_v0); if (pack_lockfile) { printf("lock %s\n", pack_lockfile); fflush(stdout); diff --git a/builtin/fetch.c b/builtin/fetch.c index dcdfc66..7ee83ac 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -264,7 +264,7 @@ static void find_non_local_tags(struct transport *transport, struct string_list_item *item = NULL; for_each_ref(add_existing, &existing_refs); - for (ref = transport_get_remote_refs(transport); ref; ref = ref->next) { + for (ref = transport_get_remote_refs(transport, NULL); ref; ref = ref->next) { if (!starts_with(ref->name, "refs/tags/")) continue; @@ -346,11 +346,28 @@ static struct ref *get_ref_map(struct transport *transport, struct ref *rm; struct ref *ref_map = NULL; struct ref **tail = &ref_map; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; /* opportunistically-updated references: */ struct ref *orefs = NULL, **oref_tail = &orefs; - const struct ref *remote_refs = transport_get_remote_refs(transport); + const struct ref *remote_refs; + + for (i = 0; i < refspec_count; i++) { + if (!refspecs[i].exact_sha1) { + const char *glob = strchr(refspecs[i].src, '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - refspecs[i].src), + refspecs[i].src); + else + expand_ref_prefix(&ref_prefixes, refspecs[i].src); + } + } + + remote_refs = transport_get_remote_refs(transport, &ref_prefixes); + + argv_array_clear(&ref_prefixes); if (refspec_count) { struct refspec *fetch_refspec; diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index 540d564..380c180 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -2,6 +2,7 @@ #include "cache.h" #include "transport.h" #include "remote.h" +#include "refs.h" static const char * const ls_remote_usage[] = { N_("git ls-remote [--heads] [--tags] [--refs] [--upload-pack=<exec>]\n" @@ -43,6 +44,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) int show_symref_target = 0; const char *uploadpack = NULL; const char **pattern = NULL; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; struct remote *remote; struct transport *transport; @@ -75,8 +77,17 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (argc > 1) { int i; pattern = xcalloc(argc, sizeof(const char *)); - for (i = 1; i < argc; i++) + for (i = 1; i < argc; i++) { + const char *glob; pattern[i - 1] = xstrfmt("*/%s", argv[i]); + + glob = strchr(argv[i], '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - argv[i]), argv[i]); + else + expand_ref_prefix(&ref_prefixes, argv[i]); + } } remote = remote_get(dest); @@ -97,7 +108,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) if (uploadpack != NULL) transport_set_option(transport, TRANS_OPT_UPLOADPACK, uploadpack); - ref = transport_get_remote_refs(transport); + ref = transport_get_remote_refs(transport, &ref_prefixes); if (transport_disconnect(transport)) return 1; diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 4b68a28..0dd1632 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -1965,6 +1965,12 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) unpack_limit = receive_unpack_limit; switch (determine_protocol_version_server()) { + case protocol_v2: + /* + * push support for protocol v2 has not been implemented yet, + * so ignore the request to use v2 and fallback to using v0. + */ + break; case protocol_v1: /* * v1 is just the original protocol with a version string, diff --git a/builtin/remote.c b/builtin/remote.c index 805ffc0..8708e58 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -862,7 +862,7 @@ static int get_remote_ref_states(const char *name, if (query) { transport = transport_get(states->remote, states->remote->url_nr > 0 ? states->remote->url[0] : NULL); - remote_refs = transport_get_remote_refs(transport); + remote_refs = transport_get_remote_refs(transport, NULL); transport_disconnect(transport); states->queried = 1; diff --git a/builtin/send-pack.c b/builtin/send-pack.c index fc4f0bb..b5427f7 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -14,6 +14,7 @@ #include "sha1-array.h" #include "gpg-interface.h" #include "gettext.h" +#include "protocol.h" static const char * const send_pack_usage[] = { N_("git send-pack [--all | --mirror] [--dry-run] [--force] " @@ -154,6 +155,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) int progress = -1; int from_stdin = 0; struct push_cas_option cas = {0}; + struct packet_reader reader; struct option options[] = { OPT__VERBOSITY(&verbose), @@ -256,8 +258,22 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) args.verbose ? CONNECT_VERBOSE : 0); } - get_remote_heads(fd[0], NULL, 0, &remote_refs, REF_NORMAL, - &extra_have, &shallow); + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + switch (discover_version(&reader)) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &remote_refs, REF_NORMAL, + &extra_have, &shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } transport_verify_remote_names(nr_refspecs, refspecs); diff --git a/builtin/serve.c b/builtin/serve.c new file mode 100644 index 0000000..d3fd240 --- /dev/null +++ b/builtin/serve.c @@ -0,0 +1,30 @@ +#include "cache.h" +#include "builtin.h" +#include "parse-options.h" +#include "serve.h" + +static char const * const serve_usage[] = { + N_("git serve [<options>]"), + NULL +}; + +int cmd_serve(int argc, const char **argv, const char *prefix) +{ + struct serve_options opts = SERVE_OPTIONS_INIT; + + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-capabilities", &opts.advertise_capabilities, + N_("exit immediately after advertising capabilities")), + OPT_END() + }; + + /* ignore all unknown cmdline switches for now */ + argc = parse_options(argc, argv, prefix, options, serve_usage, + PARSE_OPT_KEEP_DASHDASH | + PARSE_OPT_KEEP_UNKNOWN); + serve(&opts); + + return 0; +} diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c new file mode 100644 index 0000000..decde5a --- /dev/null +++ b/builtin/upload-pack.c @@ -0,0 +1,74 @@ +#include "cache.h" +#include "builtin.h" +#include "exec-cmd.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "protocol.h" +#include "upload-pack.h" +#include "serve.h" + +static const char * const upload_pack_usage[] = { + N_("git upload-pack [<options>] <dir>"), + NULL +}; + +int cmd_upload_pack(int argc, const char **argv, const char *prefix) +{ + const char *dir; + int strict = 0; + struct upload_pack_options opts = { 0 }; + struct serve_options serve_opts = SERVE_OPTIONS_INIT; + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-refs", &opts.advertise_refs, + N_("exit immediately after initial ref advertisement")), + OPT_BOOL(0, "strict", &strict, + N_("do not try <directory>/.git/ if <directory> is no Git directory")), + OPT_INTEGER(0, "timeout", &opts.timeout, + N_("interrupt transfer after <n> seconds of inactivity")), + OPT_END() + }; + + packet_trace_identity("upload-pack"); + check_replace_refs = 0; + + argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); + + if (argc != 1) + usage_with_options(upload_pack_usage, options); + + if (opts.timeout) + opts.daemon_mode = 1; + + setup_path(); + + dir = argv[0]; + + if (!enter_repo(dir, strict)) + die("'%s' does not appear to be a git repository", dir); + + switch (determine_protocol_version_server()) { + case protocol_v2: + serve_opts.advertise_capabilities = opts.advertise_refs; + serve_opts.stateless_rpc = opts.stateless_rpc; + serve(&serve_opts); + break; + case protocol_v1: + /* + * v1 is just the original protocol with a version string, + * so just fall through after writing the version string. + */ + if (opts.advertise_refs || !opts.stateless_rpc) + packet_write_fmt(1, "version 1\n"); + + /* fallthrough */ + case protocol_v0: + upload_pack(&opts); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return 0; +} @@ -12,9 +12,11 @@ #include "sha1-array.h" #include "transport.h" #include "strbuf.h" +#include "version.h" #include "protocol.h" -static char *server_capabilities; +static char *server_capabilities_v1; +static struct argv_array server_capabilities_v2 = ARGV_ARRAY_INIT; static const char *parse_feature_value(const char *, const char *, int *); static int check_ref(const char *name, unsigned int flags) @@ -48,6 +50,12 @@ int check_ref_type(const struct ref *ref, int flags) static void die_initial_contact(int unexpected) { + /* + * A hang-up after seeing some response from the other end + * means that it is unexpected, as we know the other end is + * willing to talk to us. A hang-up before seeing any + * response does not necessarily mean an ACL problem, though. + */ if (unexpected) die(_("The remote end hung up upon initial contact")); else @@ -56,6 +64,92 @@ static void die_initial_contact(int unexpected) "and the repository exists.")); } +/* Checks if the server supports the capability 'c' */ +int server_supports_v2(const char *c, int die_on_error) +{ + int i; + + for (i = 0; i < server_capabilities_v2.argc; i++) { + const char *out; + if (skip_prefix(server_capabilities_v2.argv[i], c, &out) && + (!*out || *out == '=')) + return 1; + } + + if (die_on_error) + die("server doesn't support '%s'", c); + + return 0; +} + +int server_supports_feature(const char *c, const char *feature, + int die_on_error) +{ + int i; + + for (i = 0; i < server_capabilities_v2.argc; i++) { + const char *out; + if (skip_prefix(server_capabilities_v2.argv[i], c, &out) && + (!*out || *(out++) == '=')) { + if (parse_feature_request(out, feature)) + return 1; + else + break; + } + } + + if (die_on_error) + die("server doesn't support feature '%s'", feature); + + return 0; +} + +static void process_capabilities_v2(struct packet_reader *reader) +{ + while (packet_reader_read(reader) == PACKET_READ_NORMAL) + argv_array_push(&server_capabilities_v2, reader->line); + + if (reader->status != PACKET_READ_FLUSH) + die("expected flush after capabilities"); +} + +enum protocol_version discover_version(struct packet_reader *reader) +{ + enum protocol_version version = protocol_unknown_version; + + /* + * Peek the first line of the server's response to + * determine the protocol version the server is speaking. + */ + switch (packet_reader_peek(reader)) { + case PACKET_READ_EOF: + die_initial_contact(0); + case PACKET_READ_FLUSH: + case PACKET_READ_DELIM: + version = protocol_v0; + break; + case PACKET_READ_NORMAL: + version = determine_protocol_version_client(reader->line); + break; + } + + switch (version) { + case protocol_v2: + process_capabilities_v2(reader); + break; + case protocol_v1: + /* Read the peeked version line */ + packet_reader_read(reader); + break; + case protocol_v0: + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return version; +} + static void parse_one_symref_info(struct string_list *symref, const char *val, int len) { char *sym, *target; @@ -85,7 +179,7 @@ reject: static void annotate_refs_with_symref_info(struct ref *ref) { struct string_list symref = STRING_LIST_INIT_DUP; - const char *feature_list = server_capabilities; + const char *feature_list = server_capabilities_v1; while (feature_list) { int len; @@ -109,60 +203,21 @@ static void annotate_refs_with_symref_info(struct ref *ref) string_list_clear(&symref, 0); } -/* - * Read one line of a server's ref advertisement into packet_buffer. - */ -static int read_remote_ref(int in, char **src_buf, size_t *src_len, - int *responded) +static void process_capabilities(const char *line, int *len) { - int len = packet_read(in, src_buf, src_len, - packet_buffer, sizeof(packet_buffer), - PACKET_READ_GENTLE_ON_EOF | - PACKET_READ_CHOMP_NEWLINE); - const char *arg; - if (len < 0) - die_initial_contact(*responded); - if (len > 4 && skip_prefix(packet_buffer, "ERR ", &arg)) - die("remote error: %s", arg); - - *responded = 1; - - return len; -} - -#define EXPECTING_PROTOCOL_VERSION 0 -#define EXPECTING_FIRST_REF 1 -#define EXPECTING_REF 2 -#define EXPECTING_SHALLOW 3 - -/* Returns 1 if packet_buffer is a protocol version pkt-line, 0 otherwise. */ -static int process_protocol_version(void) -{ - switch (determine_protocol_version_client(packet_buffer)) { - case protocol_v1: - return 1; - case protocol_v0: - return 0; - default: - die("server is speaking an unknown protocol"); - } -} - -static void process_capabilities(int *len) -{ - int nul_location = strlen(packet_buffer); + int nul_location = strlen(line); if (nul_location == *len) return; - server_capabilities = xstrdup(packet_buffer + nul_location + 1); + server_capabilities_v1 = xstrdup(line + nul_location + 1); *len = nul_location; } -static int process_dummy_ref(void) +static int process_dummy_ref(const char *line) { struct object_id oid; const char *name; - if (parse_oid_hex(packet_buffer, &oid, &name)) + if (parse_oid_hex(line, &oid, &name)) return 0; if (*name != ' ') return 0; @@ -171,20 +226,20 @@ static int process_dummy_ref(void) return !oidcmp(&null_oid, &oid) && !strcmp(name, "capabilities^{}"); } -static void check_no_capabilities(int len) +static void check_no_capabilities(const char *line, int len) { - if (strlen(packet_buffer) != len) + if (strlen(line) != len) warning("Ignoring capabilities after first line '%s'", - packet_buffer + strlen(packet_buffer)); + line + strlen(line)); } -static int process_ref(int len, struct ref ***list, unsigned int flags, - struct oid_array *extra_have) +static int process_ref(const char *line, int len, struct ref ***list, + unsigned int flags, struct oid_array *extra_have) { struct object_id old_oid; const char *name; - if (parse_oid_hex(packet_buffer, &old_oid, &name)) + if (parse_oid_hex(line, &old_oid, &name)) return 0; if (*name != ' ') return 0; @@ -200,16 +255,17 @@ static int process_ref(int len, struct ref ***list, unsigned int flags, **list = ref; *list = &ref->next; } - check_no_capabilities(len); + check_no_capabilities(line, len); return 1; } -static int process_shallow(int len, struct oid_array *shallow_points) +static int process_shallow(const char *line, int len, + struct oid_array *shallow_points) { const char *arg; struct object_id old_oid; - if (!skip_prefix(packet_buffer, "shallow ", &arg)) + if (!skip_prefix(line, "shallow ", &arg)) return 0; if (get_oid_hex(arg, &old_oid)) @@ -217,60 +273,68 @@ static int process_shallow(int len, struct oid_array *shallow_points) if (!shallow_points) die("repository on the other end cannot be shallow"); oid_array_append(shallow_points, &old_oid); - check_no_capabilities(len); + check_no_capabilities(line, len); return 1; } +enum get_remote_heads_state { + EXPECTING_FIRST_REF = 0, + EXPECTING_REF, + EXPECTING_SHALLOW, + EXPECTING_DONE, +}; + /* * Read all the refs from the other end */ -struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, struct oid_array *shallow_points) { struct ref **orig_list = list; - - /* - * A hang-up after seeing some response from the other end - * means that it is unexpected, as we know the other end is - * willing to talk to us. A hang-up before seeing any - * response does not necessarily mean an ACL problem, though. - */ - int responded = 0; - int len; - int state = EXPECTING_PROTOCOL_VERSION; + int len = 0; + enum get_remote_heads_state state = EXPECTING_FIRST_REF; + const char *arg; *list = NULL; - while ((len = read_remote_ref(in, &src_buf, &src_len, &responded))) { + while (state != EXPECTING_DONE) { + switch (packet_reader_read(reader)) { + case PACKET_READ_EOF: + die_initial_contact(1); + case PACKET_READ_NORMAL: + len = reader->pktlen; + if (len > 4 && skip_prefix(reader->line, "ERR ", &arg)) + die("remote error: %s", arg); + break; + case PACKET_READ_FLUSH: + state = EXPECTING_DONE; + break; + case PACKET_READ_DELIM: + die("invalid packet"); + } + switch (state) { - case EXPECTING_PROTOCOL_VERSION: - if (process_protocol_version()) { - state = EXPECTING_FIRST_REF; - break; - } - state = EXPECTING_FIRST_REF; - /* fallthrough */ case EXPECTING_FIRST_REF: - process_capabilities(&len); - if (process_dummy_ref()) { + process_capabilities(reader->line, &len); + if (process_dummy_ref(reader->line)) { state = EXPECTING_SHALLOW; break; } state = EXPECTING_REF; /* fallthrough */ case EXPECTING_REF: - if (process_ref(len, &list, flags, extra_have)) + if (process_ref(reader->line, len, &list, flags, extra_have)) break; state = EXPECTING_SHALLOW; /* fallthrough */ case EXPECTING_SHALLOW: - if (process_shallow(len, shallow_points)) + if (process_shallow(reader->line, len, shallow_points)) break; - die("protocol error: unexpected '%s'", packet_buffer); - default: - die("unexpected state %d", state); + die("protocol error: unexpected '%s'", reader->line); + case EXPECTING_DONE: + break; } } @@ -279,6 +343,105 @@ struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, return list; } +/* Returns 1 when a valid ref has been added to `list`, 0 otherwise */ +static int process_ref_v2(const char *line, struct ref ***list) +{ + int ret = 1; + int i = 0; + struct object_id old_oid; + struct ref *ref; + struct string_list line_sections = STRING_LIST_INIT_DUP; + const char *end; + + /* + * Ref lines have a number of fields which are space deliminated. The + * first field is the OID of the ref. The second field is the ref + * name. Subsequent fields (symref-target and peeled) are optional and + * don't have a particular order. + */ + if (string_list_split(&line_sections, line, ' ', -1) < 2) { + ret = 0; + goto out; + } + + if (parse_oid_hex(line_sections.items[i++].string, &old_oid, &end) || + *end) { + ret = 0; + goto out; + } + + ref = alloc_ref(line_sections.items[i++].string); + + oidcpy(&ref->old_oid, &old_oid); + **list = ref; + *list = &ref->next; + + for (; i < line_sections.nr; i++) { + const char *arg = line_sections.items[i].string; + if (skip_prefix(arg, "symref-target:", &arg)) + ref->symref = xstrdup(arg); + + if (skip_prefix(arg, "peeled:", &arg)) { + struct object_id peeled_oid; + char *peeled_name; + struct ref *peeled; + if (parse_oid_hex(arg, &peeled_oid, &end) || *end) { + ret = 0; + goto out; + } + + peeled_name = xstrfmt("%s^{}", ref->name); + peeled = alloc_ref(peeled_name); + + oidcpy(&peeled->old_oid, &peeled_oid); + **list = peeled; + *list = &peeled->next; + + free(peeled_name); + } + } + +out: + string_list_clear(&line_sections, 0); + return ret; +} + +struct ref **get_remote_refs(int fd_out, struct packet_reader *reader, + struct ref **list, int for_push, + const struct argv_array *ref_prefixes) +{ + int i; + *list = NULL; + + if (server_supports_v2("ls-refs", 1)) + packet_write_fmt(fd_out, "command=ls-refs\n"); + + if (server_supports_v2("agent", 0)) + packet_write_fmt(fd_out, "agent=%s", git_user_agent_sanitized()); + + packet_delim(fd_out); + /* When pushing we don't want to request the peeled tags */ + if (!for_push) + packet_write_fmt(fd_out, "peel\n"); + packet_write_fmt(fd_out, "symrefs\n"); + for (i = 0; ref_prefixes && i < ref_prefixes->argc; i++) { + packet_write_fmt(fd_out, "ref-prefix %s\n", + ref_prefixes->argv[i]); + } + packet_flush(fd_out); + + /* Process response from server */ + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + if (!process_ref_v2(reader->line, &list)) + die("invalid ls-refs response: %s", reader->line); + } + + if (reader->status != PACKET_READ_FLUSH) + die("expected flush after ref listing"); + + return list; +} + static const char *parse_feature_value(const char *feature_list, const char *feature, int *lenp) { int len; @@ -323,7 +486,7 @@ int parse_feature_request(const char *feature_list, const char *feature) const char *server_feature_value(const char *feature, int *len) { - return parse_feature_value(server_capabilities, feature, len); + return parse_feature_value(server_capabilities_v1, feature, len); } int server_supports(const char *feature) @@ -872,6 +1035,7 @@ static enum ssh_variant determine_ssh_variant(const char *ssh_command, */ static struct child_process *git_connect_git(int fd[2], char *hostandport, const char *path, const char *prog, + enum protocol_version version, int flags) { struct child_process *conn; @@ -910,10 +1074,10 @@ static struct child_process *git_connect_git(int fd[2], char *hostandport, target_host, 0); /* If using a new version put that stuff here after a second null byte */ - if (get_protocol_version_config() > 0) { + if (version > 0) { strbuf_addch(&request, '\0'); strbuf_addf(&request, "version=%d%c", - get_protocol_version_config(), '\0'); + version, '\0'); } packet_write(fd[1], request.buf, request.len); @@ -929,14 +1093,14 @@ static struct child_process *git_connect_git(int fd[2], char *hostandport, */ static void push_ssh_options(struct argv_array *args, struct argv_array *env, enum ssh_variant variant, const char *port, - int flags) + enum protocol_version version, int flags) { if (variant == VARIANT_SSH && - get_protocol_version_config() > 0) { + version > 0) { argv_array_push(args, "-o"); argv_array_push(args, "SendEnv=" GIT_PROTOCOL_ENVIRONMENT); argv_array_pushf(env, GIT_PROTOCOL_ENVIRONMENT "=version=%d", - get_protocol_version_config()); + version); } if (flags & CONNECT_IPV4) { @@ -989,7 +1153,8 @@ static void push_ssh_options(struct argv_array *args, struct argv_array *env, /* Prepare a child_process for use by Git's SSH-tunneled transport. */ static void fill_ssh_args(struct child_process *conn, const char *ssh_host, - const char *port, int flags) + const char *port, enum protocol_version version, + int flags) { const char *ssh; enum ssh_variant variant; @@ -1023,14 +1188,14 @@ static void fill_ssh_args(struct child_process *conn, const char *ssh_host, argv_array_push(&detect.args, ssh); argv_array_push(&detect.args, "-G"); push_ssh_options(&detect.args, &detect.env_array, - VARIANT_SSH, port, flags); + VARIANT_SSH, port, version, flags); argv_array_push(&detect.args, ssh_host); variant = run_command(&detect) ? VARIANT_SIMPLE : VARIANT_SSH; } argv_array_push(&conn->args, ssh); - push_ssh_options(&conn->args, &conn->env_array, variant, port, flags); + push_ssh_options(&conn->args, &conn->env_array, variant, port, version, flags); argv_array_push(&conn->args, ssh_host); } @@ -1051,6 +1216,15 @@ struct child_process *git_connect(int fd[2], const char *url, char *hostandport, *path; struct child_process *conn; enum protocol protocol; + enum protocol_version version = get_protocol_version_config(); + + /* + * NEEDSWORK: If we are trying to use protocol v2 and we are planning + * to perform a push, then fallback to v0 since the client doesn't know + * how to push yet using v2. + */ + if (version == protocol_v2 && !strcmp("git-receive-pack", prog)) + version = protocol_v0; /* Without this we cannot rely on waitpid() to tell * what happened to our children. @@ -1065,7 +1239,7 @@ struct child_process *git_connect(int fd[2], const char *url, printf("Diag: path=%s\n", path ? path : "NULL"); conn = NULL; } else if (protocol == PROTO_GIT) { - conn = git_connect_git(fd, hostandport, path, prog, flags); + conn = git_connect_git(fd, hostandport, path, prog, version, flags); } else { struct strbuf cmd = STRBUF_INIT; const char *const *var; @@ -1108,12 +1282,12 @@ struct child_process *git_connect(int fd[2], const char *url, strbuf_release(&cmd); return NULL; } - fill_ssh_args(conn, ssh_host, port, flags); + fill_ssh_args(conn, ssh_host, port, version, flags); } else { transport_check_allowed("file"); - if (get_protocol_version_config() > 0) { + if (version > 0) { argv_array_pushf(&conn->env_array, GIT_PROTOCOL_ENVIRONMENT "=version=%d", - get_protocol_version_config()); + version); } } argv_array_push(&conn->args, cmd.buf); @@ -13,4 +13,11 @@ extern int parse_feature_request(const char *features, const char *feature); extern const char *server_feature_value(const char *feature, int *len_ret); extern int url_is_local_not_ssh(const char *url); +struct packet_reader; +extern enum protocol_version discover_version(struct packet_reader *reader); + +extern int server_supports_v2(const char *c, int die_on_error); +extern int server_supports_feature(const char *c, const char *feature, + int die_on_error); + #endif diff --git a/fetch-pack.c b/fetch-pack.c index 4a8bad8..f93723f 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -305,9 +305,9 @@ static void insert_one_alternate_object(struct object *obj) #define PIPESAFE_FLUSH 32 #define LARGE_FLUSH 16384 -static int next_flush(struct fetch_pack_args *args, int count) +static int next_flush(int stateless_rpc, int count) { - if (args->stateless_rpc) { + if (stateless_rpc) { if (count < LARGE_FLUSH) count <<= 1; else @@ -470,7 +470,7 @@ static int find_common(struct fetch_pack_args *args, send_request(args, fd[1], &req_buf); strbuf_setlen(&req_buf, state_len); flushes++; - flush_at = next_flush(args, count); + flush_at = next_flush(args->stateless_rpc, count); /* * We keep one window "ahead" of the other side, and @@ -1080,6 +1080,328 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, return ref; } +static void add_shallow_requests(struct strbuf *req_buf, + const struct fetch_pack_args *args) +{ + if (is_repository_shallow()) + write_shallow_commits(req_buf, 1, NULL); + if (args->depth > 0) + packet_buf_write(req_buf, "deepen %d", args->depth); + if (args->deepen_since) { + timestamp_t max_age = approxidate(args->deepen_since); + packet_buf_write(req_buf, "deepen-since %"PRItime, max_age); + } + if (args->deepen_not) { + int i; + for (i = 0; i < args->deepen_not->nr; i++) { + struct string_list_item *s = args->deepen_not->items + i; + packet_buf_write(req_buf, "deepen-not %s", s->string); + } + } +} + +static void add_wants(const struct ref *wants, struct strbuf *req_buf) +{ + for ( ; wants ; wants = wants->next) { + const struct object_id *remote = &wants->old_oid; + const char *remote_hex; + struct object *o; + + /* + * If that object is complete (i.e. it is an ancestor of a + * local ref), we tell them we have it but do not have to + * tell them about its ancestors, which they already know + * about. + * + * We use lookup_object here because we are only + * interested in the case we *know* the object is + * reachable and we have already scanned it. + */ + if (((o = lookup_object(remote->hash)) != NULL) && + (o->flags & COMPLETE)) { + continue; + } + + remote_hex = oid_to_hex(remote); + packet_buf_write(req_buf, "want %s\n", remote_hex); + } +} + +static void add_common(struct strbuf *req_buf, struct oidset *common) +{ + struct oidset_iter iter; + const struct object_id *oid; + oidset_iter_init(common, &iter); + + while ((oid = oidset_iter_next(&iter))) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + } +} + +static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain) +{ + int ret = 0; + int haves_added = 0; + const struct object_id *oid; + + while ((oid = get_rev())) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + if (++haves_added >= *haves_to_send) + break; + } + + *in_vain += haves_added; + if (!haves_added || *in_vain >= MAX_IN_VAIN) { + /* Send Done */ + packet_buf_write(req_buf, "done\n"); + ret = 1; + } + + /* Increase haves to send on next round */ + *haves_to_send = next_flush(1, *haves_to_send); + + return ret; +} + +static int send_fetch_request(int fd_out, const struct fetch_pack_args *args, + const struct ref *wants, struct oidset *common, + int *haves_to_send, int *in_vain) +{ + int ret = 0; + struct strbuf req_buf = STRBUF_INIT; + + if (server_supports_v2("fetch", 1)) + packet_buf_write(&req_buf, "command=fetch"); + if (server_supports_v2("agent", 0)) + packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized()); + + packet_buf_delim(&req_buf); + if (args->use_thin_pack) + packet_buf_write(&req_buf, "thin-pack"); + if (args->no_progress) + packet_buf_write(&req_buf, "no-progress"); + if (args->include_tag) + packet_buf_write(&req_buf, "include-tag"); + if (prefer_ofs_delta) + packet_buf_write(&req_buf, "ofs-delta"); + + /* Add shallow-info and deepen request */ + if (server_supports_feature("fetch", "shallow", 0)) + add_shallow_requests(&req_buf, args); + else if (is_repository_shallow() || args->deepen) + die(_("Server does not support shallow requests")); + + /* add wants */ + add_wants(wants, &req_buf); + + /* Add all of the common commits we've found in previous rounds */ + add_common(&req_buf, common); + + /* Add initial haves */ + ret = add_haves(&req_buf, haves_to_send, in_vain); + + /* Send request */ + packet_buf_flush(&req_buf); + write_or_die(fd_out, req_buf.buf, req_buf.len); + + strbuf_release(&req_buf); + return ret; +} + +/* + * Processes a section header in a server's response and checks if it matches + * `section`. If the value of `peek` is 1, the header line will be peeked (and + * not consumed); if 0, the line will be consumed and the function will die if + * the section header doesn't match what was expected. + */ +static int process_section_header(struct packet_reader *reader, + const char *section, int peek) +{ + int ret; + + if (packet_reader_peek(reader) != PACKET_READ_NORMAL) + die("error reading section header '%s'", section); + + ret = !strcmp(reader->line, section); + + if (!peek) { + if (!ret) + die("expected '%s', received '%s'", + section, reader->line); + packet_reader_read(reader); + } + + return ret; +} + +static int process_acks(struct packet_reader *reader, struct oidset *common) +{ + /* received */ + int received_ready = 0; + int received_ack = 0; + + process_section_header(reader, "acknowledgments", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + + if (!strcmp(reader->line, "NAK")) + continue; + + if (skip_prefix(reader->line, "ACK ", &arg)) { + struct object_id oid; + if (!get_oid_hex(arg, &oid)) { + struct commit *commit; + oidset_insert(common, &oid); + commit = lookup_commit(&oid); + mark_common(commit, 0, 1); + } + continue; + } + + if (!strcmp(reader->line, "ready")) { + clear_prio_queue(&rev_list); + received_ready = 1; + continue; + } + + die("unexpected acknowledgment line: '%s'", reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die("error processing acks: %d", reader->status); + + /* return 0 if no common, 1 if there are common, or 2 if ready */ + return received_ready ? 2 : (received_ack ? 1 : 0); +} + +static void receive_shallow_info(struct fetch_pack_args *args, + struct packet_reader *reader) +{ + process_section_header(reader, "shallow-info", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + struct object_id oid; + + if (skip_prefix(reader->line, "shallow ", &arg)) { + if (get_oid_hex(arg, &oid)) + die(_("invalid shallow line: %s"), reader->line); + register_shallow(&oid); + continue; + } + if (skip_prefix(reader->line, "unshallow ", &arg)) { + if (get_oid_hex(arg, &oid)) + die(_("invalid unshallow line: %s"), reader->line); + if (!lookup_object(oid.hash)) + die(_("object not found: %s"), reader->line); + /* make sure that it is parsed as shallow */ + if (!parse_object(&oid)) + die(_("error in object: %s"), reader->line); + if (unregister_shallow(&oid)) + die(_("no shallow found: %s"), reader->line); + continue; + } + die(_("expected shallow/unshallow, got %s"), reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die("error processing shallow info: %d", reader->status); + + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); + args->deepen = 1; +} + +enum fetch_state { + FETCH_CHECK_LOCAL = 0, + FETCH_SEND_REQUEST, + FETCH_PROCESS_ACKS, + FETCH_GET_PACK, + FETCH_DONE, +}; + +static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, + int fd[2], + const struct ref *orig_ref, + struct ref **sought, int nr_sought, + char **pack_lockfile) +{ + struct ref *ref = copy_ref_list(orig_ref); + enum fetch_state state = FETCH_CHECK_LOCAL; + struct oidset common = OIDSET_INIT; + struct packet_reader reader; + int in_vain = 0; + int haves_to_send = INITIAL_FLUSH; + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE); + + while (state != FETCH_DONE) { + switch (state) { + case FETCH_CHECK_LOCAL: + sort_ref_list(&ref, ref_compare_name); + QSORT(sought, nr_sought, cmp_ref_by_name); + + /* v2 supports these by default */ + allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; + use_sideband = 2; + if (args->depth > 0 || args->deepen_since || args->deepen_not) + args->deepen = 1; + + if (marked) + for_each_ref(clear_marks, NULL); + marked = 1; + + for_each_ref(rev_list_insert_ref_oid, NULL); + for_each_cached_alternate(insert_one_alternate_object); + + /* Filter 'ref' by 'sought' and those that aren't local */ + if (everything_local(args, &ref, sought, nr_sought)) + state = FETCH_DONE; + else + state = FETCH_SEND_REQUEST; + break; + case FETCH_SEND_REQUEST: + if (send_fetch_request(fd[1], args, ref, &common, + &haves_to_send, &in_vain)) + state = FETCH_GET_PACK; + else + state = FETCH_PROCESS_ACKS; + break; + case FETCH_PROCESS_ACKS: + /* Process ACKs/NAKs */ + switch (process_acks(&reader, &common)) { + case 2: + state = FETCH_GET_PACK; + break; + case 1: + in_vain = 0; + /* fallthrough */ + default: + state = FETCH_SEND_REQUEST; + break; + } + break; + case FETCH_GET_PACK: + /* Check for shallow-info section */ + if (process_section_header(&reader, "shallow-info", 1)) + receive_shallow_info(args, &reader); + + /* get the pack */ + process_section_header(&reader, "packfile", 0); + if (get_pack(args, fd, pack_lockfile)) + die(_("git fetch-pack: fetch failed.")); + + state = FETCH_DONE; + break; + case FETCH_DONE: + continue; + } + } + + oidset_clear(&common); + return ref; +} + static void fetch_pack_config(void) { git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit); @@ -1225,7 +1547,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const char *dest, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile) + char **pack_lockfile, + enum protocol_version version) { struct ref *ref_cpy; struct shallow_info si; @@ -1239,8 +1562,12 @@ struct ref *fetch_pack(struct fetch_pack_args *args, die(_("no matching remote head")); } prepare_shallow_info(&si, shallow); - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, - &si, pack_lockfile); + if (version == protocol_v2) + ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought, + pack_lockfile); + else + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); reprepare_packed_git(the_repository); update_shallow(args, sought, nr_sought, &si); clear_shallow_info(&si); diff --git a/fetch-pack.h b/fetch-pack.h index 3e224a1..6afa08b 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -3,6 +3,7 @@ #include "string-list.h" #include "run-command.h" +#include "protocol.h" #include "list-objects-filter-options.h" struct oid_array; @@ -53,7 +54,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile); + char **pack_lockfile, + enum protocol_version version); /* * Print an appropriate error message for each sought ref that wasn't @@ -465,6 +465,7 @@ static struct cmd_struct commands[] = { { "revert", cmd_revert, RUN_SETUP | NEED_WORK_TREE }, { "rm", cmd_rm, RUN_SETUP }, { "send-pack", cmd_send_pack, RUN_SETUP }, + { "serve", cmd_serve, RUN_SETUP }, { "shortlog", cmd_shortlog, RUN_SETUP_GENTLY | USE_PAGER }, { "show", cmd_show, RUN_SETUP }, { "show-branch", cmd_show_branch, RUN_SETUP }, @@ -482,6 +483,7 @@ static struct cmd_struct commands[] = { { "update-server-info", cmd_update_server_info, RUN_SETUP }, { "upload-archive", cmd_upload_archive, NO_PARSEOPT }, { "upload-archive--writer", cmd_upload_archive_writer, NO_PARSEOPT }, + { "upload-pack", cmd_upload_pack }, { "var", cmd_var, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "verify-commit", cmd_verify_commit, RUN_SETUP }, { "verify-pack", cmd_verify_pack }, diff --git a/http-backend.c b/http-backend.c index cc16cd0..adaef16 100644 --- a/http-backend.c +++ b/http-backend.c @@ -12,6 +12,7 @@ #include "argv-array.h" #include "packfile.h" #include "object-store.h" +#include "protocol.h" static const char content_type[] = "Content-Type"; static const char content_length[] = "Content-Length"; @@ -468,8 +469,11 @@ static void get_info_refs(struct strbuf *hdr, char *arg) hdr_str(hdr, content_type, buf.buf); end_headers(hdr); - packet_write_fmt(1, "# service=git-%s\n", svc->name); - packet_flush(1); + + if (determine_protocol_version_server() != protocol_v2) { + packet_write_fmt(1, "# service=git-%s\n", svc->name); + packet_flush(1); + } argv[0] = svc->name; run_service(argv, 0); @@ -976,21 +976,6 @@ static void set_from_env(const char **var, const char *envname) *var = val; } -static void protocol_http_header(void) -{ - if (get_protocol_version_config() > 0) { - struct strbuf protocol_header = STRBUF_INIT; - - strbuf_addf(&protocol_header, GIT_PROTOCOL_HEADER ": version=%d", - get_protocol_version_config()); - - - extra_http_headers = curl_slist_append(extra_http_headers, - protocol_header.buf); - strbuf_release(&protocol_header); - } -} - void http_init(struct remote *remote, const char *url, int proactive_auth) { char *low_speed_limit; @@ -1021,8 +1006,6 @@ void http_init(struct remote *remote, const char *url, int proactive_auth) if (remote) var_override(&http_proxy_authmethod, remote->http_proxy_authmethod); - protocol_http_header(); - pragma_header = curl_slist_append(http_copy_default_headers(), "Pragma: no-cache"); no_pragma_header = curl_slist_append(http_copy_default_headers(), @@ -1795,6 +1778,14 @@ static int http_request(const char *url, headers = curl_slist_append(headers, buf.buf); + /* Add additional headers here */ + if (options && options->extra_headers) { + const struct string_list_item *item; + for_each_string_list_item(item, options->extra_headers) { + headers = curl_slist_append(headers, item->string); + } + } + curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip"); @@ -172,6 +172,13 @@ struct http_get_options { * for details. */ struct strbuf *base_url; + + /* + * If not NULL, contains additional HTTP headers to be sent with the + * request. The strings in the list must not be freed until after the + * request has completed. + */ + struct string_list *extra_headers; }; /* Return values for http_get_*() */ diff --git a/ls-refs.c b/ls-refs.c new file mode 100644 index 0000000..a06f12e --- /dev/null +++ b/ls-refs.c @@ -0,0 +1,96 @@ +#include "cache.h" +#include "repository.h" +#include "refs.h" +#include "remote.h" +#include "argv-array.h" +#include "ls-refs.h" +#include "pkt-line.h" + +/* + * Check if one of the prefixes is a prefix of the ref. + * If no prefixes were provided, all refs match. + */ +static int ref_match(const struct argv_array *prefixes, const char *refname) +{ + int i; + + if (!prefixes->argc) + return 1; /* no restriction */ + + for (i = 0; i < prefixes->argc; i++) { + const char *prefix = prefixes->argv[i]; + + if (starts_with(refname, prefix)) + return 1; + } + + return 0; +} + +struct ls_refs_data { + unsigned peel; + unsigned symrefs; + struct argv_array prefixes; +}; + +static int send_ref(const char *refname, const struct object_id *oid, + int flag, void *cb_data) +{ + struct ls_refs_data *data = cb_data; + const char *refname_nons = strip_namespace(refname); + struct strbuf refline = STRBUF_INIT; + + if (!ref_match(&data->prefixes, refname)) + return 0; + + strbuf_addf(&refline, "%s %s", oid_to_hex(oid), refname_nons); + if (data->symrefs && flag & REF_ISSYMREF) { + struct object_id unused; + const char *symref_target = resolve_ref_unsafe(refname, 0, + &unused, + &flag); + + if (!symref_target) + die("'%s' is a symref but it is not?", refname); + + strbuf_addf(&refline, " symref-target:%s", symref_target); + } + + if (data->peel) { + struct object_id peeled; + if (!peel_ref(refname, &peeled)) + strbuf_addf(&refline, " peeled:%s", oid_to_hex(&peeled)); + } + + strbuf_addch(&refline, '\n'); + packet_write(1, refline.buf, refline.len); + + strbuf_release(&refline); + return 0; +} + +int ls_refs(struct repository *r, struct argv_array *keys, + struct packet_reader *request) +{ + struct ls_refs_data data; + + memset(&data, 0, sizeof(data)); + + while (packet_reader_read(request) != PACKET_READ_FLUSH) { + const char *arg = request->line; + const char *out; + + if (!strcmp("peel", arg)) + data.peel = 1; + else if (!strcmp("symrefs", arg)) + data.symrefs = 1; + else if (skip_prefix(arg, "ref-prefix ", &out)) + argv_array_push(&data.prefixes, out); + } + + head_ref_namespaced(send_ref, &data); + for_each_namespaced_ref(send_ref, &data); + packet_flush(1); + argv_array_clear(&data.prefixes); + return 0; +} diff --git a/ls-refs.h b/ls-refs.h new file mode 100644 index 0000000..b62877e --- /dev/null +++ b/ls-refs.h @@ -0,0 +1,10 @@ +#ifndef LS_REFS_H +#define LS_REFS_H + +struct repository; +struct argv_array; +struct packet_reader; +extern int ls_refs(struct repository *r, struct argv_array *keys, + struct packet_reader *request); + +#endif /* LS_REFS_H */ @@ -91,6 +91,12 @@ void packet_flush(int fd) write_or_die(fd, "0000", 4); } +void packet_delim(int fd) +{ + packet_trace("0001", 4, 1); + write_or_die(fd, "0001", 4); +} + int packet_flush_gently(int fd) { packet_trace("0000", 4, 1); @@ -105,6 +111,12 @@ void packet_buf_flush(struct strbuf *buf) strbuf_add(buf, "0000", 4); } +void packet_buf_delim(struct strbuf *buf) +{ + packet_trace("0001", 4, 1); + strbuf_add(buf, "0001", 4); +} + static void set_packet_header(char *buf, const int size) { static char hexchar[] = "0123456789abcdef"; @@ -203,6 +215,22 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) va_end(args); } +void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len) +{ + size_t orig_len, n; + + orig_len = buf->len; + strbuf_addstr(buf, "0000"); + strbuf_add(buf, data, len); + n = buf->len - orig_len; + + if (n > LARGE_PACKET_MAX) + die("protocol error: impossibly long line"); + + set_packet_header(&buf->buf[orig_len], n); + packet_trace(data, len, 1); +} + int write_packetized_from_fd(int fd_in, int fd_out) { static char buf[LARGE_PACKET_DATA_MAX]; @@ -280,28 +308,43 @@ static int packet_length(const char *linelen) return (val < 0) ? val : (val << 8) | hex2chr(linelen + 2); } -int packet_read(int fd, char **src_buf, size_t *src_len, - char *buffer, unsigned size, int options) +enum packet_read_status packet_read_with_status(int fd, char **src_buffer, + size_t *src_len, char *buffer, + unsigned size, int *pktlen, + int options) { - int len, ret; + int len; char linelen[4]; - ret = get_packet_data(fd, src_buf, src_len, linelen, 4, options); - if (ret < 0) - return ret; + if (get_packet_data(fd, src_buffer, src_len, linelen, 4, options) < 0) { + *pktlen = -1; + return PACKET_READ_EOF; + } + len = packet_length(linelen); - if (len < 0) + + if (len < 0) { die("protocol error: bad line length character: %.4s", linelen); - if (!len) { + } else if (!len) { packet_trace("0000", 4, 0); - return 0; + *pktlen = 0; + return PACKET_READ_FLUSH; + } else if (len == 1) { + packet_trace("0001", 4, 0); + *pktlen = 0; + return PACKET_READ_DELIM; + } else if (len < 4) { + die("protocol error: bad line length %d", len); } + len -= 4; - if (len >= size) + if ((unsigned)len >= size) die("protocol error: bad line length %d", len); - ret = get_packet_data(fd, src_buf, src_len, buffer, len, options); - if (ret < 0) - return ret; + + if (get_packet_data(fd, src_buffer, src_len, buffer, len, options) < 0) { + *pktlen = -1; + return PACKET_READ_EOF; + } if ((options & PACKET_READ_CHOMP_NEWLINE) && len && buffer[len-1] == '\n') @@ -309,7 +352,19 @@ int packet_read(int fd, char **src_buf, size_t *src_len, buffer[len] = 0; packet_trace(buffer, len, 0); - return len; + *pktlen = len; + return PACKET_READ_NORMAL; +} + +int packet_read(int fd, char **src_buffer, size_t *src_len, + char *buffer, unsigned size, int options) +{ + int pktlen = -1; + + packet_read_with_status(fd, src_buffer, src_len, buffer, size, + &pktlen, options); + + return pktlen; } static char *packet_read_line_generic(int fd, @@ -377,3 +432,53 @@ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out) } return sb_out->len - orig_len; } + +/* Packet Reader Functions */ +void packet_reader_init(struct packet_reader *reader, int fd, + char *src_buffer, size_t src_len, + int options) +{ + memset(reader, 0, sizeof(*reader)); + + reader->fd = fd; + reader->src_buffer = src_buffer; + reader->src_len = src_len; + reader->buffer = packet_buffer; + reader->buffer_size = sizeof(packet_buffer); + reader->options = options; +} + +enum packet_read_status packet_reader_read(struct packet_reader *reader) +{ + if (reader->line_peeked) { + reader->line_peeked = 0; + return reader->status; + } + + reader->status = packet_read_with_status(reader->fd, + &reader->src_buffer, + &reader->src_len, + reader->buffer, + reader->buffer_size, + &reader->pktlen, + reader->options); + + if (reader->status == PACKET_READ_NORMAL) + reader->line = reader->buffer; + else + reader->line = NULL; + + return reader->status; +} + +enum packet_read_status packet_reader_peek(struct packet_reader *reader) +{ + /* Only allow peeking a single line */ + if (reader->line_peeked) + return reader->status; + + /* Peek a line by reading it and setting peeked flag */ + packet_reader_read(reader); + reader->line_peeked = 1; + return reader->status; +} @@ -20,10 +20,13 @@ * side can't, we stay with pure read/write interfaces. */ void packet_flush(int fd); +void packet_delim(int fd); void packet_write_fmt(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); void packet_buf_flush(struct strbuf *buf); +void packet_buf_delim(struct strbuf *buf); void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int write_packetized_from_fd(int fd_in, int fd_out); @@ -66,6 +69,23 @@ int packet_read(int fd, char **src_buffer, size_t *src_len, char *buffer, unsigned size, int options); /* + * Read a packetized line into a buffer like the 'packet_read()' function but + * returns an 'enum packet_read_status' which indicates the status of the read. + * The number of bytes read will be assigined to *pktlen if the status of the + * read was 'PACKET_READ_NORMAL'. + */ +enum packet_read_status { + PACKET_READ_EOF, + PACKET_READ_NORMAL, + PACKET_READ_FLUSH, + PACKET_READ_DELIM, +}; +enum packet_read_status packet_read_with_status(int fd, char **src_buffer, + size_t *src_len, char *buffer, + unsigned size, int *pktlen, + int options); + +/* * Convenience wrapper for packet_read that is not gentle, and sets the * CHOMP_NEWLINE option. The return value is NULL for a flush packet, * and otherwise points to a static buffer (that may be overwritten by @@ -96,6 +116,64 @@ char *packet_read_line_buf(char **src_buf, size_t *src_len, int *size); */ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out); +struct packet_reader { + /* source file descriptor */ + int fd; + + /* source buffer and its size */ + char *src_buffer; + size_t src_len; + + /* buffer that pkt-lines are read into and its size */ + char *buffer; + unsigned buffer_size; + + /* options to be used during reads */ + int options; + + /* status of the last read */ + enum packet_read_status status; + + /* length of data read during the last read */ + int pktlen; + + /* the last line read */ + const char *line; + + /* indicates if a line has been peeked */ + int line_peeked; +}; + +/* + * Initialize a 'struct packet_reader' object which is an + * abstraction around the 'packet_read_with_status()' function. + */ +extern void packet_reader_init(struct packet_reader *reader, int fd, + char *src_buffer, size_t src_len, + int options); + +/* + * Perform a packet read and return the status of the read. + * The values of 'pktlen' and 'line' are updated based on the status of the + * read as follows: + * + * PACKET_READ_ERROR: 'pktlen' is set to '-1' and 'line' is set to NULL + * PACKET_READ_NORMAL: 'pktlen' is set to the number of bytes read + * 'line' is set to point at the read line + * PACKET_READ_FLUSH: 'pktlen' is set to '0' and 'line' is set to NULL + */ +extern enum packet_read_status packet_reader_read(struct packet_reader *reader); + +/* + * Peek the next packet line without consuming it and return the status. + * The next call to 'packet_reader_read()' will perform a read of the same line + * that was peeked, consuming the line. + * + * Peeking multiple times without calling 'packet_reader_read()' will return + * the same result. + */ +extern enum packet_read_status packet_reader_peek(struct packet_reader *reader); + #define DEFAULT_PACKET_MAX 1000 #define LARGE_PACKET_MAX 65520 #define LARGE_PACKET_DATA_MAX (LARGE_PACKET_MAX - 4) @@ -8,6 +8,8 @@ static enum protocol_version parse_protocol_version(const char *value) return protocol_v0; else if (!strcmp(value, "1")) return protocol_v1; + else if (!strcmp(value, "2")) + return protocol_v2; else return protocol_unknown_version; } @@ -5,6 +5,7 @@ enum protocol_version { protocol_unknown_version = -1, protocol_v0 = 0, protocol_v1 = 1, + protocol_v2 = 2, }; /* @@ -13,6 +13,7 @@ #include "tag.h" #include "submodule.h" #include "worktree.h" +#include "argv-array.h" /* * List of all available backends @@ -502,6 +503,19 @@ int refname_match(const char *abbrev_name, const char *full_name) } /* + * Given a 'prefix' expand it by the rules in 'ref_rev_parse_rules' and add + * the results to 'prefixes' + */ +void expand_ref_prefix(struct argv_array *prefixes, const char *prefix) +{ + const char **p; + int len = strlen(prefix); + + for (p = ref_rev_parse_rules; *p; p++) + argv_array_pushf(prefixes, *p, len, prefix); +} + +/* * *string and *len will only be substituted, and *string returned (for * later free()ing) if the string passed in is a magic short-hand form * to name a branch. @@ -139,6 +139,13 @@ int resolve_gitlink_ref(const char *submodule, const char *refname, */ int refname_match(const char *abbrev_name, const char *full_name); +/* + * Given a 'prefix' expand it by the rules in 'ref_rev_parse_rules' and add + * the results to 'prefixes' + */ +struct argv_array; +void expand_ref_prefix(struct argv_array *prefixes, const char *prefix); + int expand_ref(const char *str, int len, struct object_id *oid, char **ref); int dwim_ref(const char *str, int len, struct object_id *oid, char **ref); int dwim_log(const char *str, int len, struct object_id *oid, char **ref); diff --git a/remote-curl.c b/remote-curl.c index 8d2ffaf..68e7b1c 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1,6 +1,7 @@ #include "cache.h" #include "config.h" #include "remote.h" +#include "connect.h" #include "strbuf.h" #include "walker.h" #include "http.h" @@ -13,6 +14,7 @@ #include "credential.h" #include "sha1-array.h" #include "send-pack.h" +#include "protocol.h" #include "quote.h" static struct remote *remote; @@ -184,12 +186,13 @@ static int set_option(const char *name, const char *value) } struct discovery { - const char *service; + char *service; char *buf_alloc; char *buf; size_t len; struct ref *refs; struct oid_array shallow; + enum protocol_version version; unsigned proto_git : 1; }; static struct discovery *last_discovery; @@ -197,8 +200,31 @@ static struct discovery *last_discovery; static struct ref *parse_git_refs(struct discovery *heads, int for_push) { struct ref *list = NULL; - get_remote_heads(-1, heads->buf, heads->len, &list, - for_push ? REF_NORMAL : 0, NULL, &heads->shallow); + struct packet_reader reader; + + packet_reader_init(&reader, -1, heads->buf, heads->len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + heads->version = discover_version(&reader); + switch (heads->version) { + case protocol_v2: + /* + * Do nothing. This isn't a list of refs but rather a + * capability advertisement. Client would have run + * 'stateless-connect' so we'll dump this capability listing + * and let them request the refs themselves. + */ + break; + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, + NULL, &heads->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + return list; } @@ -259,6 +285,7 @@ static void free_discovery(struct discovery *d) free(d->shallow.oid); free(d->buf_alloc); free_refs(d->refs); + free(d->service); free(d); } } @@ -290,6 +317,19 @@ static int show_http_message(struct strbuf *type, struct strbuf *charset, return 0; } +static int get_protocol_http_header(enum protocol_version version, + struct strbuf *header) +{ + if (version > 0) { + strbuf_addf(header, GIT_PROTOCOL_HEADER ": version=%d", + version); + + return 1; + } + + return 0; +} + static struct discovery *discover_refs(const char *service, int for_push) { struct strbuf exp = STRBUF_INIT; @@ -298,9 +338,12 @@ static struct discovery *discover_refs(const char *service, int for_push) struct strbuf buffer = STRBUF_INIT; struct strbuf refs_url = STRBUF_INIT; struct strbuf effective_url = STRBUF_INIT; + struct strbuf protocol_header = STRBUF_INIT; + struct string_list extra_headers = STRING_LIST_INIT_DUP; struct discovery *last = last_discovery; int http_ret, maybe_smart = 0; struct http_get_options http_options; + enum protocol_version version = get_protocol_version_config(); if (last && !strcmp(service, last->service)) return last; @@ -317,11 +360,24 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_addf(&refs_url, "service=%s", service); } + /* + * NEEDSWORK: If we are trying to use protocol v2 and we are planning + * to perform a push, then fallback to v0 since the client doesn't know + * how to push yet using v2. + */ + if (version == protocol_v2 && !strcmp("git-receive-pack", service)) + version = protocol_v0; + + /* Add the extra Git-Protocol header */ + if (get_protocol_http_header(version, &protocol_header)) + string_list_append(&extra_headers, protocol_header.buf); + memset(&http_options, 0, sizeof(http_options)); http_options.content_type = &type; http_options.charset = &charset; http_options.effective_url = &effective_url; http_options.base_url = &url; + http_options.extra_headers = &extra_headers; http_options.initial_request = 1; http_options.no_cache = 1; http_options.keep_error = 1; @@ -345,7 +401,7 @@ static struct discovery *discover_refs(const char *service, int for_push) warning(_("redirecting to %s"), url.buf); last= xcalloc(1, sizeof(*last_discovery)); - last->service = service; + last->service = xstrdup(service); last->buf_alloc = strbuf_detach(&buffer, &last->len); last->buf = last->buf_alloc; @@ -377,6 +433,9 @@ static struct discovery *discover_refs(const char *service, int for_push) ; last->proto_git = 1; + } else if (maybe_smart && + last->len > 5 && starts_with(last->buf + 4, "version 2")) { + last->proto_git = 1; } if (last->proto_git) @@ -390,6 +449,8 @@ static struct discovery *discover_refs(const char *service, int for_push) strbuf_release(&charset); strbuf_release(&effective_url); strbuf_release(&buffer); + strbuf_release(&protocol_header); + string_list_clear(&extra_headers, 0); last_discovery = last; return last; } @@ -426,6 +487,7 @@ struct rpc_state { char *service_url; char *hdr_content_type; char *hdr_accept; + char *protocol_header; char *buf; size_t alloc; size_t len; @@ -612,6 +674,10 @@ static int post_rpc(struct rpc_state *rpc) headers = curl_slist_append(headers, needs_100_continue ? "Expect: 100-continue" : "Expect:"); + /* Add the extra Git-Protocol header */ + if (rpc->protocol_header) + headers = curl_slist_append(headers, rpc->protocol_header); + retry: slot = get_active_slot(); @@ -752,6 +818,11 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) strbuf_addf(&buf, "Accept: application/x-%s-result", svc); rpc->hdr_accept = strbuf_detach(&buf, NULL); + if (get_protocol_http_header(heads->version, &buf)) + rpc->protocol_header = strbuf_detach(&buf, NULL); + else + rpc->protocol_header = NULL; + while (!err) { int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0); if (!n) @@ -779,6 +850,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) free(rpc->service_url); free(rpc->hdr_content_type); free(rpc->hdr_accept); + free(rpc->protocol_header); free(rpc->buf); strbuf_release(&buf); return err; @@ -1056,6 +1128,202 @@ static void parse_push(struct strbuf *buf) free(specs); } +/* + * Used to represent the state of a connection to an HTTP server when + * communicating using git's wire-protocol version 2. + */ +struct proxy_state { + char *service_name; + char *service_url; + struct curl_slist *headers; + struct strbuf request_buffer; + int in; + int out; + struct packet_reader reader; + size_t pos; + int seen_flush; +}; + +static void proxy_state_init(struct proxy_state *p, const char *service_name, + enum protocol_version version) +{ + struct strbuf buf = STRBUF_INIT; + + memset(p, 0, sizeof(*p)); + p->service_name = xstrdup(service_name); + + p->in = 0; + p->out = 1; + strbuf_init(&p->request_buffer, 0); + + strbuf_addf(&buf, "%s%s", url.buf, p->service_name); + p->service_url = strbuf_detach(&buf, NULL); + + p->headers = http_copy_default_headers(); + + strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name); + p->headers = curl_slist_append(p->headers, buf.buf); + strbuf_reset(&buf); + + strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name); + p->headers = curl_slist_append(p->headers, buf.buf); + strbuf_reset(&buf); + + p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked"); + + /* Add the Git-Protocol header */ + if (get_protocol_http_header(version, &buf)) + p->headers = curl_slist_append(p->headers, buf.buf); + + packet_reader_init(&p->reader, p->in, NULL, 0, + PACKET_READ_GENTLE_ON_EOF); + + strbuf_release(&buf); +} + +static void proxy_state_clear(struct proxy_state *p) +{ + free(p->service_name); + free(p->service_url); + curl_slist_free_all(p->headers); + strbuf_release(&p->request_buffer); +} + +/* + * CURLOPT_READFUNCTION callback function. + * Attempts to copy over a single packet-line at a time into the + * curl provided buffer. + */ +static size_t proxy_in(char *buffer, size_t eltsize, + size_t nmemb, void *userdata) +{ + size_t max; + struct proxy_state *p = userdata; + size_t avail = p->request_buffer.len - p->pos; + + + if (eltsize != 1) + BUG("curl read callback called with size = %"PRIuMAX" != 1", + (uintmax_t)eltsize); + max = nmemb; + + if (!avail) { + if (p->seen_flush) { + p->seen_flush = 0; + return 0; + } + + strbuf_reset(&p->request_buffer); + switch (packet_reader_read(&p->reader)) { + case PACKET_READ_EOF: + die("unexpected EOF when reading from parent process"); + case PACKET_READ_NORMAL: + packet_buf_write_len(&p->request_buffer, p->reader.line, + p->reader.pktlen); + break; + case PACKET_READ_DELIM: + packet_buf_delim(&p->request_buffer); + break; + case PACKET_READ_FLUSH: + packet_buf_flush(&p->request_buffer); + p->seen_flush = 1; + break; + } + p->pos = 0; + avail = p->request_buffer.len; + } + + if (max < avail) + avail = max; + memcpy(buffer, p->request_buffer.buf + p->pos, avail); + p->pos += avail; + return avail; +} + +static size_t proxy_out(char *buffer, size_t eltsize, + size_t nmemb, void *userdata) +{ + size_t size; + struct proxy_state *p = userdata; + + if (eltsize != 1) + BUG("curl read callback called with size = %"PRIuMAX" != 1", + (uintmax_t)eltsize); + size = nmemb; + + write_or_die(p->out, buffer, size); + return size; +} + +/* Issues a request to the HTTP server configured in `p` */ +static int proxy_request(struct proxy_state *p) +{ + struct active_request_slot *slot; + + slot = get_active_slot(); + + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(slot->curl, CURLOPT_POST, 1); + curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers); + + /* Setup function to read request from client */ + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in); + curl_easy_setopt(slot->curl, CURLOPT_READDATA, p); + + /* Setup function to write server response to client */ + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out); + curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p); + + if (run_slot(slot, NULL) != HTTP_OK) + return -1; + + return 0; +} + +static int stateless_connect(const char *service_name) +{ + struct discovery *discover; + struct proxy_state p; + + /* + * Run the info/refs request and see if the server supports protocol + * v2. If and only if the server supports v2 can we successfully + * establish a stateless connection, otherwise we need to tell the + * client to fallback to using other transport helper functions to + * complete their request. + */ + discover = discover_refs(service_name, 0); + if (discover->version != protocol_v2) { + printf("fallback\n"); + fflush(stdout); + return -1; + } else { + /* Stateless Connection established */ + printf("\n"); + fflush(stdout); + } + + proxy_state_init(&p, service_name, discover->version); + + /* + * Dump the capability listing that we got from the server earlier + * during the info/refs request. + */ + write_or_die(p.out, discover->buf, discover->len); + + /* Peek the next packet line. Until we see EOF keep sending POSTs */ + while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) { + if (proxy_request(&p)) { + /* We would have an err here */ + break; + } + } + + proxy_state_clear(&p); + return 0; +} + int cmd_main(int argc, const char **argv) { struct strbuf buf = STRBUF_INIT; @@ -1124,12 +1392,16 @@ int cmd_main(int argc, const char **argv) fflush(stdout); } else if (!strcmp(buf.buf, "capabilities")) { + printf("stateless-connect\n"); printf("fetch\n"); printf("option\n"); printf("push\n"); printf("check-connectivity\n"); printf("\n"); fflush(stdout); + } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) { + if (!stateless_connect(arg)) + break; } else { error("remote-curl: unknown command '%s' from git", buf.buf); return 1; @@ -151,10 +151,17 @@ int check_ref_type(const struct ref *ref, int flags); void free_refs(struct ref *ref); struct oid_array; -extern struct ref **get_remote_heads(int in, char *src_buf, size_t src_len, +struct packet_reader; +struct argv_array; +extern struct ref **get_remote_heads(struct packet_reader *reader, struct ref **list, unsigned int flags, struct oid_array *extra_have, - struct oid_array *shallow); + struct oid_array *shallow_points); + +/* Used for protocol v2 in order to retrieve refs from a remote */ +extern struct ref **get_remote_refs(int fd_out, struct packet_reader *reader, + struct ref **list, int for_push, + const struct argv_array *ref_prefixes); int resolve_remote_symref(struct ref *ref, struct ref *list); int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid); @@ -0,0 +1,257 @@ +#include "cache.h" +#include "repository.h" +#include "config.h" +#include "pkt-line.h" +#include "version.h" +#include "argv-array.h" +#include "ls-refs.h" +#include "serve.h" +#include "upload-pack.h" + +static int always_advertise(struct repository *r, + struct strbuf *value) +{ + return 1; +} + +static int agent_advertise(struct repository *r, + struct strbuf *value) +{ + if (value) + strbuf_addstr(value, git_user_agent_sanitized()); + return 1; +} + +struct protocol_capability { + /* + * The name of the capability. The server uses this name when + * advertising this capability, and the client uses this name to + * specify this capability. + */ + const char *name; + + /* + * Function queried to see if a capability should be advertised. + * Optionally a value can be specified by adding it to 'value'. + * If a value is added to 'value', the server will advertise this + * capability as "<name>=<value>" instead of "<name>". + */ + int (*advertise)(struct repository *r, struct strbuf *value); + + /* + * Function called when a client requests the capability as a command. + * The function will be provided the capabilities requested via 'keys' + * as well as a struct packet_reader 'request' which the command should + * use to read the command specific part of the request. Every command + * MUST read until a flush packet is seen before sending a response. + * + * This field should be NULL for capabilities which are not commands. + */ + int (*command)(struct repository *r, + struct argv_array *keys, + struct packet_reader *request); +}; + +static struct protocol_capability capabilities[] = { + { "agent", agent_advertise, NULL }, + { "ls-refs", always_advertise, ls_refs }, + { "fetch", upload_pack_advertise, upload_pack_v2 }, +}; + +static void advertise_capabilities(void) +{ + struct strbuf capability = STRBUF_INIT; + struct strbuf value = STRBUF_INIT; + int i; + + for (i = 0; i < ARRAY_SIZE(capabilities); i++) { + struct protocol_capability *c = &capabilities[i]; + + if (c->advertise(the_repository, &value)) { + strbuf_addstr(&capability, c->name); + + if (value.len) { + strbuf_addch(&capability, '='); + strbuf_addbuf(&capability, &value); + } + + strbuf_addch(&capability, '\n'); + packet_write(1, capability.buf, capability.len); + } + + strbuf_reset(&capability); + strbuf_reset(&value); + } + + packet_flush(1); + strbuf_release(&capability); + strbuf_release(&value); +} + +static struct protocol_capability *get_capability(const char *key) +{ + int i; + + if (!key) + return NULL; + + for (i = 0; i < ARRAY_SIZE(capabilities); i++) { + struct protocol_capability *c = &capabilities[i]; + const char *out; + if (skip_prefix(key, c->name, &out) && (!*out || *out == '=')) + return c; + } + + return NULL; +} + +static int is_valid_capability(const char *key) +{ + const struct protocol_capability *c = get_capability(key); + + return c && c->advertise(the_repository, NULL); +} + +static int is_command(const char *key, struct protocol_capability **command) +{ + const char *out; + + if (skip_prefix(key, "command=", &out)) { + struct protocol_capability *cmd = get_capability(out); + + if (*command) + die("command '%s' requested after already requesting command '%s'", + out, (*command)->name); + if (!cmd || !cmd->advertise(the_repository, NULL) || !cmd->command) + die("invalid command '%s'", out); + + *command = cmd; + return 1; + } + + return 0; +} + +int has_capability(const struct argv_array *keys, const char *capability, + const char **value) +{ + int i; + for (i = 0; i < keys->argc; i++) { + const char *out; + if (skip_prefix(keys->argv[i], capability, &out) && + (!*out || *out == '=')) { + if (value) { + if (*out == '=') + out++; + *value = out; + } + return 1; + } + } + + return 0; +} + +enum request_state { + PROCESS_REQUEST_KEYS, + PROCESS_REQUEST_DONE, +}; + +static int process_request(void) +{ + enum request_state state = PROCESS_REQUEST_KEYS; + struct packet_reader reader; + struct argv_array keys = ARGV_ARRAY_INIT; + struct protocol_capability *command = NULL; + + packet_reader_init(&reader, 0, NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + /* + * Check to see if the client closed their end before sending another + * request. If so we can terminate the connection. + */ + if (packet_reader_peek(&reader) == PACKET_READ_EOF) + return 1; + reader.options = PACKET_READ_CHOMP_NEWLINE; + + while (state != PROCESS_REQUEST_DONE) { + switch (packet_reader_peek(&reader)) { + case PACKET_READ_EOF: + BUG("Should have already died when seeing EOF"); + case PACKET_READ_NORMAL: + /* collect request; a sequence of keys and values */ + if (is_command(reader.line, &command) || + is_valid_capability(reader.line)) + argv_array_push(&keys, reader.line); + else + die("unknown capability '%s'", reader.line); + + /* Consume the peeked line */ + packet_reader_read(&reader); + break; + case PACKET_READ_FLUSH: + /* + * If no command and no keys were given then the client + * wanted to terminate the connection. + */ + if (!keys.argc) + return 1; + + /* + * The flush packet isn't consume here like it is in + * the other parts of this switch statement. This is + * so that the command can read the flush packet and + * see the end of the request in the same way it would + * if command specific arguments were provided after a + * delim packet. + */ + state = PROCESS_REQUEST_DONE; + break; + case PACKET_READ_DELIM: + /* Consume the peeked line */ + packet_reader_read(&reader); + + state = PROCESS_REQUEST_DONE; + break; + } + } + + if (!command) + die("no command requested"); + + command->command(the_repository, &keys, &reader); + + argv_array_clear(&keys); + return 0; +} + +/* Main serve loop for protocol version 2 */ +void serve(struct serve_options *options) +{ + if (options->advertise_capabilities || !options->stateless_rpc) { + /* serve by default supports v2 */ + packet_write_fmt(1, "version 2\n"); + + advertise_capabilities(); + /* + * If only the list of capabilities was requested exit + * immediately after advertising capabilities + */ + if (options->advertise_capabilities) + return; + } + + /* + * If stateless-rpc was requested then exit after + * a single request/response exchange + */ + if (options->stateless_rpc) { + process_request(); + } else { + for (;;) + if (process_request()) + break; + } +} @@ -0,0 +1,15 @@ +#ifndef SERVE_H +#define SERVE_H + +struct argv_array; +extern int has_capability(const struct argv_array *keys, const char *capability, + const char **value); + +struct serve_options { + unsigned advertise_capabilities; + unsigned stateless_rpc; +}; +#define SERVE_OPTIONS_INIT { 0 } +extern void serve(struct serve_options *options); + +#endif /* SERVE_H */ diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c new file mode 100644 index 0000000..0f19e53 --- /dev/null +++ b/t/helper/test-pkt-line.c @@ -0,0 +1,64 @@ +#include "pkt-line.h" + +static void pack_line(const char *line) +{ + if (!strcmp(line, "0000") || !strcmp(line, "0000\n")) + packet_flush(1); + else if (!strcmp(line, "0001") || !strcmp(line, "0001\n")) + packet_delim(1); + else + packet_write_fmt(1, "%s", line); +} + +static void pack(int argc, const char **argv) +{ + if (argc) { /* read from argv */ + int i; + for (i = 0; i < argc; i++) + pack_line(argv[i]); + } else { /* read from stdin */ + char line[LARGE_PACKET_MAX]; + while (fgets(line, sizeof(line), stdin)) { + pack_line(line); + } + } +} + +static void unpack(void) +{ + struct packet_reader reader; + packet_reader_init(&reader, 0, NULL, 0, + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_CHOMP_NEWLINE); + + while (packet_reader_read(&reader) != PACKET_READ_EOF) { + switch (reader.status) { + case PACKET_READ_EOF: + break; + case PACKET_READ_NORMAL: + printf("%s\n", reader.line); + break; + case PACKET_READ_FLUSH: + printf("0000\n"); + break; + case PACKET_READ_DELIM: + printf("0001\n"); + break; + } + } +} + +int cmd_main(int argc, const char **argv) +{ + if (argc < 2) + die("too few arguments"); + + if (!strcmp(argv[1], "pack")) + pack(argc - 2, argv + 2); + else if (!strcmp(argv[1], "unpack")) + unpack(); + else + die("invalid argument '%s'", argv[1]); + + return 0; +} diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh new file mode 100755 index 0000000..72d7bc5 --- /dev/null +++ b/t/t5701-git-serve.sh @@ -0,0 +1,176 @@ +#!/bin/sh + +test_description='test git-serve and server commands' + +. ./test-lib.sh + +test_expect_success 'test capability advertisement' ' + cat >expect <<-EOF && + version 2 + agent=git/$(git version | cut -d" " -f3) + ls-refs + fetch=shallow + 0000 + EOF + + git serve --advertise-capabilities >out && + test-pkt-line unpack <out >actual && + test_cmp actual expect +' + +test_expect_success 'stateless-rpc flag does not list capabilities' ' + # Empty request + test-pkt-line pack >in <<-EOF && + 0000 + EOF + git serve --stateless-rpc >out <in && + test_must_be_empty out && + + # EOF + git serve --stateless-rpc >out && + test_must_be_empty out +' + +test_expect_success 'request invalid capability' ' + test-pkt-line pack >in <<-EOF && + foobar + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err <in && + test_i18ngrep "unknown capability" err +' + +test_expect_success 'request with no command' ' + test-pkt-line pack >in <<-EOF && + agent=git/test + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err <in && + test_i18ngrep "no command requested" err +' + +test_expect_success 'request invalid command' ' + test-pkt-line pack >in <<-EOF && + command=foo + agent=git/test + 0000 + EOF + test_must_fail git serve --stateless-rpc 2>err <in && + test_i18ngrep "invalid command" err +' + +# Test the basics of ls-refs +# +test_expect_success 'setup some refs and tags' ' + test_commit one && + git branch dev master && + test_commit two && + git symbolic-ref refs/heads/release refs/heads/master && + git tag -a -m "annotated tag" annotated-tag +' + +test_expect_success 'basics of ls-refs' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse HEAD) HEAD + $(git rev-parse refs/heads/dev) refs/heads/dev + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/heads/release) refs/heads/release + $(git rev-parse refs/tags/annotated-tag) refs/tags/annotated-tag + $(git rev-parse refs/tags/one) refs/tags/one + $(git rev-parse refs/tags/two) refs/tags/two + 0000 + EOF + + git serve --stateless-rpc <in >out && + test-pkt-line unpack <out >actual && + test_cmp actual expect +' + +test_expect_success 'basic ref-prefixes' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + ref-prefix refs/heads/master + ref-prefix refs/tags/one + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/tags/one) refs/tags/one + 0000 + EOF + + git serve --stateless-rpc <in >out && + test-pkt-line unpack <out >actual && + test_cmp actual expect +' + +test_expect_success 'refs/heads prefix' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + ref-prefix refs/heads/ + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/heads/dev) refs/heads/dev + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/heads/release) refs/heads/release + 0000 + EOF + + git serve --stateless-rpc <in >out && + test-pkt-line unpack <out >actual && + test_cmp actual expect +' + +test_expect_success 'peel parameter' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + peel + ref-prefix refs/tags/ + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/tags/annotated-tag) refs/tags/annotated-tag peeled:$(git rev-parse refs/tags/annotated-tag^{}) + $(git rev-parse refs/tags/one) refs/tags/one + $(git rev-parse refs/tags/two) refs/tags/two + 0000 + EOF + + git serve --stateless-rpc <in >out && + test-pkt-line unpack <out >actual && + test_cmp actual expect +' + +test_expect_success 'symrefs parameter' ' + test-pkt-line pack >in <<-EOF && + command=ls-refs + 0001 + symrefs + ref-prefix refs/heads/ + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse refs/heads/dev) refs/heads/dev + $(git rev-parse refs/heads/master) refs/heads/master + $(git rev-parse refs/heads/release) refs/heads/release symref-target:refs/heads/master + 0000 + EOF + + git serve --stateless-rpc <in >out && + test-pkt-line unpack <out >actual && + test_cmp actual expect +' + +test_done diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh new file mode 100755 index 0000000..56f7c3c --- /dev/null +++ b/t/t5702-protocol-v2.sh @@ -0,0 +1,273 @@ +#!/bin/sh + +test_description='test git wire-protocol version 2' + +TEST_NO_CREATE_REPO=1 + +. ./test-lib.sh + +# Test protocol v2 with 'git://' transport +# +. "$TEST_DIRECTORY"/lib-git-daemon.sh +start_git_daemon --export-all --enable=receive-pack +daemon_parent=$GIT_DAEMON_DOCUMENT_ROOT_PATH/parent + +test_expect_success 'create repo to be served by git-daemon' ' + git init "$daemon_parent" && + test_commit -C "$daemon_parent" one +' + +test_expect_success 'list refs with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote --symref "$GIT_DAEMON_URL/parent" >actual && + + # Client requested to use protocol v2 + grep "git> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "git< version 2" log && + + git ls-remote --symref "$GIT_DAEMON_URL/parent" >expect && + test_cmp actual expect +' + +test_expect_success 'ref advertisment is filtered with ls-remote using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote "$GIT_DAEMON_URL/parent" master >actual && + + cat >expect <<-EOF && + $(git -C "$daemon_parent" rev-parse refs/heads/master)$(printf "\t")refs/heads/master + EOF + + test_cmp actual expect +' + +test_expect_success 'clone with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + clone "$GIT_DAEMON_URL/parent" daemon_child && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "clone> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "clone< version 2" log +' + +test_expect_success 'fetch with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C "$daemon_parent" two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + fetch && + + git -C daemon_child log -1 --format=%s origin/master >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "fetch> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'pull with git:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + pull && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "fetch> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'push with git:// and a config of v2 does not request v2' ' + test_when_finished "rm -f log" && + + # Till v2 for push is designed, make sure that if a client has + # protocol.version configured to use v2, that the client instead falls + # back and uses v0. + + test_commit -C daemon_child three && + + # Push to another branch, as the target repository has the + # master branch checked out and we cannot push into it. + GIT_TRACE_PACKET="$(pwd)/log" git -C daemon_child -c protocol.version=2 \ + push origin HEAD:client_branch && + + git -C daemon_child log -1 --format=%s >actual && + git -C "$daemon_parent" log -1 --format=%s client_branch >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + ! grep "push> .*\\\0\\\0version=2\\\0$" log && + # Server responded using protocol v2 + ! grep "push< version 2" log +' + +stop_git_daemon + +# Test protocol v2 with 'file://' transport +# +test_expect_success 'create repo to be served by file:// transport' ' + git init file_parent && + test_commit -C file_parent one +' + +test_expect_success 'list refs with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote --symref "file://$(pwd)/file_parent" >actual && + + # Server responded using protocol v2 + grep "git< version 2" log && + + git ls-remote --symref "file://$(pwd)/file_parent" >expect && + test_cmp actual expect +' + +test_expect_success 'ref advertisment is filtered with ls-remote using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + ls-remote "file://$(pwd)/file_parent" master >actual && + + cat >expect <<-EOF && + $(git -C file_parent rev-parse refs/heads/master)$(printf "\t")refs/heads/master + EOF + + test_cmp actual expect +' + +test_expect_success 'clone with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \ + clone "file://$(pwd)/file_parent" file_child && + + git -C file_child log -1 --format=%s >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "clone< version 2" log +' + +test_expect_success 'fetch with file:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C file_parent two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \ + fetch origin && + + git -C file_child log -1 --format=%s origin/master >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "fetch< version 2" log +' + +test_expect_success 'ref advertisment is filtered during fetch using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C file_parent three && + + GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \ + fetch origin master && + + git -C file_child log -1 --format=%s origin/master >actual && + git -C file_parent log -1 --format=%s >expect && + test_cmp expect actual && + + ! grep "refs/tags/one" log && + ! grep "refs/tags/two" log && + ! grep "refs/tags/three" log +' + +# Test protocol v2 with 'http://' transport +# +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'create repo to be served by http:// transport' ' + git init "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config http.receivepack true && + test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" one +' + +test_expect_success 'clone with http:// using protocol v2' ' + test_when_finished "rm -f log" && + + GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git -c protocol.version=2 \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + git -C http_child log -1 --format=%s >actual && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Client requested to use protocol v2 + grep "Git-Protocol: version=2" log && + # Server responded using protocol v2 + grep "git< version 2" log +' + +test_expect_success 'fetch with http:// using protocol v2' ' + test_when_finished "rm -f log" && + + test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" two && + + GIT_TRACE_PACKET="$(pwd)/log" git -C http_child -c protocol.version=2 \ + fetch && + + git -C http_child log -1 --format=%s origin/master >actual && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s >expect && + test_cmp expect actual && + + # Server responded using protocol v2 + grep "git< version 2" log +' + +test_expect_success 'push with http:// and a config of v2 does not request v2' ' + test_when_finished "rm -f log" && + # Till v2 for push is designed, make sure that if a client has + # protocol.version configured to use v2, that the client instead falls + # back and uses v0. + + test_commit -C http_child three && + + # Push to another branch, as the target repository has the + # master branch checked out and we cannot push into it. + GIT_TRACE_PACKET="$(pwd)/log" git -C http_child -c protocol.version=2 \ + push origin HEAD:client_branch && + + git -C http_child log -1 --format=%s >actual && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s client_branch >expect && + test_cmp expect actual && + + # Client didnt request to use protocol v2 + ! grep "Git-Protocol: version=2" log && + # Server didnt respond using protocol v2 + ! grep "git< version 2" log +' + + +stop_httpd + +test_done diff --git a/transport-helper.c b/transport-helper.c index 3f380d8..11f1055 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -12,6 +12,7 @@ #include "argv-array.h" #include "refs.h" #include "transport-internal.h" +#include "protocol.h" static int debug; @@ -26,6 +27,7 @@ struct helper_data { option : 1, push : 1, connect : 1, + stateless_connect : 1, signed_tags : 1, check_connectivity : 1, no_disconnect_req : 1, @@ -49,7 +51,7 @@ static void sendline(struct helper_data *helper, struct strbuf *buffer) die_errno("Full write to remote helper failed"); } -static int recvline_fh(FILE *helper, struct strbuf *buffer, const char *name) +static int recvline_fh(FILE *helper, struct strbuf *buffer) { strbuf_reset(buffer); if (debug) @@ -67,7 +69,7 @@ static int recvline_fh(FILE *helper, struct strbuf *buffer, const char *name) static int recvline(struct helper_data *helper, struct strbuf *buffer) { - return recvline_fh(helper->out, buffer, helper->name); + return recvline_fh(helper->out, buffer); } static void write_constant(int fd, const char *str) @@ -188,6 +190,8 @@ static struct child_process *get_helper(struct transport *transport) refspecs[refspec_nr++] = xstrdup(arg); } else if (!strcmp(capname, "connect")) { data->connect = 1; + } else if (!strcmp(capname, "stateless-connect")) { + data->stateless_connect = 1; } else if (!strcmp(capname, "signed-tags")) { data->signed_tags = 1; } else if (skip_prefix(capname, "export-marks ", &arg)) { @@ -545,14 +549,13 @@ static int fetch_with_import(struct transport *transport, return 0; } -static int process_connect_service(struct transport *transport, - const char *name, const char *exec) +static int run_connect(struct transport *transport, struct strbuf *cmdbuf) { struct helper_data *data = transport->data; - struct strbuf cmdbuf = STRBUF_INIT; - struct child_process *helper; - int r, duped, ret = 0; + int ret = 0; + int duped; FILE *input; + struct child_process *helper; helper = get_helper(transport); @@ -568,44 +571,61 @@ static int process_connect_service(struct transport *transport, input = xfdopen(duped, "r"); setvbuf(input, NULL, _IONBF, 0); + sendline(data, cmdbuf); + if (recvline_fh(input, cmdbuf)) + exit(128); + + if (!strcmp(cmdbuf->buf, "")) { + data->no_disconnect_req = 1; + if (debug) + fprintf(stderr, "Debug: Smart transport connection " + "ready.\n"); + ret = 1; + } else if (!strcmp(cmdbuf->buf, "fallback")) { + if (debug) + fprintf(stderr, "Debug: Falling back to dumb " + "transport.\n"); + } else { + die("Unknown response to connect: %s", + cmdbuf->buf); + } + + fclose(input); + return ret; +} + +static int process_connect_service(struct transport *transport, + const char *name, const char *exec) +{ + struct helper_data *data = transport->data; + struct strbuf cmdbuf = STRBUF_INIT; + int ret = 0; + /* * Handle --upload-pack and friends. This is fire and forget... * just warn if it fails. */ if (strcmp(name, exec)) { - r = set_helper_option(transport, "servpath", exec); + int r = set_helper_option(transport, "servpath", exec); if (r > 0) warning("Setting remote service path not supported by protocol."); else if (r < 0) warning("Invalid remote service path."); } - if (data->connect) + if (data->connect) { strbuf_addf(&cmdbuf, "connect %s\n", name); - else - goto exit; - - sendline(data, &cmdbuf); - if (recvline_fh(input, &cmdbuf, name)) - exit(128); - - if (!strcmp(cmdbuf.buf, "")) { - data->no_disconnect_req = 1; - if (debug) - fprintf(stderr, "Debug: Smart transport connection " - "ready.\n"); - ret = 1; - } else if (!strcmp(cmdbuf.buf, "fallback")) { - if (debug) - fprintf(stderr, "Debug: Falling back to dumb " - "transport.\n"); - } else - die("Unknown response to connect: %s", - cmdbuf.buf); + ret = run_connect(transport, &cmdbuf); + } else if (data->stateless_connect && + (get_protocol_version_config() == protocol_v2) && + !strcmp("git-upload-pack", name)) { + strbuf_addf(&cmdbuf, "stateless-connect %s\n", name); + ret = run_connect(transport, &cmdbuf); + if (ret) + transport->stateless_rpc = 1; + } -exit: strbuf_release(&cmdbuf); - fclose(input); return ret; } @@ -1031,7 +1051,8 @@ static int has_attribute(const char *attrs, const char *attr) { } } -static struct ref *get_refs_list(struct transport *transport, int for_push) +static struct ref *get_refs_list(struct transport *transport, int for_push, + const struct argv_array *ref_prefixes) { struct helper_data *data = transport->data; struct child_process *helper; @@ -1044,7 +1065,7 @@ static struct ref *get_refs_list(struct transport *transport, int for_push) if (process_connect(transport, for_push)) { do_take_over(transport); - return transport->vtable->get_refs_list(transport, for_push); + return transport->vtable->get_refs_list(transport, for_push, ref_prefixes); } if (data->push && for_push) diff --git a/transport-internal.h b/transport-internal.h index 3c1a29d..1cde625 100644 --- a/transport-internal.h +++ b/transport-internal.h @@ -3,6 +3,7 @@ struct ref; struct transport; +struct argv_array; struct transport_vtable { /** @@ -17,11 +18,19 @@ struct transport_vtable { * the transport to try to share connections, for_push is a * hint as to whether the ultimate operation is a push or a fetch. * + * If communicating using protocol v2 a list of prefixes can be + * provided to be sent to the server to enable it to limit the ref + * advertisement. Since ref filtering is done on the server's end, and + * only when using protocol v2, this list will be ignored when not + * using protocol v2 meaning this function can return refs which don't + * match the provided ref_prefixes. + * * If the transport is able to determine the remote hash for * the ref without a huge amount of effort, it should store it * in the ref's old_sha1 field; otherwise it should be all 0. **/ - struct ref *(*get_refs_list)(struct transport *transport, int for_push); + struct ref *(*get_refs_list)(struct transport *transport, int for_push, + const struct argv_array *ref_prefixes); /** * Fetch the objects for the given refs. Note that this gets diff --git a/transport.c b/transport.c index 94eccf2..16b2f54 100644 --- a/transport.c +++ b/transport.c @@ -18,6 +18,7 @@ #include "sha1-array.h" #include "sigchain.h" #include "transport-internal.h" +#include "protocol.h" #include "object-store.h" static void set_upstreams(struct transport *transport, struct ref *refs, @@ -72,7 +73,9 @@ struct bundle_transport_data { struct bundle_header header; }; -static struct ref *get_refs_from_bundle(struct transport *transport, int for_push) +static struct ref *get_refs_from_bundle(struct transport *transport, + int for_push, + const struct argv_array *ref_prefixes) { struct bundle_transport_data *data = transport->data; struct ref *result = NULL; @@ -118,6 +121,7 @@ struct git_transport_data { struct child_process *conn; int fd[2]; unsigned got_remote_heads : 1; + enum protocol_version version; struct oid_array extra_have; struct oid_array shallow; }; @@ -197,16 +201,35 @@ static int connect_setup(struct transport *transport, int for_push) return 0; } -static struct ref *get_refs_via_connect(struct transport *transport, int for_push) +static struct ref *get_refs_via_connect(struct transport *transport, int for_push, + const struct argv_array *ref_prefixes) { struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; + struct packet_reader reader; connect_setup(transport, for_push); - get_remote_heads(data->fd[0], NULL, 0, &refs, - for_push ? REF_NORMAL : 0, - &data->extra_have, - &data->shallow); + + packet_reader_init(&reader, data->fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + + data->version = discover_version(&reader); + switch (data->version) { + case protocol_v2: + get_remote_refs(data->fd[1], &reader, &refs, for_push, + ref_prefixes); + break; + case protocol_v1: + case protocol_v0: + get_remote_heads(&reader, &refs, + for_push ? REF_NORMAL : 0, + &data->extra_have, + &data->shallow); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } data->got_remote_heads = 1; return refs; @@ -217,7 +240,7 @@ static int fetch_refs_via_pack(struct transport *transport, { int ret = 0; struct git_transport_data *data = transport->data; - struct ref *refs; + struct ref *refs = NULL; char *dest = xstrdup(transport->url); struct fetch_pack_args args; struct ref *refs_tmp = NULL; @@ -242,18 +265,29 @@ static int fetch_refs_via_pack(struct transport *transport, args.from_promisor = data->options.from_promisor; args.no_dependents = data->options.no_dependents; args.filter_options = data->options.filter_options; + args.stateless_rpc = transport->stateless_rpc; - if (!data->got_remote_heads) { - connect_setup(transport, 0); - get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0, - NULL, &data->shallow); - data->got_remote_heads = 1; + if (!data->got_remote_heads) + refs_tmp = get_refs_via_connect(transport, 0, NULL); + + switch (data->version) { + case protocol_v2: + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile, data->version); + break; + case protocol_v1: + case protocol_v0: + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile, data->version); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); } - refs = fetch_pack(&args, data->fd, data->conn, - refs_tmp ? refs_tmp : transport->remote_refs, - dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile); close(data->fd[0]); close(data->fd[1]); if (finish_connect(data->conn)) @@ -552,16 +586,10 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re { struct git_transport_data *data = transport->data; struct send_pack_args args; - int ret; - - if (!data->got_remote_heads) { - struct ref *tmp_refs; - connect_setup(transport, 1); + int ret = 0; - get_remote_heads(data->fd[0], NULL, 0, &tmp_refs, REF_NORMAL, - NULL, &data->shallow); - data->got_remote_heads = 1; - } + if (!data->got_remote_heads) + get_refs_via_connect(transport, 1, NULL); memset(&args, 0, sizeof(args)); args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR); @@ -583,8 +611,18 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re else args.push_cert = SEND_PACK_PUSH_CERT_NEVER; - ret = send_pack(&args, data->fd, data->conn, remote_refs, - &data->extra_have); + switch (data->version) { + case protocol_v2: + die("support for protocol v2 not implemented yet"); + break; + case protocol_v1: + case protocol_v0: + ret = send_pack(&args, data->fd, data->conn, remote_refs, + &data->extra_have); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } close(data->fd[1]); close(data->fd[0]); @@ -1007,11 +1045,38 @@ int transport_push(struct transport *transport, int porcelain = flags & TRANSPORT_PUSH_PORCELAIN; int pretend = flags & TRANSPORT_PUSH_DRY_RUN; int push_ret, ret, err; + struct refspec *tmp_rs; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; + int i; if (check_push_refs(local_refs, refspec_nr, refspec) < 0) return -1; - remote_refs = transport->vtable->get_refs_list(transport, 1); + tmp_rs = parse_push_refspec(refspec_nr, refspec); + for (i = 0; i < refspec_nr; i++) { + const char *prefix = NULL; + + if (tmp_rs[i].dst) + prefix = tmp_rs[i].dst; + else if (tmp_rs[i].src && !tmp_rs[i].exact_sha1) + prefix = tmp_rs[i].src; + + if (prefix) { + const char *glob = strchr(prefix, '*'); + if (glob) + argv_array_pushf(&ref_prefixes, "%.*s", + (int)(glob - prefix), + prefix); + else + expand_ref_prefix(&ref_prefixes, prefix); + } + } + + remote_refs = transport->vtable->get_refs_list(transport, 1, + &ref_prefixes); + + argv_array_clear(&ref_prefixes); + free_refspec(refspec_nr, tmp_rs); if (flags & TRANSPORT_PUSH_ALL) match_flags |= MATCH_REFS_ALL; @@ -1117,10 +1182,13 @@ int transport_push(struct transport *transport, return 1; } -const struct ref *transport_get_remote_refs(struct transport *transport) +const struct ref *transport_get_remote_refs(struct transport *transport, + const struct argv_array *ref_prefixes) { if (!transport->got_remote_refs) { - transport->remote_refs = transport->vtable->get_refs_list(transport, 0); + transport->remote_refs = + transport->vtable->get_refs_list(transport, 0, + ref_prefixes); transport->got_remote_refs = 1; } diff --git a/transport.h b/transport.h index 3c68d73..e783cfa 100644 --- a/transport.h +++ b/transport.h @@ -60,6 +60,12 @@ struct transport { unsigned cloning : 1; /* + * Indicates that the transport is connected via a half-duplex + * connection and should operate in stateless-rpc mode. + */ + unsigned stateless_rpc : 1; + + /* * These strings will be passed to the {pre, post}-receive hook, * on the remote side, if both sides support the push options capability. */ @@ -194,7 +200,17 @@ int transport_push(struct transport *connection, int refspec_nr, const char **refspec, int flags, unsigned int * reject_reasons); -const struct ref *transport_get_remote_refs(struct transport *transport); +/* + * Retrieve refs from a remote. + * + * Optionally a list of ref prefixes can be provided which can be sent to the + * server (when communicating using protocol v2) to enable it to limit the ref + * advertisement. Since ref filtering is done on the server's end (and only + * when using protocol v2), this can return refs which don't match the provided + * ref_prefixes. + */ +const struct ref *transport_get_remote_refs(struct transport *transport, + const struct argv_array *ref_prefixes); int transport_fetch_refs(struct transport *transport, struct ref *refs); void transport_unlock_pack(struct transport *transport); diff --git a/upload-pack.c b/upload-pack.c index 6261d4f..87b4d32 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -6,7 +6,6 @@ #include "tag.h" #include "object.h" #include "commit.h" -#include "exec-cmd.h" #include "diff.h" #include "revision.h" #include "list-objects.h" @@ -17,16 +16,12 @@ #include "sigchain.h" #include "version.h" #include "string-list.h" -#include "parse-options.h" #include "argv-array.h" #include "prio-queue.h" #include "protocol.h" #include "quote.h" - -static const char * const upload_pack_usage[] = { - N_("git upload-pack [<options>] <dir>"), - NULL -}; +#include "upload-pack.h" +#include "serve.h" /* Remember to update object flag allocation in object.h */ #define THEY_HAVE (1u << 11) @@ -64,7 +59,6 @@ static int keepalive = 5; * otherwise maximum packet size (up to 65520 bytes). */ static int use_sideband; -static int advertise_refs; static int stateless_rpc; static const char *pack_objects_hook; @@ -734,7 +728,6 @@ static void deepen(int depth, int deepen_relative, } send_unshallow(shallows); - packet_flush(1); } static void deepen_by_rev_list(int ac, const char **av, @@ -746,7 +739,122 @@ static void deepen_by_rev_list(int ac, const char **av, send_shallow(result); free_commit_list(result); send_unshallow(shallows); - packet_flush(1); +} + +/* Returns 1 if a shallow list is sent or 0 otherwise */ +static int send_shallow_list(int depth, int deepen_rev_list, + timestamp_t deepen_since, + struct string_list *deepen_not, + struct object_array *shallows) +{ + int ret = 0; + + if (depth > 0 && deepen_rev_list) + die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); + if (depth > 0) { + deepen(depth, deepen_relative, shallows); + ret = 1; + } else if (deepen_rev_list) { + struct argv_array av = ARGV_ARRAY_INIT; + int i; + + argv_array_push(&av, "rev-list"); + if (deepen_since) + argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since); + if (deepen_not->nr) { + argv_array_push(&av, "--not"); + for (i = 0; i < deepen_not->nr; i++) { + struct string_list_item *s = deepen_not->items + i; + argv_array_push(&av, s->string); + } + argv_array_push(&av, "--not"); + } + for (i = 0; i < want_obj.nr; i++) { + struct object *o = want_obj.objects[i].item; + argv_array_push(&av, oid_to_hex(&o->oid)); + } + deepen_by_rev_list(av.argc, av.argv, shallows); + argv_array_clear(&av); + ret = 1; + } else { + if (shallows->nr > 0) { + int i; + for (i = 0; i < shallows->nr; i++) + register_shallow(&shallows->objects[i].item->oid); + } + } + + shallow_nr += shallows->nr; + return ret; +} + +static int process_shallow(const char *line, struct object_array *shallows) +{ + const char *arg; + if (skip_prefix(line, "shallow ", &arg)) { + struct object_id oid; + struct object *object; + if (get_oid_hex(arg, &oid)) + die("invalid shallow line: %s", line); + object = parse_object(&oid); + if (!object) + return 1; + if (object->type != OBJ_COMMIT) + die("invalid shallow object %s", oid_to_hex(&oid)); + if (!(object->flags & CLIENT_SHALLOW)) { + object->flags |= CLIENT_SHALLOW; + add_object_array(object, NULL, shallows); + } + return 1; + } + + return 0; +} + +static int process_deepen(const char *line, int *depth) +{ + const char *arg; + if (skip_prefix(line, "deepen ", &arg)) { + char *end = NULL; + *depth = (int)strtol(arg, &end, 0); + if (!end || *end || *depth <= 0) + die("Invalid deepen: %s", line); + return 1; + } + + return 0; +} + +static int process_deepen_since(const char *line, timestamp_t *deepen_since, int *deepen_rev_list) +{ + const char *arg; + if (skip_prefix(line, "deepen-since ", &arg)) { + char *end = NULL; + *deepen_since = parse_timestamp(arg, &end, 0); + if (!end || *end || !deepen_since || + /* revisions.c's max_age -1 is special */ + *deepen_since == -1) + die("Invalid deepen-since: %s", line); + *deepen_rev_list = 1; + return 1; + } + return 0; +} + +static int process_deepen_not(const char *line, struct string_list *deepen_not, int *deepen_rev_list) +{ + const char *arg; + if (skip_prefix(line, "deepen-not ", &arg)) { + char *ref = NULL; + struct object_id oid; + if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) + die("git upload-pack: ambiguous deepen-not: %s", line); + string_list_append(deepen_not, ref); + free(ref); + *deepen_rev_list = 1; + return 1; + } + return 0; } static void receive_needs(void) @@ -770,55 +878,22 @@ static void receive_needs(void) if (!line) break; - if (skip_prefix(line, "shallow ", &arg)) { - struct object_id oid; - struct object *object; - if (get_oid_hex(arg, &oid)) - die("invalid shallow line: %s", line); - object = parse_object(&oid); - if (!object) - continue; - if (object->type != OBJ_COMMIT) - die("invalid shallow object %s", oid_to_hex(&oid)); - if (!(object->flags & CLIENT_SHALLOW)) { - object->flags |= CLIENT_SHALLOW; - add_object_array(object, NULL, &shallows); - } + if (process_shallow(line, &shallows)) continue; - } - if (skip_prefix(line, "deepen ", &arg)) { - char *end = NULL; - depth = strtol(arg, &end, 0); - if (!end || *end || depth <= 0) - die("Invalid deepen: %s", line); + if (process_deepen(line, &depth)) continue; - } - if (skip_prefix(line, "deepen-since ", &arg)) { - char *end = NULL; - deepen_since = parse_timestamp(arg, &end, 0); - if (!end || *end || !deepen_since || - /* revisions.c's max_age -1 is special */ - deepen_since == -1) - die("Invalid deepen-since: %s", line); - deepen_rev_list = 1; + if (process_deepen_since(line, &deepen_since, &deepen_rev_list)) continue; - } - if (skip_prefix(line, "deepen-not ", &arg)) { - char *ref = NULL; - struct object_id oid; - if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) - die("git upload-pack: ambiguous deepen-not: %s", line); - string_list_append(&deepen_not, ref); - free(ref); - deepen_rev_list = 1; + if (process_deepen_not(line, &deepen_not, &deepen_rev_list)) continue; - } + if (skip_prefix(line, "filter ", &arg)) { if (!filter_capability_requested) die("git upload-pack: filtering capability not negotiated"); parse_list_objects_filter(&filter_options, arg); continue; } + if (!skip_prefix(line, "want ", &arg) || get_oid_hex(arg, &oid_buf)) die("git upload-pack: protocol error, " @@ -881,40 +956,10 @@ static void receive_needs(void) if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; - if (depth > 0 && deepen_rev_list) - die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); - if (depth > 0) - deepen(depth, deepen_relative, &shallows); - else if (deepen_rev_list) { - struct argv_array av = ARGV_ARRAY_INIT; - int i; - argv_array_push(&av, "rev-list"); - if (deepen_since) - argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since); - if (deepen_not.nr) { - argv_array_push(&av, "--not"); - for (i = 0; i < deepen_not.nr; i++) { - struct string_list_item *s = deepen_not.items + i; - argv_array_push(&av, s->string); - } - argv_array_push(&av, "--not"); - } - for (i = 0; i < want_obj.nr; i++) { - struct object *o = want_obj.objects[i].item; - argv_array_push(&av, oid_to_hex(&o->oid)); - } - deepen_by_rev_list(av.argc, av.argv, &shallows); - argv_array_clear(&av); - } - else - if (shallows.nr > 0) { - int i; - for (i = 0; i < shallows.nr; i++) - register_shallow(&shallows.objects[i].item->oid); - } - - shallow_nr += shallows.nr; + if (send_shallow_list(depth, deepen_rev_list, deepen_since, + &deepen_not, &shallows)) + packet_flush(1); object_array_clear(&shallows); } @@ -1004,33 +1049,6 @@ static int find_symref(const char *refname, const struct object_id *oid, return 0; } -static void upload_pack(void) -{ - struct string_list symref = STRING_LIST_INIT_DUP; - - head_ref_namespaced(find_symref, &symref); - - if (advertise_refs || !stateless_rpc) { - reset_timeout(); - head_ref_namespaced(send_ref, &symref); - for_each_namespaced_ref(send_ref, &symref); - advertise_shallow_grafts(1); - packet_flush(1); - } else { - head_ref_namespaced(check_ref, NULL); - for_each_namespaced_ref(check_ref, NULL); - } - string_list_clear(&symref, 1); - if (advertise_refs) - return; - - receive_needs(); - if (want_obj.nr) { - get_common_commits(); - create_pack_file(); - } -} - static int upload_pack_config(const char *var, const char *value, void *unused) { if (!strcmp("uploadpack.allowtipsha1inwant", var)) { @@ -1061,58 +1079,356 @@ static int upload_pack_config(const char *var, const char *value, void *unused) return parse_hide_refs_config(var, value, "uploadpack"); } -int cmd_main(int argc, const char **argv) +void upload_pack(struct upload_pack_options *options) { - const char *dir; - int strict = 0; - struct option options[] = { - OPT_BOOL(0, "stateless-rpc", &stateless_rpc, - N_("quit after a single request/response exchange")), - OPT_BOOL(0, "advertise-refs", &advertise_refs, - N_("exit immediately after initial ref advertisement")), - OPT_BOOL(0, "strict", &strict, - N_("do not try <directory>/.git/ if <directory> is no Git directory")), - OPT_INTEGER(0, "timeout", &timeout, - N_("interrupt transfer after <n> seconds of inactivity")), - OPT_END() - }; + struct string_list symref = STRING_LIST_INIT_DUP; - packet_trace_identity("upload-pack"); - check_replace_refs = 0; + stateless_rpc = options->stateless_rpc; + timeout = options->timeout; + daemon_mode = options->daemon_mode; - argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); + git_config(upload_pack_config, NULL); - if (argc != 1) - usage_with_options(upload_pack_usage, options); + head_ref_namespaced(find_symref, &symref); - if (timeout) - daemon_mode = 1; + if (options->advertise_refs || !stateless_rpc) { + reset_timeout(); + head_ref_namespaced(send_ref, &symref); + for_each_namespaced_ref(send_ref, &symref); + advertise_shallow_grafts(1); + packet_flush(1); + } else { + head_ref_namespaced(check_ref, NULL); + for_each_namespaced_ref(check_ref, NULL); + } + string_list_clear(&symref, 1); + if (options->advertise_refs) + return; - setup_path(); + receive_needs(); + if (want_obj.nr) { + get_common_commits(); + create_pack_file(); + } +} - dir = argv[0]; +struct upload_pack_data { + struct object_array wants; + struct oid_array haves; - if (!enter_repo(dir, strict)) - die("'%s' does not appear to be a git repository", dir); + struct object_array shallows; + struct string_list deepen_not; + int depth; + timestamp_t deepen_since; + int deepen_rev_list; + int deepen_relative; - git_config(upload_pack_config, NULL); + unsigned stateless_rpc : 1; - switch (determine_protocol_version_server()) { - case protocol_v1: - /* - * v1 is just the original protocol with a version string, - * so just fall through after writing the version string. - */ - if (advertise_refs || !stateless_rpc) - packet_write_fmt(1, "version 1\n"); - - /* fallthrough */ - case protocol_v0: - upload_pack(); - break; - case protocol_unknown_version: - BUG("unknown protocol version"); + unsigned use_thin_pack : 1; + unsigned use_ofs_delta : 1; + unsigned no_progress : 1; + unsigned use_include_tag : 1; + unsigned done : 1; +}; + +static void upload_pack_data_init(struct upload_pack_data *data) +{ + struct object_array wants = OBJECT_ARRAY_INIT; + struct oid_array haves = OID_ARRAY_INIT; + struct object_array shallows = OBJECT_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; + + memset(data, 0, sizeof(*data)); + data->wants = wants; + data->haves = haves; + data->shallows = shallows; + data->deepen_not = deepen_not; +} + +static void upload_pack_data_clear(struct upload_pack_data *data) +{ + object_array_clear(&data->wants); + oid_array_clear(&data->haves); + object_array_clear(&data->shallows); + string_list_clear(&data->deepen_not, 0); +} + +static int parse_want(const char *line) +{ + const char *arg; + if (skip_prefix(line, "want ", &arg)) { + struct object_id oid; + struct object *o; + + if (get_oid_hex(arg, &oid)) + die("git upload-pack: protocol error, " + "expected to get oid, not '%s'", line); + + o = parse_object(&oid); + if (!o) { + packet_write_fmt(1, + "ERR upload-pack: not our ref %s", + oid_to_hex(&oid)); + die("git upload-pack: not our ref %s", + oid_to_hex(&oid)); + } + + if (!(o->flags & WANTED)) { + o->flags |= WANTED; + add_object_array(o, NULL, &want_obj); + } + + return 1; + } + + return 0; +} + +static int parse_have(const char *line, struct oid_array *haves) +{ + const char *arg; + if (skip_prefix(line, "have ", &arg)) { + struct object_id oid; + + if (get_oid_hex(arg, &oid)) + die("git upload-pack: expected SHA1 object, got '%s'", arg); + oid_array_append(haves, &oid); + return 1; } return 0; } + +static void process_args(struct packet_reader *request, + struct upload_pack_data *data) +{ + while (packet_reader_read(request) != PACKET_READ_FLUSH) { + const char *arg = request->line; + + /* process want */ + if (parse_want(arg)) + continue; + /* process have line */ + if (parse_have(arg, &data->haves)) + continue; + + /* process args like thin-pack */ + if (!strcmp(arg, "thin-pack")) { + use_thin_pack = 1; + continue; + } + if (!strcmp(arg, "ofs-delta")) { + use_ofs_delta = 1; + continue; + } + if (!strcmp(arg, "no-progress")) { + no_progress = 1; + continue; + } + if (!strcmp(arg, "include-tag")) { + use_include_tag = 1; + continue; + } + if (!strcmp(arg, "done")) { + data->done = 1; + continue; + } + + /* Shallow related arguments */ + if (process_shallow(arg, &data->shallows)) + continue; + if (process_deepen(arg, &data->depth)) + continue; + if (process_deepen_since(arg, &data->deepen_since, + &data->deepen_rev_list)) + continue; + if (process_deepen_not(arg, &data->deepen_not, + &data->deepen_rev_list)) + continue; + if (!strcmp(arg, "deepen-relative")) { + data->deepen_relative = 1; + continue; + } + + /* ignore unknown lines maybe? */ + die("unexpect line: '%s'", arg); + } +} + +static int process_haves(struct oid_array *haves, struct oid_array *common) +{ + int i; + + /* Process haves */ + for (i = 0; i < haves->nr; i++) { + const struct object_id *oid = &haves->oid[i]; + struct object *o; + int we_knew_they_have = 0; + + if (!has_object_file(oid)) + continue; + + oid_array_append(common, oid); + + o = parse_object(oid); + if (!o) + die("oops (%s)", oid_to_hex(oid)); + if (o->type == OBJ_COMMIT) { + struct commit_list *parents; + struct commit *commit = (struct commit *)o; + if (o->flags & THEY_HAVE) + we_knew_they_have = 1; + else + o->flags |= THEY_HAVE; + if (!oldest_have || (commit->date < oldest_have)) + oldest_have = commit->date; + for (parents = commit->parents; + parents; + parents = parents->next) + parents->item->object.flags |= THEY_HAVE; + } + if (!we_knew_they_have) + add_object_array(o, NULL, &have_obj); + } + + return 0; +} + +static int send_acks(struct oid_array *acks, struct strbuf *response) +{ + int i; + + packet_buf_write(response, "acknowledgments\n"); + + /* Send Acks */ + if (!acks->nr) + packet_buf_write(response, "NAK\n"); + + for (i = 0; i < acks->nr; i++) { + packet_buf_write(response, "ACK %s\n", + oid_to_hex(&acks->oid[i])); + } + + if (ok_to_give_up()) { + /* Send Ready */ + packet_buf_write(response, "ready\n"); + return 1; + } + + return 0; +} + +static int process_haves_and_send_acks(struct upload_pack_data *data) +{ + struct oid_array common = OID_ARRAY_INIT; + struct strbuf response = STRBUF_INIT; + int ret = 0; + + process_haves(&data->haves, &common); + if (data->done) { + ret = 1; + } else if (send_acks(&common, &response)) { + packet_buf_delim(&response); + ret = 1; + } else { + /* Add Flush */ + packet_buf_flush(&response); + ret = 0; + } + + /* Send response */ + write_or_die(1, response.buf, response.len); + strbuf_release(&response); + + oid_array_clear(&data->haves); + oid_array_clear(&common); + return ret; +} + +static void send_shallow_info(struct upload_pack_data *data) +{ + /* No shallow info needs to be sent */ + if (!data->depth && !data->deepen_rev_list && !data->shallows.nr && + !is_repository_shallow()) + return; + + packet_write_fmt(1, "shallow-info\n"); + + if (!send_shallow_list(data->depth, data->deepen_rev_list, + data->deepen_since, &data->deepen_not, + &data->shallows) && is_repository_shallow()) + deepen(INFINITE_DEPTH, data->deepen_relative, &data->shallows); + + packet_delim(1); +} + +enum fetch_state { + FETCH_PROCESS_ARGS = 0, + FETCH_SEND_ACKS, + FETCH_SEND_PACK, + FETCH_DONE, +}; + +int upload_pack_v2(struct repository *r, struct argv_array *keys, + struct packet_reader *request) +{ + enum fetch_state state = FETCH_PROCESS_ARGS; + struct upload_pack_data data; + + upload_pack_data_init(&data); + use_sideband = LARGE_PACKET_MAX; + + while (state != FETCH_DONE) { + switch (state) { + case FETCH_PROCESS_ARGS: + process_args(request, &data); + + if (!want_obj.nr) { + /* + * Request didn't contain any 'want' lines, + * guess they didn't want anything. + */ + state = FETCH_DONE; + } else if (data.haves.nr) { + /* + * Request had 'have' lines, so lets ACK them. + */ + state = FETCH_SEND_ACKS; + } else { + /* + * Request had 'want's but no 'have's so we can + * immedietly go to construct and send a pack. + */ + state = FETCH_SEND_PACK; + } + break; + case FETCH_SEND_ACKS: + if (process_haves_and_send_acks(&data)) + state = FETCH_SEND_PACK; + else + state = FETCH_DONE; + break; + case FETCH_SEND_PACK: + send_shallow_info(&data); + + packet_write_fmt(1, "packfile\n"); + create_pack_file(); + state = FETCH_DONE; + break; + case FETCH_DONE: + continue; + } + } + + upload_pack_data_clear(&data); + return 0; +} + +int upload_pack_advertise(struct repository *r, + struct strbuf *value) +{ + if (value) + strbuf_addstr(value, "shallow"); + return 1; +} diff --git a/upload-pack.h b/upload-pack.h new file mode 100644 index 0000000..cab2178 --- /dev/null +++ b/upload-pack.h @@ -0,0 +1,23 @@ +#ifndef UPLOAD_PACK_H +#define UPLOAD_PACK_H + +struct upload_pack_options { + int stateless_rpc; + int advertise_refs; + unsigned int timeout; + int daemon_mode; +}; + +void upload_pack(struct upload_pack_options *options); + +struct repository; +struct argv_array; +struct packet_reader; +extern int upload_pack_v2(struct repository *r, struct argv_array *keys, + struct packet_reader *request); + +struct strbuf; +extern int upload_pack_advertise(struct repository *r, + struct strbuf *value); + +#endif /* UPLOAD_PACK_H */ |