From 17b219f5461d6eb7e30dbb9fb4fbf3200c2db6c4 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 5 Jan 2026 15:36:42 -0500 Subject: [PATCH 1/8] fixup! gvfs-helper: create tool to fetch objects using the GVFS Protocol Before modifying the config documentation more, fill in these blanks. Signed-off-by: Derrick Stolee --- Documentation/config/gvfs.adoc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/config/gvfs.adoc b/Documentation/config/gvfs.adoc index 7224939ac0b270..da771a0535943a 100644 --- a/Documentation/config/gvfs.adoc +++ b/Documentation/config/gvfs.adoc @@ -1,8 +1,10 @@ gvfs.cache-server:: - TODO + When set, redirect all GVFS Protocol requests to this base URL instead + of the origin server. gvfs.sharedcache:: - TODO + When set, place all object data downloaded via the GVFS Protocol into + this Git alternate. gvfs.fallback:: If set to `false`, then never fallback to the origin server when the cache From 62489c70a965b2d0e774cfdb648b30955f4efec0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sat, 17 Jan 2026 11:01:48 -0500 Subject: [PATCH 2/8] t5799: update cache-server methods for multiple instances In anticipation of tests for multiple cache-servers, update the existing logic that sets up and tears down cache-servers to allow multiple instances on different ports. Signed-off-by: Derrick Stolee --- t/t5799-gvfs-helper.sh | 102 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 91 insertions(+), 11 deletions(-) diff --git a/t/t5799-gvfs-helper.sh b/t/t5799-gvfs-helper.sh index 8fcab959e7b8af..87cac9f021084c 100755 --- a/t/t5799-gvfs-helper.sh +++ b/t/t5799-gvfs-helper.sh @@ -58,6 +58,42 @@ SHARED_CACHE_T2="$(pwd)"/shared_cache_t2 PID_FILE="$(pwd)"/pid-file.pid SERVER_LOG="$(pwd)"/OUT.server.log +# Helper functions to compute port, pid-file, and log for a given +# port increment. An increment of 0 (or empty) uses the base values. +# +server_port () { + local instance="${1:-0}" + echo "$(($GIT_TEST_GVFS_PROTOCOL_PORT + $instance))" +} + +server_pid_file () { + local instance="${1:-0}" + if test "$instance" -eq 0 + then + echo "$PID_FILE" + else + echo "$(pwd)/pid-file-$instance.pid" + fi +} + +server_log_file () { + local instance="${1:-0}" + if test "$instance" -eq 0 + then + echo "$SERVER_LOG" + else + echo "$(pwd)/OUT.server-$instance.log" + fi +} + +# Helper to build a cache-server URL for a given port increment. +# +cache_server_url () { + local instance="${1:-0}" + local port=$(server_port "$instance") + echo "http://127.0.0.1:$port/servertype/cache" +} + PATH="$GIT_BUILD_DIR/t/helper/:$PATH" && export PATH OIDS_FILE="$(pwd)"/oid_list.txt @@ -229,15 +265,25 @@ test_expect_success 'setup repos' ' get_one_commit_oid ' +# Stop a gvfs-protocol server. +# Usage: stop_gvfs_protocol_server [] +# +# The optional port_increment (default 0) specifies which server to stop. +# Increment 0 uses the base port, 1 uses base+1, etc. +# stop_gvfs_protocol_server () { - if ! test -f "$PID_FILE" + local instance="${1:-0}" + local pid_file=$(server_pid_file "$instance") + local log_file=$(server_log_file "$instance") + + if ! test -f "$pid_file" then return 0 fi # # The server will shutdown automatically when we delete the pid-file. # - rm -f "$PID_FILE" + rm -f "$pid_file" # # Give it a few seconds to shutdown (mainly to completely release the # port before the next test start another instance and it attempts to @@ -245,18 +291,29 @@ stop_gvfs_protocol_server () { # for k in 0 1 2 3 4 do - if grep -q "Starting graceful shutdown" "$SERVER_LOG" + if grep -q "Starting graceful shutdown" "$log_file" then return 0 fi sleep 1 done - echo "stop_gvfs_protocol_server: timeout waiting for server shutdown" + echo "stop_gvfs_protocol_server($instance): timeout waiting for server shutdown" return 1 } +# Start a gvfs-protocol server. +# Usage: start_gvfs_protocol_server [] +# +# The optional port_increment (default 0) specifies which server to start. +# Increment 0 uses the base port, 1 uses base+1, etc. +# This allows running multiple servers simultaneously on different ports. +# start_gvfs_protocol_server () { + local instance="${1:-0}" + local port=$(server_port "$instance") + local pid_file=$(server_pid_file "$instance") + local log_file=$(server_log_file "$instance") # # Launch our server into the background in repo_src. # @@ -264,24 +321,24 @@ start_gvfs_protocol_server () { cd "$REPO_SRC" test-gvfs-protocol --verbose \ --listen=127.0.0.1 \ - --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --port=$port \ --reuseaddr \ - --pid-file="$PID_FILE" \ - 2>"$SERVER_LOG" & + --pid-file="$pid_file" \ + 2>"$log_file" & ) # # Give it a few seconds to get started. # for k in 0 1 2 3 4 do - if test -f "$PID_FILE" + if test -f "$pid_file" then return 0 fi sleep 1 done - echo "start_gvfs_protocol_server: timeout waiting for server startup" + echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" return 1 } @@ -322,10 +379,28 @@ start_gvfs_protocol_server_with_mayhem () { sleep 1 done - echo "start_gvfs_protocol_server: timeout waiting for server startup" + echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" return 1 } +# Verify that a server received at least one connection. +# Usage: verify_server_was_contacted [] +# +verify_server_was_contacted () { + local instance="${1:-0}" + local log_file=$(server_log_file "$instance") + grep -q "Connection from" "$log_file" +} + +# Verify that a server was NOT contacted. +# Usage: verify_server_was_not_contacted [] +# +verify_server_was_not_contacted () { + local instance="${1:-0}" + local log_file=$(server_log_file "$instance") + ! grep -q "Connection from" "$log_file" +} + # Verify the number of connections from the client. # # If keep-alive is working, a series of successful sequential requests to the @@ -449,7 +524,12 @@ verify_vfs_packfile_count () { } per_test_cleanup () { - stop_gvfs_protocol_server + # Stop servers with port increments 0, 1, 2, 3 to handle tests + # that may use multiple servers. + for instance in 0 1 2 3 + do + stop_gvfs_protocol_server "$instance" + done rm -rf "$SHARED_CACHE_T1"/[0-9a-f][0-9a-f]/ rm -rf "$SHARED_CACHE_T1"/info/* From de32ee7af7eb3ac14fde99cbcaf7e3b05f112e52 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 5 Jan 2026 15:56:50 -0500 Subject: [PATCH 3/8] gvfs-helper: override cache server for prefetch This extension of the gvfs.cache-server config now allows a new key, gvfs.prefetch.cache-server, to override the cache-server URL for only the prefetch endpoint. The purpose of this config is to allow for incremental testing and deployment of new cache-server infrastructure. Hypothetically, we could have special-purpose cache-servers that are glorified bundle servers and other servers that focus on the object and size endpoints. More realistically, this will allow us to test cache servers that have only the prefetch endpoint ready to go. This allows some incremental rollout that is more controlled than a flag day replacing the entire infrastructure. Signed-off-by: Derrick Stolee --- Documentation/config/gvfs.adoc | 12 ++++++- gvfs-helper.c | 63 +++++++++++++++++++++++++++++++++- t/t5799-gvfs-helper.sh | 61 ++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/Documentation/config/gvfs.adoc b/Documentation/config/gvfs.adoc index da771a0535943a..ed6e94f31aa990 100644 --- a/Documentation/config/gvfs.adoc +++ b/Documentation/config/gvfs.adoc @@ -1,6 +1,16 @@ gvfs.cache-server:: When set, redirect all GVFS Protocol requests to this base URL instead - of the origin server. + of the origin server. Individual verbs can be overridden with the + `gvfs..cache-server` config keys. + +gvfs..cache-server:: + Override the base value of `gvfs.cache-server` when using this specific + ``. The verbs available are: ++ +-- + prefetch:: + Use this cache server when prefetching commits and tree packfiles. +-- gvfs.sharedcache:: When set, place all object data downloaded via the GVFS Protocol into diff --git a/gvfs-helper.c b/gvfs-helper.c index 77bc65996ded36..20b4067ddabdc1 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -374,7 +374,8 @@ static struct gh__global { struct credential cache_creds; const char *main_url; - const char *cache_server_url; + char *cache_server_url; + char *cache_server_url_backup; struct strbuf buf_odb_path; @@ -392,6 +393,54 @@ enum gh__server_type { GH__SERVER_TYPE__NR, }; +enum gh__verb { + PREFETCH, +}; + +static void update_cache_server_for_verb(enum gh__verb verb) +{ + const char *verbstr = NULL; + char *value = NULL; + struct strbuf key = STRBUF_INIT; + + switch (verb) { + case PREFETCH: + verbstr = "prefetch"; + break; + + default: + gh__global.cache_server_url_backup = NULL; + return; + } + + gh__global.cache_server_url_backup = gh__global.cache_server_url; + + strbuf_addf(&key, "gvfs.%s.cache-server", verbstr); + + if (!repo_config_get_string(the_repository, key.buf, &value) && + value) { + trace2_data_string("gvfs-helper", the_repository, key.buf, value); + gh__global.cache_server_url = value; + } else { + gh__global.cache_server_url_backup = NULL; + } + + strbuf_release(&key); +} + +static void reset_cache_server(void) +{ + /* + * The backup exists only if the base was replaced with a + * freeable value. + */ + if (gh__global.cache_server_url_backup) { + free(gh__global.cache_server_url); + gh__global.cache_server_url = gh__global.cache_server_url_backup; + gh__global.cache_server_url_backup = NULL; + } +} + static const char *gh__server_type_label[GH__SERVER_TYPE__NR] = { "(main)", "(cs)" @@ -3168,6 +3217,7 @@ static void do_req__with_fallback(const char *url_component, struct gh__request_params *params, struct gh__response_status *status) { +retry_backup: if (gh__global.cache_server_url && params->b_permit_cache_server_if_defined) { do_req__to_cache_server(url_component, params, status); @@ -3178,6 +3228,15 @@ static void do_req__with_fallback(const char *url_component, if (!gh__cmd_opts.try_fallback) return; + /* + * If we overrode the cache-server using a custom key, + * then fall back to the regular cache server on failure. + */ + if (gh__global.cache_server_url_backup) { + reset_cache_server(); + goto retry_backup; + } + /* * The cache-server shares creds with the main Git server, * so if our creds failed against the cache-server, they @@ -3493,7 +3552,9 @@ static void do__http_get__gvfs_prefetch(struct gh__response_status *status, show_date(seconds_since_epoch, 0, DATE_MODE(ISO8601))); + update_cache_server_for_verb(PREFETCH); do_req__with_fallback(component_url.buf, ¶ms, status); + reset_cache_server(); gh__request_params__release(¶ms); strbuf_release(&component_url); diff --git a/t/t5799-gvfs-helper.sh b/t/t5799-gvfs-helper.sh index 87cac9f021084c..b299d23e2972be 100755 --- a/t/t5799-gvfs-helper.sh +++ b/t/t5799-gvfs-helper.sh @@ -1664,4 +1664,65 @@ test_expect_success 'prefetch corrupt pack with corrupt idx' ' stop_gvfs_protocol_server ' +################################################################# +# Tests for gvfs..cache-server config. +# +# These tests verify that verb-specific cache-server overrides work +# correctly. We run two servers on different ports: +# - Server 0 (base port): configured as gvfs.cache-server (default) +# - Server 1 (base port + 1): configured as gvfs..cache-server +# +# For each verb (prefetch, get, post), we verify that: +# 1. When using the verb-specific override, the request goes to server 1 +# 2. When using a different verb, the request goes to server 0 +################################################################# + +test_expect_success 'verb-specific cache-server: prefetch uses gvfs.prefetch.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for prefetch. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + + # Run prefetch - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (prefetch-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for prefetch. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 0 (default), not server 1 (prefetch). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (prefetch-specific). + verify_server_was_not_contacted 1 +' + test_done From bcf6149a4063fa41fc1181338afba96f6afa3652 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 5 Jan 2026 16:00:40 -0500 Subject: [PATCH 4/8] gvfs-helper: override cache server for get This extension of the gvfs.cache-server config now allows a new key, gvfs.get.cache-server, to override the cache-server URL for only the prefetch endpoint. The purpose of this config is to allow for incremental testing and deployment of new cache-server infrastructure. Signed-off-by: Derrick Stolee --- Documentation/config/gvfs.adoc | 3 +++ gvfs-helper.c | 7 +++++ t/t5799-gvfs-helper.sh | 48 ++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/Documentation/config/gvfs.adoc b/Documentation/config/gvfs.adoc index ed6e94f31aa990..0a9b6bd524ffc3 100644 --- a/Documentation/config/gvfs.adoc +++ b/Documentation/config/gvfs.adoc @@ -10,6 +10,9 @@ gvfs..cache-server:: -- prefetch:: Use this cache server when prefetching commits and tree packfiles. + get:: + Use this cache server when downloading objects immediately via the + GET endpoint. -- gvfs.sharedcache:: diff --git a/gvfs-helper.c b/gvfs-helper.c index 20b4067ddabdc1..dadd409bdad384 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -395,6 +395,7 @@ enum gh__server_type { enum gh__verb { PREFETCH, + GET, }; static void update_cache_server_for_verb(enum gh__verb verb) @@ -408,6 +409,10 @@ static void update_cache_server_for_verb(enum gh__verb verb) verbstr = "prefetch"; break; + case GET: + verbstr = "get"; + break; + default: gh__global.cache_server_url_backup = NULL; return; @@ -3360,7 +3365,9 @@ static void do__http_get__gvfs_object(struct gh__response_status *status, setup_gvfs_objects_progress(¶ms, l_num, l_den); + update_cache_server_for_verb(GET); do_req__with_fallback(component_url.buf, ¶ms, status); + reset_cache_server(); gh__request_params__release(¶ms); strbuf_release(&component_url); diff --git a/t/t5799-gvfs-helper.sh b/t/t5799-gvfs-helper.sh index b299d23e2972be..994c2c31b87dfe 100755 --- a/t/t5799-gvfs-helper.sh +++ b/t/t5799-gvfs-helper.sh @@ -1725,4 +1725,52 @@ test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch. verify_server_was_not_contacted 1 ' +test_expect_success 'verb-specific cache-server: get uses gvfs.get.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for get. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (get-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for get. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && + + # Run prefetch - should go to server 0 (default), not server 1 (get). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (get-specific). + verify_server_was_not_contacted 1 +' + test_done From abd4b2cedd3bfd7bb38c45579851e7ae02a87209 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 5 Jan 2026 16:02:42 -0500 Subject: [PATCH 5/8] gvfs-helper: override cache server for post This extension of the gvfs.cache-server config now allows a new key, gvfs.post.cache-server, to overrid the cache-server URL for only the batched objects endpoint. The purpose of this config is to allow for incremental testing and deployment of new cache-server infrastructure. Signed-off-by: Derrick Stolee --- Documentation/config/gvfs.adoc | 3 +++ gvfs-helper.c | 7 +++++ t/t5799-gvfs-helper.sh | 49 ++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/Documentation/config/gvfs.adoc b/Documentation/config/gvfs.adoc index 0a9b6bd524ffc3..4d988598425cec 100644 --- a/Documentation/config/gvfs.adoc +++ b/Documentation/config/gvfs.adoc @@ -13,6 +13,9 @@ gvfs..cache-server:: get:: Use this cache server when downloading objects immediately via the GET endpoint. + post:: + Use this cache server when downloading objects in batches using the + POST endpoint. -- gvfs.sharedcache:: diff --git a/gvfs-helper.c b/gvfs-helper.c index dadd409bdad384..8602c7764f7e2d 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -396,6 +396,7 @@ enum gh__server_type { enum gh__verb { PREFETCH, GET, + POST, }; static void update_cache_server_for_verb(enum gh__verb verb) @@ -413,6 +414,10 @@ static void update_cache_server_for_verb(enum gh__verb verb) verbstr = "get"; break; + case POST: + verbstr = "post"; + break; + default: gh__global.cache_server_url_backup = NULL; return; @@ -3439,7 +3444,9 @@ static void do__http_post__gvfs_objects(struct gh__response_status *status, setup_gvfs_objects_progress(¶ms, j_pack_num, j_pack_den); + update_cache_server_for_verb(POST); do_req__with_fallback("gvfs/objects", ¶ms, status); + reset_cache_server(); gh__request_params__release(¶ms); jw_release(&jw_req); diff --git a/t/t5799-gvfs-helper.sh b/t/t5799-gvfs-helper.sh index 994c2c31b87dfe..2bcd38025a6aea 100755 --- a/t/t5799-gvfs-helper.sh +++ b/t/t5799-gvfs-helper.sh @@ -1773,4 +1773,53 @@ test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get. verify_server_was_not_contacted 1 ' +test_expect_success 'verb-specific cache-server: post uses gvfs.post.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for post. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && + + # Run post - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (post-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for post. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 0 (default), not server 1 (post). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (post-specific). + verify_server_was_not_contacted 1 +' + test_done From c69b69e269ca79c3257441c61b5f3d2c8cde1379 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sat, 17 Jan 2026 11:07:07 -0500 Subject: [PATCH 6/8] t5799: add test for all verb-specific cache-servers together Test that all three verb-specific cache-server configs can be used simultaneously, each directing requests to a different server. This verifies that prefetch, get, and post verbs each respect their own override and don't interfere with each other. --- t/t5799-gvfs-helper.sh | 57 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/t/t5799-gvfs-helper.sh b/t/t5799-gvfs-helper.sh index 2bcd38025a6aea..00a573ba21f405 100755 --- a/t/t5799-gvfs-helper.sh +++ b/t/t5799-gvfs-helper.sh @@ -1822,4 +1822,61 @@ test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cach verify_server_was_not_contacted 1 ' +test_expect_success 'verb-specific cache-server: all verbs with different servers' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 3 && + + # Configure each verb to use a different server: + # - server 0: default (unused in this test) + # - server 1: prefetch + # - server 2: get + # - server 3: post + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 2)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 3)" && + + # Run prefetch - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + verify_server_was_contacted 1 && + verify_server_was_not_contacted 0 && + verify_server_was_not_contacted 2 && + verify_server_was_not_contacted 3 && + + # Clean up shared cache for next verb. + rm -rf "$SHARED_CACHE_T1"/pack/* && + + # Run get - should go to server 2. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + verify_server_was_contacted 2 && + + # Clean up shared cache for next verb. + rm -rf "$SHARED_CACHE_T1"/[0-9a-f][0-9a-f]/ && + rm -rf "$SHARED_CACHE_T1"/pack/* && + + # Run post - should go to server 3. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + verify_server_was_contacted 3 +' + test_done From 547b49ac2b9033c8fe0f46ac4f1d8926152999c0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sat, 17 Jan 2026 20:50:24 -0500 Subject: [PATCH 7/8] lib-gvfs-helper: create helper script for protocol tests The t5799-gvfs-helper.sh script is long and takes forever. This slows down PR merges and the local development inner loop is a pain. Before distributing the tests into a set of new test scripts by topic, extract important helper methods that can be imported by the new scripts. Signed-off-by: Derrick Stolee --- t/lib-gvfs-helper.sh | 551 +++++++++++++++++++++++++++++++++++++++++ t/t5799-gvfs-helper.sh | 543 +--------------------------------------- 2 files changed, 552 insertions(+), 542 deletions(-) create mode 100644 t/lib-gvfs-helper.sh diff --git a/t/lib-gvfs-helper.sh b/t/lib-gvfs-helper.sh new file mode 100644 index 00000000000000..fac3b9210cb551 --- /dev/null +++ b/t/lib-gvfs-helper.sh @@ -0,0 +1,551 @@ +# Shared library for gvfs-helper tests +# +# This file is sourced by t579*-gvfs-helper*.sh scripts. +# The sourcing script MUST call: +# 1. . ./test-lib.sh +# 2. . "$TEST_DIRECTORY"/lib-gvfs-helper.sh +# 3. init_gvfs_helper_vars +# 4. test_expect_success 'setup repos' 'setup_gvfs_repos' + +# Set the port for t/helper/test-gvfs-protocol.exe from either the +# environment or from the test number of this shell script. +# +test_set_port GIT_TEST_GVFS_PROTOCOL_PORT + +# Setup the following repos: +# +# repo_src: +# A normal, no-magic, fully-populated clone of something. +# No GVFS (aka VFS4G). No Scalar. No partial-clone. +# This will be used by "t/helper/test-gvfs-protocol.exe" +# to serve objects. +# +# repo_t1: +# An empty repo with no contents nor commits. That is, +# everything is missing. For the tests based on this repo, +# we don't care why it is missing objects (or if we could +# actually use it). We are only testing explicit object +# fetching using gvfs-helper.exe in isolation. +# +# repo_t2: +# Another empty repo to use after we contaminate t1. +# +REPO_SRC="$(pwd)"/repo_src +REPO_T1="$(pwd)"/repo_t1 +REPO_T2="$(pwd)"/repo_t2 + +# Setup some loopback URLs where test-gvfs-protocol.exe will be +# listening. We will spawn it directly inside the repo_src directory, +# so we don't need any of the directory mapping or configuration +# machinery found in "git-daemon.exe" or "git-http-backend.exe". +# +# This lets us use the "uri-base" part of the URL (prior to the REST +# API "/gvfs/") to control how our mock server responds. For +# example, only the origin (main Git) server supports "/gvfs/config". +# +# For example, this means that if we add a remote containing $ORIGIN_URL, +# it will work with gvfs-helper, but not for fetch (without some mapping +# tricks). +# +HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT +ORIGIN_URL=http://$HOST_PORT/servertype/origin +CACHE_URL=http://$HOST_PORT/servertype/cache + +SHARED_CACHE_T1="$(pwd)"/shared_cache_t1 +SHARED_CACHE_T2="$(pwd)"/shared_cache_t2 + +# The pid-file is created by test-gvfs-protocol.exe when it starts. +# The server will shut down if/when we delete it. (This is a little +# easier than killing it by PID.) +# +PID_FILE="$(pwd)"/pid-file.pid +SERVER_LOG="$(pwd)"/OUT.server.log + +# Helper functions to compute port, pid-file, and log for a given +# port increment. An increment of 0 (or empty) uses the base values. +# +server_port () { + local instance="${1:-0}" + echo $(($GIT_TEST_GVFS_PROTOCOL_PORT + "$instance")) +} + +server_pid_file () { + local instance="${1:-0}" + if test "$instance" -eq 0 + then + echo "$PID_FILE" + else + echo "$(pwd)/pid-file-$instance.pid" + fi +} + +server_log_file () { + local instance="${1:-0}" + if test "$instance" -eq 0 + then + echo "$SERVER_LOG" + else + echo "$(pwd)/OUT.server-$instance.log" + fi +} + +# Helper to build a cache-server URL for a given port increment. +# +cache_server_url () { + local instance="${1:-0}" + local port=$(server_port "$instance") + echo "http://127.0.0.1:$port/servertype/cache" +} + +PATH="$GIT_BUILD_DIR/t/helper/:$PATH" && export PATH + +OIDS_FILE="$(pwd)"/oid_list.txt +OIDS_CT_FILE="$(pwd)"/oid_ct_list.txt +OIDS_BLOBS_FILE="$(pwd)"/oids_blobs_file.txt +OID_ONE_BLOB_FILE="$(pwd)"/oid_one_blob_file.txt +OID_ONE_COMMIT_FILE="$(pwd)"/oid_one_commit_file.txt + +# Get a list of available OIDs in repo_src so that we can try to fetch +# them and so that we don't have to hard-code a list of known OIDs. +# This doesn't need to be a complete list -- just enough to drive some +# representative tests. +# +# Optionally require that we find a minimum number of OIDs. +# +get_list_of_oids () { + git -C "$REPO_SRC" rev-list --objects HEAD | sed 's/ .*//' | sort >"$OIDS_FILE" + + if test $# -eq 1 + then + actual_nr=$(wc -l <"$OIDS_FILE") + if test $actual_nr -lt $1 + then + echo "get_list_of_oids: insufficient data. Need $1 OIDs." + return 1 + fi + fi + return 0 +} + +get_list_of_blobs_oids () { + git -C "$REPO_SRC" ls-tree HEAD | grep ' blob ' | awk "{print \$3}" | sort >"$OIDS_BLOBS_FILE" + head -1 <"$OIDS_BLOBS_FILE" >"$OID_ONE_BLOB_FILE" +} + +get_list_of_commit_and_tree_oids () { + git -C "$REPO_SRC" cat-file --batch-check --batch-all-objects | awk "/commit|tree/ {print \$1}" | sort >"$OIDS_CT_FILE" + + if test $# -eq 1 + then + actual_nr=$(wc -l <"$OIDS_CT_FILE") + if test $actual_nr -lt $1 + then + echo "get_list_of_commit_and_tree_oids: insufficient data. Need $1 OIDs." + return 1 + fi + fi + return 0 +} + +get_one_commit_oid () { + git -C "$REPO_SRC" rev-parse HEAD >"$OID_ONE_COMMIT_FILE" + return 0 +} + +# Create a commits-and-trees packfile for use with "prefetch" +# using the given range of commits. +# +create_commits_and_trees_packfile () { + if test $# -eq 2 + then + epoch=$1 + revs=$2 + else + echo "create_commits_and_trees_packfile: Need 2 args" + return 1 + fi + + pack_file="$REPO_SRC"/.git/objects/pack/ct-$epoch.pack + idx_file="$REPO_SRC"/.git/objects/pack/ct-$epoch.idx + + git -C "$REPO_SRC" pack-objects --stdout --revs --filter=blob:none \ + >"$pack_file" <<-EOF + $revs + EOF + git -C "$REPO_SRC" index-pack -o "$idx_file" "$pack_file" + return 0 +} + +test_expect_success 'setup repos' ' + test_create_repo "$REPO_SRC" && + git -C "$REPO_SRC" branch -M main && + # + # test_commit_bulk() does magic to create a packfile containing + # the new commits. + # + # We create branches in repo_src, but also remember the branch OIDs + # in files so that we can refer to them in repo_t1, which will not + # have the commits locally (because we do not clone or fetch). + # + test_commit_bulk -C "$REPO_SRC" --filename="batch_a.%s.t" 9 && + git -C "$REPO_SRC" branch B1 && + git -C "$REPO_SRC" rev-parse refs/heads/main >m1.branch && + # + test_commit_bulk -C "$REPO_SRC" --filename="batch_b.%s.t" 9 && + git -C "$REPO_SRC" branch B2 && + git -C "$REPO_SRC" rev-parse refs/heads/main >m2.branch && + # + # test_commit() creates commits, trees, tags, and blobs and leave + # them loose. + # + test_config gc.auto 0 && + # + test_commit -C "$REPO_SRC" file1.txt && + test_commit -C "$REPO_SRC" file2.txt && + test_commit -C "$REPO_SRC" file3.txt && + test_commit -C "$REPO_SRC" file4.txt && + test_commit -C "$REPO_SRC" file5.txt && + test_commit -C "$REPO_SRC" file6.txt && + test_commit -C "$REPO_SRC" file7.txt && + test_commit -C "$REPO_SRC" file8.txt && + test_commit -C "$REPO_SRC" file9.txt && + git -C "$REPO_SRC" branch B3 && + git -C "$REPO_SRC" rev-parse refs/heads/main >m3.branch && + # + # Create some commits-and-trees-only packfiles for testing prefetch. + # Set arbitrary EPOCH times to make it easier to test fetch-since. + # + create_commits_and_trees_packfile 1000000000 B1 && + create_commits_and_trees_packfile 1100000000 B1..B2 && + create_commits_and_trees_packfile 1200000000 B2..B3 && + # + # gvfs-helper.exe writes downloaded objects to a shared-cache directory + # rather than the ODB inside the .git directory. + # + mkdir "$SHARED_CACHE_T1" && + mkdir "$SHARED_CACHE_T1/pack" && + mkdir "$SHARED_CACHE_T1/info" && + # + mkdir "$SHARED_CACHE_T2" && + mkdir "$SHARED_CACHE_T2/pack" && + mkdir "$SHARED_CACHE_T2/info" && + # + # setup repo_t1 and point all of the gvfs.* values to repo_src. + # + test_create_repo "$REPO_T1" && + git -C "$REPO_T1" branch -M main && + git -C "$REPO_T1" remote add origin $ORIGIN_URL && + git -C "$REPO_T1" config --local gvfs.cache-server $CACHE_URL && + git -C "$REPO_T1" config --local gvfs.sharedCache "$SHARED_CACHE_T1" && + echo "$SHARED_CACHE_T1" >> "$REPO_T1"/.git/objects/info/alternates && + # + test_create_repo "$REPO_T2" && + git -C "$REPO_T2" branch -M main && + git -C "$REPO_T2" remote add origin $ORIGIN_URL && + git -C "$REPO_T2" config --local gvfs.cache-server $CACHE_URL && + git -C "$REPO_T2" config --local gvfs.sharedCache "$SHARED_CACHE_T2" && + echo "$SHARED_CACHE_T2" >> "$REPO_T2"/.git/objects/info/alternates && + # + # + # + cat <<-EOF >creds.txt && + username=x + password=y + EOF + cat <<-EOF >creds.sh && + #!/bin/sh + cat "$(pwd)"/creds.txt + EOF + chmod 755 creds.sh && + git -C "$REPO_T1" config --local credential.helper "!f() { cat \"$(pwd)\"/creds.txt; }; f" && + git -C "$REPO_T2" config --local credential.helper "!f() { cat \"$(pwd)\"/creds.txt; }; f" && + # + # Create some test data sets. + # + get_list_of_oids 30 && + get_list_of_commit_and_tree_oids 30 && + get_list_of_blobs_oids && + get_one_commit_oid +' + +# Stop a gvfs-protocol server. +# Usage: stop_gvfs_protocol_server [] +# +# The optional port_increment (default 0) specifies which server to stop. +# Increment 0 uses the base port, 1 uses base+1, etc. +# +stop_gvfs_protocol_server () { + local instance="${1:-0}" + local pid_file=$(server_pid_file "$instance") + local log_file=$(server_log_file "$instance") + + if ! test -f "$pid_file" + then + return 0 + fi + # + # The server will shutdown automatically when we delete the pid-file. + # + rm -f "$pid_file" + # + # Give it a few seconds to shutdown (mainly to completely release the + # port before the next test start another instance and it attempts to + # bind to it). + # + for k in 0 1 2 3 4 + do + if grep -q "Starting graceful shutdown" "$log_file" + then + return 0 + fi + sleep 1 + done + + echo "stop_gvfs_protocol_server($instance): timeout waiting for server shutdown" + return 1 +} + +# Start a gvfs-protocol server. +# Usage: start_gvfs_protocol_server [] +# +# The optional port_increment (default 0) specifies which server to start. +# Increment 0 uses the base port, 1 uses base+1, etc. +# This allows running multiple servers simultaneously on different ports. +# +start_gvfs_protocol_server () { + local instance="${1:-0}" + local port=$(server_port "$instance") + local pid_file=$(server_pid_file "$instance") + local log_file=$(server_log_file "$instance") + # + # Launch our server into the background in repo_src. + # + ( + cd "$REPO_SRC" + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$port \ + --reuseaddr \ + --pid-file="$pid_file" \ + 2>"$log_file" & + ) + # + # Give it a few seconds to get started. + # + for k in 0 1 2 3 4 + do + if test -f "$pid_file" + then + return 0 + fi + sleep 1 + done + + echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" + return 1 +} + +start_gvfs_protocol_server_with_mayhem () { + if test $# -lt 1 + then + echo "start_gvfs_protocol_server_with_mayhem: need mayhem args" + return 1 + fi + + mayhem="" + for k in $* + do + mayhem="$mayhem --mayhem=$k" + done + # + # Launch our server into the background in repo_src. + # + ( + cd "$REPO_SRC" + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --reuseaddr \ + --pid-file="$PID_FILE" \ + $mayhem \ + 2>"$SERVER_LOG" & + ) + # + # Give it a few seconds to get started. + # + for k in 0 1 2 3 4 + do + if test -f "$PID_FILE" + then + return 0 + fi + sleep 1 + done + + echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" + return 1 +} + +# Verify that a server received at least one connection. +# Usage: verify_server_was_contacted [] +# +verify_server_was_contacted () { + local instance="${1:-0}" + local log_file=$(server_log_file "$instance") + grep -q "Connection from" "$log_file" +} + +# Verify that a server was NOT contacted. +# Usage: verify_server_was_not_contacted [] +# +verify_server_was_not_contacted () { + local instance="${1:-0}" + local log_file=$(server_log_file "$instance") + ! grep -q "Connection from" "$log_file" +} + +# Verify the number of connections from the client. +# +# If keep-alive is working, a series of successful sequential requests to the +# same server should use the same TCP connection, so a simple multi-get would +# only have one connection. +# +# On the other hand, an auto-retry after a network error (mayhem) will have +# more than one for a single object request. +# +# TODO This may generate false alarm when we get to complicated tests, so +# TODO we might only want to use it for basic tests. +# +verify_connection_count () { + if test $# -eq 1 + then + expected_nr=$1 + else + expected_nr=1 + fi + + actual_nr=$(grep -c "Connection from" "$SERVER_LOG") + + if test $actual_nr -ne $expected_nr + then + echo "verify_keep_live: expected $expected_nr; actual $actual_nr" + return 1 + fi + return 0 +} + +# Verify that the set of requested objects are present in +# the shared-cache and that there is no corruption. We use +# cat-file to hide whether the object is packed or loose in +# the test repo. +# +# Usage: +# +verify_objects_in_shared_cache () { + # + # See if any of the objects are missing from repo_t1. + # + git -C "$REPO_T1" cat-file --batch-check <"$1" >OUT.bc_actual || return 1 + test_grep " missing" OUT.bc_actual && return 1 + # + # See if any of the objects have different sizes or types than repo_src. + # + git -C "$REPO_SRC" cat-file --batch-check <"$1" >OUT.bc_expect || return 1 + test_cmp OUT.bc_expect OUT.bc_actual || return 1 + # + # See if any of the objects are corrupt in repo_t1. This fully + # reconstructs the objects and verifies the hash and therefore + # detects corruption not found by the earlier "batch-check" step. + # + git -C "$REPO_T1" cat-file --batch <"$1" >OUT.b_actual || return 1 + # + # TODO move the shared-cache directory (and/or the + # TODO .git/objects/info/alternates and temporarily unset + # TODO gvfs.sharedCache) and repeat the first "batch-check" + # TODO and make sure that they are ALL missing. + # + return 0 +} + +# gvfs-helper prints a "packfile " message for each received +# packfile to stdout. Verify that we received the expected number +# of packfiles. +# +verify_received_packfile_count () { + if test $# -eq 1 + then + expected_nr=$1 + else + expected_nr=1 + fi + + actual_nr=$(grep -c "packfile " ") to control how our mock server responds. For -# example, only the origin (main Git) server supports "/gvfs/config". -# -# For example, this means that if we add a remote containing $ORIGIN_URL, -# it will work with gvfs-helper, but not for fetch (without some mapping -# tricks). -# -HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT -ORIGIN_URL=http://$HOST_PORT/servertype/origin -CACHE_URL=http://$HOST_PORT/servertype/cache - -SHARED_CACHE_T1="$(pwd)"/shared_cache_t1 -SHARED_CACHE_T2="$(pwd)"/shared_cache_t2 - -# The pid-file is created by test-gvfs-protocol.exe when it starts. -# The server will shut down if/when we delete it. (This is a little -# easier than killing it by PID.) -# -PID_FILE="$(pwd)"/pid-file.pid -SERVER_LOG="$(pwd)"/OUT.server.log - -# Helper functions to compute port, pid-file, and log for a given -# port increment. An increment of 0 (or empty) uses the base values. -# -server_port () { - local instance="${1:-0}" - echo "$(($GIT_TEST_GVFS_PROTOCOL_PORT + $instance))" -} - -server_pid_file () { - local instance="${1:-0}" - if test "$instance" -eq 0 - then - echo "$PID_FILE" - else - echo "$(pwd)/pid-file-$instance.pid" - fi -} - -server_log_file () { - local instance="${1:-0}" - if test "$instance" -eq 0 - then - echo "$SERVER_LOG" - else - echo "$(pwd)/OUT.server-$instance.log" - fi -} - -# Helper to build a cache-server URL for a given port increment. -# -cache_server_url () { - local instance="${1:-0}" - local port=$(server_port "$instance") - echo "http://127.0.0.1:$port/servertype/cache" -} - -PATH="$GIT_BUILD_DIR/t/helper/:$PATH" && export PATH - -OIDS_FILE="$(pwd)"/oid_list.txt -OIDS_CT_FILE="$(pwd)"/oid_ct_list.txt -OIDS_BLOBS_FILE="$(pwd)"/oids_blobs_file.txt -OID_ONE_BLOB_FILE="$(pwd)"/oid_one_blob_file.txt -OID_ONE_COMMIT_FILE="$(pwd)"/oid_one_commit_file.txt - -# Get a list of available OIDs in repo_src so that we can try to fetch -# them and so that we don't have to hard-code a list of known OIDs. -# This doesn't need to be a complete list -- just enough to drive some -# representative tests. -# -# Optionally require that we find a minimum number of OIDs. -# -get_list_of_oids () { - git -C "$REPO_SRC" rev-list --objects HEAD | sed 's/ .*//' | sort >"$OIDS_FILE" - - if test $# -eq 1 - then - actual_nr=$(wc -l <"$OIDS_FILE") - if test $actual_nr -lt $1 - then - echo "get_list_of_oids: insufficient data. Need $1 OIDs." - return 1 - fi - fi - return 0 -} - -get_list_of_blobs_oids () { - git -C "$REPO_SRC" ls-tree HEAD | grep ' blob ' | awk "{print \$3}" | sort >"$OIDS_BLOBS_FILE" - head -1 <"$OIDS_BLOBS_FILE" >"$OID_ONE_BLOB_FILE" -} - -get_list_of_commit_and_tree_oids () { - git -C "$REPO_SRC" cat-file --batch-check --batch-all-objects | awk "/commit|tree/ {print \$1}" | sort >"$OIDS_CT_FILE" - - if test $# -eq 1 - then - actual_nr=$(wc -l <"$OIDS_CT_FILE") - if test $actual_nr -lt $1 - then - echo "get_list_of_commit_and_tree_oids: insufficient data. Need $1 OIDs." - return 1 - fi - fi - return 0 -} - -get_one_commit_oid () { - git -C "$REPO_SRC" rev-parse HEAD >"$OID_ONE_COMMIT_FILE" - return 0 -} - -# Create a commits-and-trees packfile for use with "prefetch" -# using the given range of commits. -# -create_commits_and_trees_packfile () { - if test $# -eq 2 - then - epoch=$1 - revs=$2 - else - echo "create_commits_and_trees_packfile: Need 2 args" - return 1 - fi - - pack_file="$REPO_SRC"/.git/objects/pack/ct-$epoch.pack - idx_file="$REPO_SRC"/.git/objects/pack/ct-$epoch.idx - - git -C "$REPO_SRC" pack-objects --stdout --revs --filter=blob:none \ - >"$pack_file" <<-EOF - $revs - EOF - git -C "$REPO_SRC" index-pack -o "$idx_file" "$pack_file" - return 0 -} - -test_expect_success 'setup repos' ' - test_create_repo "$REPO_SRC" && - git -C "$REPO_SRC" branch -M main && - # - # test_commit_bulk() does magic to create a packfile containing - # the new commits. - # - # We create branches in repo_src, but also remember the branch OIDs - # in files so that we can refer to them in repo_t1, which will not - # have the commits locally (because we do not clone or fetch). - # - test_commit_bulk -C "$REPO_SRC" --filename="batch_a.%s.t" 9 && - git -C "$REPO_SRC" branch B1 && - git -C "$REPO_SRC" rev-parse refs/heads/main >m1.branch && - # - test_commit_bulk -C "$REPO_SRC" --filename="batch_b.%s.t" 9 && - git -C "$REPO_SRC" branch B2 && - git -C "$REPO_SRC" rev-parse refs/heads/main >m2.branch && - # - # test_commit() creates commits, trees, tags, and blobs and leave - # them loose. - # - test_config gc.auto 0 && - # - test_commit -C "$REPO_SRC" file1.txt && - test_commit -C "$REPO_SRC" file2.txt && - test_commit -C "$REPO_SRC" file3.txt && - test_commit -C "$REPO_SRC" file4.txt && - test_commit -C "$REPO_SRC" file5.txt && - test_commit -C "$REPO_SRC" file6.txt && - test_commit -C "$REPO_SRC" file7.txt && - test_commit -C "$REPO_SRC" file8.txt && - test_commit -C "$REPO_SRC" file9.txt && - git -C "$REPO_SRC" branch B3 && - git -C "$REPO_SRC" rev-parse refs/heads/main >m3.branch && - # - # Create some commits-and-trees-only packfiles for testing prefetch. - # Set arbitrary EPOCH times to make it easier to test fetch-since. - # - create_commits_and_trees_packfile 1000000000 B1 && - create_commits_and_trees_packfile 1100000000 B1..B2 && - create_commits_and_trees_packfile 1200000000 B2..B3 && - # - # gvfs-helper.exe writes downloaded objects to a shared-cache directory - # rather than the ODB inside the .git directory. - # - mkdir "$SHARED_CACHE_T1" && - mkdir "$SHARED_CACHE_T1/pack" && - mkdir "$SHARED_CACHE_T1/info" && - # - mkdir "$SHARED_CACHE_T2" && - mkdir "$SHARED_CACHE_T2/pack" && - mkdir "$SHARED_CACHE_T2/info" && - # - # setup repo_t1 and point all of the gvfs.* values to repo_src. - # - test_create_repo "$REPO_T1" && - git -C "$REPO_T1" branch -M main && - git -C "$REPO_T1" remote add origin $ORIGIN_URL && - git -C "$REPO_T1" config --local gvfs.cache-server $CACHE_URL && - git -C "$REPO_T1" config --local gvfs.sharedCache "$SHARED_CACHE_T1" && - echo "$SHARED_CACHE_T1" >> "$REPO_T1"/.git/objects/info/alternates && - # - test_create_repo "$REPO_T2" && - git -C "$REPO_T2" branch -M main && - git -C "$REPO_T2" remote add origin $ORIGIN_URL && - git -C "$REPO_T2" config --local gvfs.cache-server $CACHE_URL && - git -C "$REPO_T2" config --local gvfs.sharedCache "$SHARED_CACHE_T2" && - echo "$SHARED_CACHE_T2" >> "$REPO_T2"/.git/objects/info/alternates && - # - # - # - cat <<-EOF >creds.txt && - username=x - password=y - EOF - cat <<-EOF >creds.sh && - #!/bin/sh - cat "$(pwd)"/creds.txt - EOF - chmod 755 creds.sh && - git -C "$REPO_T1" config --local credential.helper "!f() { cat \"$(pwd)\"/creds.txt; }; f" && - git -C "$REPO_T2" config --local credential.helper "!f() { cat \"$(pwd)\"/creds.txt; }; f" && - # - # Create some test data sets. - # - get_list_of_oids 30 && - get_list_of_commit_and_tree_oids 30 && - get_list_of_blobs_oids && - get_one_commit_oid -' - -# Stop a gvfs-protocol server. -# Usage: stop_gvfs_protocol_server [] -# -# The optional port_increment (default 0) specifies which server to stop. -# Increment 0 uses the base port, 1 uses base+1, etc. -# -stop_gvfs_protocol_server () { - local instance="${1:-0}" - local pid_file=$(server_pid_file "$instance") - local log_file=$(server_log_file "$instance") - - if ! test -f "$pid_file" - then - return 0 - fi - # - # The server will shutdown automatically when we delete the pid-file. - # - rm -f "$pid_file" - # - # Give it a few seconds to shutdown (mainly to completely release the - # port before the next test start another instance and it attempts to - # bind to it). - # - for k in 0 1 2 3 4 - do - if grep -q "Starting graceful shutdown" "$log_file" - then - return 0 - fi - sleep 1 - done - - echo "stop_gvfs_protocol_server($instance): timeout waiting for server shutdown" - return 1 -} - -# Start a gvfs-protocol server. -# Usage: start_gvfs_protocol_server [] -# -# The optional port_increment (default 0) specifies which server to start. -# Increment 0 uses the base port, 1 uses base+1, etc. -# This allows running multiple servers simultaneously on different ports. -# -start_gvfs_protocol_server () { - local instance="${1:-0}" - local port=$(server_port "$instance") - local pid_file=$(server_pid_file "$instance") - local log_file=$(server_log_file "$instance") - # - # Launch our server into the background in repo_src. - # - ( - cd "$REPO_SRC" - test-gvfs-protocol --verbose \ - --listen=127.0.0.1 \ - --port=$port \ - --reuseaddr \ - --pid-file="$pid_file" \ - 2>"$log_file" & - ) - # - # Give it a few seconds to get started. - # - for k in 0 1 2 3 4 - do - if test -f "$pid_file" - then - return 0 - fi - sleep 1 - done - - echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" - return 1 -} - -start_gvfs_protocol_server_with_mayhem () { - if test $# -lt 1 - then - echo "start_gvfs_protocol_server_with_mayhem: need mayhem args" - return 1 - fi - - mayhem="" - for k in $* - do - mayhem="$mayhem --mayhem=$k" - done - # - # Launch our server into the background in repo_src. - # - ( - cd "$REPO_SRC" - test-gvfs-protocol --verbose \ - --listen=127.0.0.1 \ - --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ - --reuseaddr \ - --pid-file="$PID_FILE" \ - $mayhem \ - 2>"$SERVER_LOG" & - ) - # - # Give it a few seconds to get started. - # - for k in 0 1 2 3 4 - do - if test -f "$PID_FILE" - then - return 0 - fi - sleep 1 - done - - echo "start_gvfs_protocol_server($instance): timeout waiting for server startup" - return 1 -} - -# Verify that a server received at least one connection. -# Usage: verify_server_was_contacted [] -# -verify_server_was_contacted () { - local instance="${1:-0}" - local log_file=$(server_log_file "$instance") - grep -q "Connection from" "$log_file" -} - -# Verify that a server was NOT contacted. -# Usage: verify_server_was_not_contacted [] -# -verify_server_was_not_contacted () { - local instance="${1:-0}" - local log_file=$(server_log_file "$instance") - ! grep -q "Connection from" "$log_file" -} - -# Verify the number of connections from the client. -# -# If keep-alive is working, a series of successful sequential requests to the -# same server should use the same TCP connection, so a simple multi-get would -# only have one connection. -# -# On the other hand, an auto-retry after a network error (mayhem) will have -# more than one for a single object request. -# -# TODO This may generate false alarm when we get to complicated tests, so -# TODO we might only want to use it for basic tests. -# -verify_connection_count () { - if test $# -eq 1 - then - expected_nr=$1 - else - expected_nr=1 - fi - - actual_nr=$(grep -c "Connection from" "$SERVER_LOG") - - if test $actual_nr -ne $expected_nr - then - echo "verify_keep_live: expected $expected_nr; actual $actual_nr" - return 1 - fi - return 0 -} - -# Verify that the set of requested objects are present in -# the shared-cache and that there is no corruption. We use -# cat-file to hide whether the object is packed or loose in -# the test repo. -# -# Usage: -# -verify_objects_in_shared_cache () { - # - # See if any of the objects are missing from repo_t1. - # - git -C "$REPO_T1" cat-file --batch-check <"$1" >OUT.bc_actual || return 1 - test_grep " missing" OUT.bc_actual && return 1 - # - # See if any of the objects have different sizes or types than repo_src. - # - git -C "$REPO_SRC" cat-file --batch-check <"$1" >OUT.bc_expect || return 1 - test_cmp OUT.bc_expect OUT.bc_actual || return 1 - # - # See if any of the objects are corrupt in repo_t1. This fully - # reconstructs the objects and verifies the hash and therefore - # detects corruption not found by the earlier "batch-check" step. - # - git -C "$REPO_T1" cat-file --batch <"$1" >OUT.b_actual || return 1 - # - # TODO move the shared-cache directory (and/or the - # TODO .git/objects/info/alternates and temporarily unset - # TODO gvfs.sharedCache) and repeat the first "batch-check" - # TODO and make sure that they are ALL missing. - # - return 0 -} - -# gvfs-helper prints a "packfile " message for each received -# packfile to stdout. Verify that we received the expected number -# of packfiles. -# -verify_received_packfile_count () { - if test $# -eq 1 - then - expected_nr=$1 - else - expected_nr=1 - fi - - actual_nr=$(grep -c "packfile " Date: Sat, 17 Jan 2026 21:02:41 -0500 Subject: [PATCH 8/8] t579*: split t5799 into several parts Move the tests from t5799-gvfs-helper.sh into multiple scripts that can run in parallel. To ensure that the ports do not overlap, add a large multiplier on the instance when needing multiple ports within the same test (currently limited to the verb-specific cache servers). Signed-off-by: Derrick Stolee --- t/lib-gvfs-helper.sh | 20 +- t/meson.build | 7 +- t/t5790-gvfs-helper-basic.sh | 338 +++++++ t/t5791-gvfs-helper-errors.sh | 353 ++++++++ t/t5792-gvfs-helper-auth.sh | 111 +++ t/t5793-gvfs-helper-integration.sh | 166 ++++ t/t5794-gvfs-helper-packfiles.sh | 197 ++++ t/t5795-gvfs-helper-verb-cache.sh | 224 +++++ t/t5799-gvfs-helper.sh | 1341 ---------------------------- 9 files changed, 1406 insertions(+), 1351 deletions(-) create mode 100755 t/t5790-gvfs-helper-basic.sh create mode 100755 t/t5791-gvfs-helper-errors.sh create mode 100755 t/t5792-gvfs-helper-auth.sh create mode 100755 t/t5793-gvfs-helper-integration.sh create mode 100755 t/t5794-gvfs-helper-packfiles.sh create mode 100755 t/t5795-gvfs-helper-verb-cache.sh delete mode 100755 t/t5799-gvfs-helper.sh diff --git a/t/lib-gvfs-helper.sh b/t/lib-gvfs-helper.sh index fac3b9210cb551..32c951361e3282 100644 --- a/t/lib-gvfs-helper.sh +++ b/t/lib-gvfs-helper.sh @@ -64,9 +64,11 @@ SERVER_LOG="$(pwd)"/OUT.server.log # Helper functions to compute port, pid-file, and log for a given # port increment. An increment of 0 (or empty) uses the base values. # +# Ensure we don't overlap with any other test port by modifying a +# significant bit. server_port () { local instance="${1:-0}" - echo $(($GIT_TEST_GVFS_PROTOCOL_PORT + "$instance")) + echo $(($GIT_TEST_GVFS_PROTOCOL_PORT + 10000 * $instance)) } server_pid_file () { @@ -93,7 +95,7 @@ server_log_file () { # cache_server_url () { local instance="${1:-0}" - local port=$(server_port "$instance") + local port="$(server_port "$instance")" echo "http://127.0.0.1:$port/servertype/cache" } @@ -276,8 +278,8 @@ test_expect_success 'setup repos' ' # stop_gvfs_protocol_server () { local instance="${1:-0}" - local pid_file=$(server_pid_file "$instance") - local log_file=$(server_log_file "$instance") + local pid_file="$(server_pid_file "$instance")" + local log_file="$(server_log_file "$instance")" if ! test -f "$pid_file" then @@ -314,9 +316,9 @@ stop_gvfs_protocol_server () { # start_gvfs_protocol_server () { local instance="${1:-0}" - local port=$(server_port "$instance") - local pid_file=$(server_pid_file "$instance") - local log_file=$(server_log_file "$instance") + local port="$(server_port "$instance")" + local pid_file="$(server_pid_file "$instance")" + local log_file="$(server_log_file "$instance")" # # Launch our server into the background in repo_src. # @@ -391,7 +393,7 @@ start_gvfs_protocol_server_with_mayhem () { # verify_server_was_contacted () { local instance="${1:-0}" - local log_file=$(server_log_file "$instance") + local log_file="$(server_log_file "$instance")" grep -q "Connection from" "$log_file" } @@ -400,7 +402,7 @@ verify_server_was_contacted () { # verify_server_was_not_contacted () { local instance="${1:-0}" - local log_file=$(server_log_file "$instance") + local log_file="$(server_log_file "$instance")" ! grep -q "Connection from" "$log_file" } diff --git a/t/meson.build b/t/meson.build index b57583e9c53964..c7447d2d35b07c 100644 --- a/t/meson.build +++ b/t/meson.build @@ -741,7 +741,12 @@ integration_tests = [ 't5731-protocol-v2-bundle-uri-git.sh', 't5732-protocol-v2-bundle-uri-http.sh', 't5750-bundle-uri-parse.sh', - 't5799-gvfs-helper.sh', + 't5790-gvfs-helper-basic.sh', + 't5791-gvfs-helper-errors.sh', + 't5792-gvfs-helper-auth.sh', + 't5793-gvfs-helper-integration.sh', + 't5794-gvfs-helper-packfiles.sh', + 't5795-gvfs-helper-verb-cache.sh', 't5801-remote-helpers.sh', 't5802-connect-helper.sh', 't5810-proto-disable-local.sh', diff --git a/t/t5790-gvfs-helper-basic.sh b/t/t5790-gvfs-helper-basic.sh new file mode 100755 index 00000000000000..093c809dcccaa1 --- /dev/null +++ b/t/t5790-gvfs-helper-basic.sh @@ -0,0 +1,338 @@ +#!/bin/sh + +test_description='gvfs-helper basic tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Basic tests to confirm the happy path works. +################################################################# + +test_expect_success 'basic: GET origin multi-get no-auth' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OIDS_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OIDS_FILE" && + verify_connection_count 1 +' + +test_expect_success 'basic: GET cache-server multi-get trust-mode' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the cache-server and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OIDS_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OIDS_FILE" && + verify_connection_count 1 +' + +test_expect_success 'basic: GET gvfs/config' ' +# test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the cache-server and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + config \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # The cache-server URL should be listed in the gvfs/config output. + # We confirm this before assuming error-mode will work. + # + test_grep "$CACHE_URL" OUT.output +' + +test_expect_success 'basic: GET cache-server multi-get error-mode' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the cache-server and make a series of + # single-object GET requests. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=error \ + --remote=origin \ + get \ + <"$OIDS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OIDS_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OIDS_FILE" && + + # Technically, we have 1 connection to the origin server + # for the "gvfs/config" request and 1 to cache server to + # get the objects, but because we are using the same port + # for both, keep-alive will handle it. So 1 connection. + # + verify_connection_count 1 +' + +# The GVFS Protocol POST verb behaves like GET for non-commit objects +# (in that it just returns the requested object), but for commit +# objects POST *also* returns all trees referenced by the commit. +# +# The goal of this test is to confirm that gvfs-helper can send us +# a packfile at all. So, this test only passes blobs to not blur +# the issue. +# +test_expect_success 'basic: POST origin blobs' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and make + # multi-object POST request. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "packfile " message for each received + # packfile. We verify the number of expected packfile(s) and we + # individually verify that each requested object is present in the + # shared cache (and index-pack already verified the integrity of + # the packfile), so we do not bother to run "git verify-pack -v" + # and do an exact matchup here. + # + verify_received_packfile_count 1 && + + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + verify_connection_count 1 +' + +# Request a single blob via POST. Per the GVFS Protocol, the server +# should implicitly send a loose object for it. Confirm that. +# +test_expect_success 'basic: POST-request a single blob' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and request a single + # blob via POST. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received + # loose object. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_connection_count 1 +' + +# Request a single commit via POST. Per the GVFS Protocol, the server +# should implicitly send us a packfile containing the commit and the +# trees it references. Confirm that properly handled the receipt of +# the packfile. (Here, we are testing that asking for a single commit +# via POST yields a packfile rather than a loose object.) +# +# We DO NOT verify that the packfile contains commits/trees and no blobs +# because our test helper doesn't implement the filtering. +# +test_expect_success 'basic: POST-request a single commit' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Connect to the origin server (w/o auth) and request a single + # commit via POST. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OID_ONE_COMMIT_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 1 && + + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH w/o arg gets all' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Without a "since" argument gives us all "ct-*.pack" since the EPOCH + # because we do not have any prefetch packs locally. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 3 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH w/ arg' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Ask for cached packfiles NEWER THAN the given time. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH mayhem no_prefetch_idx' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem no_prefetch_idx && + + # Request prefetch packs, but tell server to not send any + # idx files and force gvfs-helper to compute them. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 1 +' + +test_expect_success 'basic: PREFETCH up-to-date' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Ask for cached packfiles NEWER THAN the given time. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + # Ask again for any packfiles newer than what we have cached locally. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output && + + # gvfs-helper prints a "packfile " message for each received + # packfile. + # + verify_received_packfile_count 0 && + verify_prefetch_keeps 1200000000 && + + stop_gvfs_protocol_server && + verify_connection_count 2 +' + +test_done diff --git a/t/t5791-gvfs-helper-errors.sh b/t/t5791-gvfs-helper-errors.sh new file mode 100755 index 00000000000000..189950c097253b --- /dev/null +++ b/t/t5791-gvfs-helper-errors.sh @@ -0,0 +1,353 @@ +#!/bin/sh + +test_description='gvfs-helper error handling tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Tests to see how gvfs-helper responds to network problems. +# +# We use small --max-retry value because of exponential backoff. +# +# These mayhem tests are interested in how gvfs-helper gracefully +# retries when there is a network error. And verify that it gives +# up gracefully too. +################################################################# + +mayhem_observed__close__connections () { + if grep "transient" OUT.stderr + then + # Transient errors should retry. + # 1 for initial request + 2 retries. + # + verify_connection_count 3 + return $? + elif grep "hard_fail" OUT.stderr + then + # Hard errors should not retry. + # + verify_connection_count 1 + return $? + else + error "mayhem_observed__close: unexpected mayhem-induced error type" + return 1 + fi +} + +mayhem_observed__close () { + # Expected error codes for mayhem events: + # close_read + # close_write + # close_no_write + # + # CURLE_PARTIAL_FILE 18 + # CURLE_GOT_NOTHING 52 + # CURLE_SEND_ERROR 55 + # CURLE_RECV_ERROR 56 + # + # I don't want to pin it down to an exact error for each because there may + # be races here because of network buffering. + # + # Also, It is unclear which of these network errors should be transient + # (with retry) and which should be a hard-fail (without retry). I'm only + # going to verify the connection counts based upon what type of error + # gvfs-helper claimed it to be. + # + if grep "error: get: (curl:18)" OUT.stderr || + grep "error: get: (curl:52)" OUT.stderr || + grep "error: get: (curl:55)" OUT.stderr || + grep "error: get: (curl:56)" OUT.stderr + then + mayhem_observed__close__connections + return $? + else + echo "mayhem_observed__close: unexpected mayhem-induced error" + return 1 + fi +} + +test_lazy_prereq CURL_8_16_0 ' + git gvfs-helper curl-version = 8.16.0 || + test 8.15.0-DEV = "$(git gvfs-helper curl-version)" +' + +test_expect_success 'curl-error: no server' ' + test_when_finished "per_test_cleanup" && + + connect_timeout_ms= && + # CURLE_COULDNT_CONNECT 7 + regex="error: get: (curl:7)" && + if test_have_prereq CURL_8_16_0 + then + connect_timeout_ms=--connect-timeout-ms=200 && + # CURLE_COULDNT_CONNECT 7 + # CURLE_OPERATION_TIMEDOUT 28 + regex="error: get: (curl:\(7\|28\))" + fi && + + # Try to do a multi-get without a server. + # + # Use small max-retry value because of exponential backoff, + # but yet do exercise retry some. + # + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + $connect_timeout_ms \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + test_grep "$regex" OUT.stderr +' + +test_expect_success 'curl-error: close socket while reading request' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_read && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + mayhem_observed__close +' + +test_expect_success 'curl-error: close socket while writing response' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_write && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + mayhem_observed__close +' + +test_expect_success 'curl-error: close socket before writing response' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_no_write && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + mayhem_observed__close +' + +################################################################# +# Tests to confirm that gvfs-helper does silently recover when +# a retry succeeds. +# +# Note: I'm only to do this for 1 of the close_* mayhem events. +################################################################# + +test_expect_success 'successful retry after curl-error: origin get' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem close_read_1 && + + # Connect to the origin server (w/o auth). + # Make a single-object GET request. + # Confirm that it succeeds without error. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + verify_connection_count 2 +' + +################################################################# +# Tests to see how gvfs-helper responds to HTTP errors/problems. +# +################################################################# + +# See "enum gh__error_code" in gvfs-helper.c +# +GH__ERROR_CODE__HTTP_404=4 +GH__ERROR_CODE__HTTP_429=5 +GH__ERROR_CODE__HTTP_503=6 + +test_expect_success 'http-error: 503 Service Unavailable (with retry)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_503 && + + test_expect_code $GH__ERROR_CODE__HTTP_503 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:503)" OUT.stderr && + verify_connection_count 3 +' + +test_expect_success 'http-error: 429 Service Unavailable (with retry)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_429 && + + test_expect_code $GH__ERROR_CODE__HTTP_429 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:429)" OUT.stderr && + verify_connection_count 3 +' + +test_expect_success 'http-error: 404 Not Found (no retry)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_404 && + + test_expect_code $GH__ERROR_CODE__HTTP_404 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:404)" OUT.stderr && + verify_connection_count 1 +' + +################################################################# +# Tests to confirm that gvfs-helper does silently recover when an +# HTTP request succeeds after a failure. +# +################################################################# + +test_expect_success 'successful retry after http-error: origin get' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_429_1 && + + # Connect to the origin server (w/o auth). + # Make a single-object GET request. + # Confirm that it succeeds without error. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + verify_connection_count 2 +' + +################################################################# +# So far we have confirmed that gvfs-helper can recover from a network +# error (with retries, since the cache-server was disabled in all of +# the above tests). Try again with fallback turned on. +# +# With mayhem "http_503" turned on both the cache and origin server +# will always throw a 503 error. +# +# Confirm that we tried to make six connections: we should hit the +# cache-server 3 times (one initial attempt and two retries) and then +# try the origin server 3 times. +# +################################################################# + +test_expect_success 'http-error: 503 Service Unavailable (with retry and fallback)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_503 && + + test_expect_code $GH__ERROR_CODE__HTTP_503 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --fallback \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:503)" OUT.stderr && + verify_connection_count 6 +' + +################################################################# +# Now repeat the above, but explicitly turn off fallback. +# +# Again, we use mayhem "http_503". However, with fallback turned +# off, we will only attempt the 3 connections to the cache server. +# We will not try to hit the origin server. +# +# So we should only see a total of 3 connections rather than the +# six in the previous test. +# +################################################################# + +test_expect_success 'http-error: 503 Service Unavailable (with retry and no-fallback)' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_503 && + + test_expect_code $GH__ERROR_CODE__HTTP_503 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-fallback \ + get \ + --max-retries=2 \ + <"$OIDS_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + test_grep "error: get: (http:503)" OUT.stderr && + verify_connection_count 3 +' + +test_done diff --git a/t/t5792-gvfs-helper-auth.sh b/t/t5792-gvfs-helper-auth.sh new file mode 100755 index 00000000000000..0aa9188664652f --- /dev/null +++ b/t/t5792-gvfs-helper-auth.sh @@ -0,0 +1,111 @@ +#!/bin/sh + +test_description='gvfs-helper authentication tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Test HTTP Auth +# +################################################################# + +test_lazy_prereq CURL_7_75_OR_NEWER ' + git gvfs-helper curl-version ">=" 7.75.0 +' + +test_expect_success 'HTTP GET Auth on Origin Server' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401 && + + # Force server to require auth. + # Connect to the origin server without auth. + # Make a single-object GET request. + # Confirm that it gets a 401 and then retries with auth. + # + GIT_CONFIG_NOSYSTEM=1 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + if test_have_prereq CURL_7_75_OR_NEWER + then + verify_connection_count 2 + fi +' + +test_expect_success 'HTTP POST Auth on Origin Server' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401 && + + # Connect to the origin server and make multi-object POST + # request and verify that it automatically handles the 401. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output && + + # Stop the server to prevent the verification steps from faulting-in + # any missing objects. + # + stop_gvfs_protocol_server && + + # gvfs-helper prints a "packfile " message for each received + # packfile. We verify the number of expected packfile(s) and we + # individually verify that each requested object is present in the + # shared cache (and index-pack already verified the integrity of + # the packfile), so we do not bother to run "git verify-pack -v" + # and do an exact matchup here. + # + verify_received_packfile_count 1 && + + verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && + verify_connection_count 2 +' + +test_expect_success 'HTTP GET Auth on Cache Server' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem http_401 && + + # Try auth to cache-server. Note that gvfs-helper *ALWAYS* sends + # creds to cache-servers, so we will never see the "400 Bad Request" + # response. And we are using "trust" mode, so we only expect 1 + # connection to the server. + # + GIT_CONFIG_NOSYSTEM=1 \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + --max-retries=2 \ + <"$OID_ONE_BLOB_FILE" >OUT.output && + + stop_gvfs_protocol_server && + + # gvfs-helper prints a "loose " message for each received object. + # Verify that gvfs-helper received each of the requested objects. + # + sed "s/loose //" OUT.actual && + test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && + + verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + verify_connection_count 1 +' + +test_done diff --git a/t/t5793-gvfs-helper-integration.sh b/t/t5793-gvfs-helper-integration.sh new file mode 100755 index 00000000000000..c74ab74dd4b410 --- /dev/null +++ b/t/t5793-gvfs-helper-integration.sh @@ -0,0 +1,166 @@ +#!/bin/sh + +test_description='gvfs-helper integration tests with Git commands' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Integration tests with Git.exe +# +# Now that we have confirmed that gvfs-helper works in isolation, +# run a series of tests using random Git commands that fault-in +# objects as needed. +# +# At this point, I'm going to stop verifying the shape of the ODB +# (loose vs packfiles) and the number of connections required to +# get them. The tests from here on are to verify that objects are +# magically fetched whenever required. +################################################################# + +test_expect_success 'integration: explicit commit/trees, implicit blobs: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # We have a very empty repo. Seed it with all of the commits + # and trees. The purpose of this test is to demand-load the + # needed blobs only, so we prefetch the commits and trees. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + get \ + <"$OIDS_CT_FILE" >OUT.output && + + # Confirm that we do not have the blobs locally. + # With gvfs-helper turned off, we should fail. + # + test_must_fail \ + git -C "$REPO_T1" -c core.useGVFSHelper=false \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + # Turn on gvfs-helper and retry. This should implicitly fetch + # any needed blobs. + # + git -C "$REPO_T1" -c core.useGVFSHelper=true \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + # Verify that gvfs-helper wrote the fetched the blobs to the + # local ODB, such that a second attempt with gvfs-helper + # turned off should succeed. + # + git -C "$REPO_T1" -c core.useGVFSHelper=false \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr +' + +trace_has_queue_oid () { + oid=$1 + grep "gh_client__queue_oid: $oid" +} + +trace_has_immediate_oid () { + oid=$1 + grep "gh_client__get_immediate: $oid" +} + +test_expect_success 'integration: fully implicit: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + # Implicitly demand-load everything without any pre-seeding. + # + GIT_TRACE2_EVENT="$(pwd)/diff-trace.txt" \ + git -C "$REPO_T1" -c core.useGVFSHelper=true \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + oid=$(git -C "$REPO_SRC" rev-parse main:file9.txt.t) && + trace_has_queue_oid $oid OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +test_expect_success 'integration: implicit-get: cache_http_503,no-fallback: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + + # Tell cache server to send 503 and origin server to send 200. + start_gvfs_protocol_server_with_mayhem cache_http_503 && + + # Implicitly demand-load everything without any pre-seeding. + # This should fail because we do not allow fallback. + # + test_must_fail \ + git -C "$REPO_T2" \ + -c core.useGVFSHelper=true \ + -c gvfs.fallback=false \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +test_expect_success 'integration: implicit-get: cache_http_503,with-fallback: diff 2 commits' ' + test_when_finished "per_test_cleanup" && + + # Tell cache server to send 503 and origin server to send 200. + start_gvfs_protocol_server_with_mayhem cache_http_503 && + + # Implicitly demand-load everything without any pre-seeding. + # + git -C "$REPO_T2" \ + -c core.useGVFSHelper=true \ + -c gvfs.fallback=true \ + diff $(cat m1.branch)..$(cat m3.branch) \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +# T2 should be considered contaminated at this point. + +test_done diff --git a/t/t5794-gvfs-helper-packfiles.sh b/t/t5794-gvfs-helper-packfiles.sh new file mode 100755 index 00000000000000..9b5aba8aa04b9c --- /dev/null +++ b/t/t5794-gvfs-helper-packfiles.sh @@ -0,0 +1,197 @@ +#!/bin/sh + +test_description='gvfs-helper packfile handling tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Duplicate packfile tests. +# +# If we request a fixed set of blobs, we should get a unique packfile +# of the form "vfs-.{pack,idx}". It we request that same set +# again, the server should create and send the exact same packfile. +# True web servers might build the custom packfile in random order, +# but our test web server should give us consistent results. +# +# Verify that we can handle the duplicate pack and idx file properly. +################################################################# + +first_received_packfile_pathname () { + sed -n "s/packfile //p" OUT.output 2>OUT.stderr && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + # Re-fetch the same packfile. We do not care if it replaces + # first one or if it silently fails to overwrite the existing + # one. We just confirm that afterwards we only have 1 packfile. + # + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + stop_gvfs_protocol_server +' + +test_expect_success 'duplicate and busy: vfs- packfile' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" \ + >OUT.output \ + 2>OUT.stderr && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + # Re-fetch the same packfile, but hold the existing packfile + # open for writing on an obscure (and randomly-chosen) file + # descriptor. + # + # This should cause the replacement-install to fail (at least + # on Windows) with an EBUSY or EPERM or something. + # + # Verify that that error is eaten. We do not care if the + # replacement is retried or if gvfs-helper simply discards the + # second instance. We just confirm that afterwards we only + # have 1 packfile on disk and that the command "lies" and reports + # that it created the existing packfile. (We want the lie because + # in normal usage, gh-client has already built the packed-git list + # in memory and is using gvfs-helper to fetch missing objects; + # gh-client does not care who does the fetch, but it needs to + # update its packed-git list and restart the object lookup.) + # + PACK=$(first_received_packfile_pathname) && + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" \ + >OUT.output \ + 2>OUT.stderr \ + 9>>"$PACK" && + verify_received_packfile_count 1 && + verify_vfs_packfile_count 1 && + + stop_gvfs_protocol_server +' + +################################################################# +# Ensure that the SHA of the blob we received matches the SHA of +# the blob we requested. +################################################################# + +# Request a loose blob from the server. Verify that we received +# content matches the requested SHA. +# +test_expect_success 'catch corrupted loose object' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem corrupt_loose && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + # Verify corruption detected. + # Verify valid blob not included in response to client. + + grep "hash failed for received loose object" OUT.stderr && + + # Verify that we did not write the corrupted blob to the ODB. + + ! verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && + git -C "$REPO_T1" fsck +' + +################################################################# +# Ensure that we can detect when we receive a corrupted packfile +# from the server. This is not concerned with network IO errors, +# but rather cases when the cache or origin server generates or +# sends an invalid packfile. +# +# For example, if the server throws an exception and writes the +# stack trace to the socket rather than or in addition to the +# packfile content. +# +# Or for example, if the packfile on the server's disk is corrupt +# and it sends it correctly, but the original data was already +# garbage, so the client still has garbage (and retrying won't +# help). +################################################################# + +# Send corrupt PACK files w/o IDX files (so that `gvfs-helper` +# must use `index-pack` to create it. (And as a side-effect, +# validate the PACK file is not corrupt.) +test_expect_success 'prefetch corrupt pack without idx' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem \ + bad_prefetch_pack_sha \ + no_prefetch_idx && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch \ + --max-retries=0 \ + --since="1000000000" \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server && + + # Verify corruption detected in pack when building + # local idx file for it. + + test_grep "error: .* index-pack failed" OUT.stderr +' + +# Send corrupt PACK files with IDX files. Since the cache server +# sends both, `gvfs-helper` might fail to verify both of them. +test_expect_success 'prefetch corrupt pack with corrupt idx' ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem \ + bad_prefetch_pack_sha && + + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch \ + --max-retries=0 \ + --since="1000000000" \ + >OUT.output 2>OUT.stderr && + + stop_gvfs_protocol_server +' + +test_done diff --git a/t/t5795-gvfs-helper-verb-cache.sh b/t/t5795-gvfs-helper-verb-cache.sh new file mode 100755 index 00000000000000..653deeedd5554d --- /dev/null +++ b/t/t5795-gvfs-helper-verb-cache.sh @@ -0,0 +1,224 @@ +#!/bin/sh + +test_description='gvfs-helper verb-specific cache-server tests' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +################################################################# +# Tests for gvfs..cache-server config. +# +# These tests verify that verb-specific cache-server overrides work +# correctly. We run two servers on different ports: +# - Server 0 (base port): configured as gvfs.cache-server (default) +# - Server 1 (base port + 1): configured as gvfs..cache-server +# +# For each verb (prefetch, get, post), we verify that: +# 1. When using the verb-specific override, the request goes to server 1 +# 2. When using a different verb, the request goes to server 0 +################################################################# + +test_expect_success 'verb-specific cache-server: prefetch uses gvfs.prefetch.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for prefetch. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + + # Run prefetch - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (prefetch-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for prefetch. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 0 (default), not server 1 (prefetch). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (prefetch-specific). + verify_server_was_not_contacted 1 +' + +test_expect_success 'verb-specific cache-server: get uses gvfs.get.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for get. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (get-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for get. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && + + # Run prefetch - should go to server 0 (default), not server 1 (get). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (get-specific). + verify_server_was_not_contacted 1 +' + +test_expect_success 'verb-specific cache-server: post uses gvfs.post.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for post. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && + + # Run post - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 1 was contacted (post-specific). + verify_server_was_contacted 1 && + + # Verify server 0 was NOT contacted. + verify_server_was_not_contacted 0 +' + +test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cache-server' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + + # Configure server 0 as default cache-server and server 1 for post. + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && + + # Run get - should go to server 0 (default), not server 1 (post). + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + + # Verify server 0 was contacted (default cache-server). + verify_server_was_contacted 0 && + + # Verify server 1 was NOT contacted (post-specific). + verify_server_was_not_contacted 1 +' + +test_expect_success 'verb-specific cache-server: all verbs with different servers' ' + test_when_finished "per_test_cleanup" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && + test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && + start_gvfs_protocol_server 0 && + start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 3 && + + # Configure each verb to use a different server: + # - server 0: default (unused in this test) + # - server 1: prefetch + # - server 2: get + # - server 3: post + git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && + git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && + git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 2)" && + git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 3)" && + + # Run prefetch - should go to server 1. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + verify_server_was_contacted 1 && + verify_server_was_not_contacted 0 && + verify_server_was_not_contacted 2 && + verify_server_was_not_contacted 3 && + + # Clean up shared cache for next verb. + rm -rf "$SHARED_CACHE_T1"/pack/* && + + # Run get - should go to server 2. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + get \ + <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && + verify_server_was_contacted 2 && + + # Clean up shared cache for next verb. + rm -rf "$SHARED_CACHE_T1"/[0-9a-f][0-9a-f]/ && + rm -rf "$SHARED_CACHE_T1"/pack/* && + + # Run post - should go to server 3. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=trust \ + --remote=origin \ + --no-progress \ + post \ + <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && + verify_server_was_contacted 3 +' + +test_done diff --git a/t/t5799-gvfs-helper.sh b/t/t5799-gvfs-helper.sh deleted file mode 100755 index 10146cb9811278..00000000000000 --- a/t/t5799-gvfs-helper.sh +++ /dev/null @@ -1,1341 +0,0 @@ -#!/bin/sh - -test_description='test gvfs-helper and GVFS Protocol' - -. ./test-lib.sh - -. "$TEST_DIRECTORY"/lib-gvfs-helper.sh - -################################################################# -# Basic tests to confirm the happy path works. -################################################################# - -test_expect_success 'basic: GET origin multi-get no-auth' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the origin server (w/o auth) and make a series of - # single-object GET requests. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - <"$OIDS_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OIDS_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OIDS_FILE" && - verify_connection_count 1 -' - -test_expect_success 'basic: GET cache-server multi-get trust-mode' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the cache-server and make a series of - # single-object GET requests. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - <"$OIDS_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OIDS_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OIDS_FILE" && - verify_connection_count 1 -' - -test_expect_success 'basic: GET gvfs/config' ' -# test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the cache-server and make a series of - # single-object GET requests. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - config \ - <"$OIDS_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # The cache-server URL should be listed in the gvfs/config output. - # We confirm this before assuming error-mode will work. - # - test_grep "$CACHE_URL" OUT.output -' - -test_expect_success 'basic: GET cache-server multi-get error-mode' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the cache-server and make a series of - # single-object GET requests. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=error \ - --remote=origin \ - get \ - <"$OIDS_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OIDS_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OIDS_FILE" && - - # Technically, we have 1 connection to the origin server - # for the "gvfs/config" request and 1 to cache server to - # get the objects, but because we are using the same port - # for both, keep-alive will handle it. So 1 connection. - # - verify_connection_count 1 -' - -# The GVFS Protocol POST verb behaves like GET for non-commit objects -# (in that it just returns the requested object), but for commit -# objects POST *also* returns all trees referenced by the commit. -# -# The goal of this test is to confirm that gvfs-helper can send us -# a packfile at all. So, this test only passes blobs to not blur -# the issue. -# -test_expect_success 'basic: POST origin blobs' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the origin server (w/o auth) and make - # multi-object POST request. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "packfile " message for each received - # packfile. We verify the number of expected packfile(s) and we - # individually verify that each requested object is present in the - # shared cache (and index-pack already verified the integrity of - # the packfile), so we do not bother to run "git verify-pack -v" - # and do an exact matchup here. - # - verify_received_packfile_count 1 && - - verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && - verify_connection_count 1 -' - -# Request a single blob via POST. Per the GVFS Protocol, the server -# should implicitly send a loose object for it. Confirm that. -# -test_expect_success 'basic: POST-request a single blob' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the origin server (w/o auth) and request a single - # blob via POST. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OID_ONE_BLOB_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received - # loose object. - # - sed "s/loose //" OUT.actual && - test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && - - verify_connection_count 1 -' - -# Request a single commit via POST. Per the GVFS Protocol, the server -# should implicitly send us a packfile containing the commit and the -# trees it references. Confirm that properly handled the receipt of -# the packfile. (Here, we are testing that asking for a single commit -# via POST yields a packfile rather than a loose object.) -# -# We DO NOT verify that the packfile contains commits/trees and no blobs -# because our test helper doesn't implement the filtering. -# -test_expect_success 'basic: POST-request a single commit' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Connect to the origin server (w/o auth) and request a single - # commit via POST. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OID_ONE_COMMIT_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "packfile " message for each received - # packfile. - # - verify_received_packfile_count 1 && - - verify_connection_count 1 -' - -test_expect_success 'basic: PREFETCH w/o arg gets all' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Without a "since" argument gives us all "ct-*.pack" since the EPOCH - # because we do not have any prefetch packs locally. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch >OUT.output && - - # gvfs-helper prints a "packfile " message for each received - # packfile. - # - verify_received_packfile_count 3 && - verify_prefetch_keeps 1200000000 && - - stop_gvfs_protocol_server && - verify_connection_count 1 -' - -test_expect_success 'basic: PREFETCH w/ arg' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Ask for cached packfiles NEWER THAN the given time. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch --since="1000000000" >OUT.output && - - # gvfs-helper prints a "packfile " message for each received - # packfile. - # - verify_received_packfile_count 2 && - verify_prefetch_keeps 1200000000 && - - stop_gvfs_protocol_server && - verify_connection_count 1 -' - -test_expect_success 'basic: PREFETCH mayhem no_prefetch_idx' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem no_prefetch_idx && - - # Request prefetch packs, but tell server to not send any - # idx files and force gvfs-helper to compute them. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch --since="1000000000" >OUT.output && - - # gvfs-helper prints a "packfile " message for each received - # packfile. - # - verify_received_packfile_count 2 && - verify_prefetch_keeps 1200000000 && - - stop_gvfs_protocol_server && - verify_connection_count 1 -' - -test_expect_success 'basic: PREFETCH up-to-date' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Ask for cached packfiles NEWER THAN the given time. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch --since="1000000000" >OUT.output && - - # gvfs-helper prints a "packfile " message for each received - # packfile. - # - verify_received_packfile_count 2 && - verify_prefetch_keeps 1200000000 && - - # Ask again for any packfiles newer than what we have cached locally. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch >OUT.output && - - # gvfs-helper prints a "packfile " message for each received - # packfile. - # - verify_received_packfile_count 0 && - verify_prefetch_keeps 1200000000 && - - stop_gvfs_protocol_server && - verify_connection_count 2 -' - -################################################################# -# Tests to see how gvfs-helper responds to network problems. -# -# We use small --max-retry value because of exponential backoff. -# -# These mayhem tests are interested in how gvfs-helper gracefully -# retries when there is a network error. And verify that it gives -# up gracefully too. -################################################################# - -mayhem_observed__close__connections () { - if grep "transient" OUT.stderr - then - # Transient errors should retry. - # 1 for initial request + 2 retries. - # - verify_connection_count 3 - return $? - elif grep "hard_fail" OUT.stderr - then - # Hard errors should not retry. - # - verify_connection_count 1 - return $? - else - error "mayhem_observed__close: unexpected mayhem-induced error type" - return 1 - fi -} - -mayhem_observed__close () { - # Expected error codes for mayhem events: - # close_read - # close_write - # close_no_write - # - # CURLE_PARTIAL_FILE 18 - # CURLE_GOT_NOTHING 52 - # CURLE_SEND_ERROR 55 - # CURLE_RECV_ERROR 56 - # - # I don't want to pin it down to an exact error for each because there may - # be races here because of network buffering. - # - # Also, It is unclear which of these network errors should be transient - # (with retry) and which should be a hard-fail (without retry). I'm only - # going to verify the connection counts based upon what type of error - # gvfs-helper claimed it to be. - # - if grep "error: get: (curl:18)" OUT.stderr || - grep "error: get: (curl:52)" OUT.stderr || - grep "error: get: (curl:55)" OUT.stderr || - grep "error: get: (curl:56)" OUT.stderr - then - mayhem_observed__close__connections - return $? - else - echo "mayhem_observed__close: unexpected mayhem-induced error" - return 1 - fi -} - -test_lazy_prereq CURL_8_16_0 ' - git gvfs-helper curl-version = 8.16.0 || - test 8.15.0-DEV = "$(git gvfs-helper curl-version)" -' - -test_expect_success 'curl-error: no server' ' - test_when_finished "per_test_cleanup" && - - connect_timeout_ms= && - # CURLE_COULDNT_CONNECT 7 - regex="error: get: (curl:7)" && - if test_have_prereq CURL_8_16_0 - then - connect_timeout_ms=--connect-timeout-ms=200 && - # CURLE_COULDNT_CONNECT 7 - # CURLE_OPERATION_TIMEDOUT 28 - regex="error: get: (curl:\(7\|28\))" - fi && - - # Try to do a multi-get without a server. - # - # Use small max-retry value because of exponential backoff, - # but yet do exercise retry some. - # - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - $connect_timeout_ms \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - test_grep "$regex" OUT.stderr -' - -test_expect_success 'curl-error: close socket while reading request' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem close_read && - - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - mayhem_observed__close -' - -test_expect_success 'curl-error: close socket while writing response' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem close_write && - - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - mayhem_observed__close -' - -test_expect_success 'curl-error: close socket before writing response' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem close_no_write && - - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - mayhem_observed__close -' - -################################################################# -# Tests to confirm that gvfs-helper does silently recover when -# a retry succeeds. -# -# Note: I'm only to do this for 1 of the close_* mayhem events. -################################################################# - -test_expect_success 'successful retry after curl-error: origin get' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem close_read_1 && - - # Connect to the origin server (w/o auth). - # Make a single-object GET request. - # Confirm that it succeeds without error. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OID_ONE_BLOB_FILE" >OUT.output && - - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && - verify_connection_count 2 -' - -################################################################# -# Tests to see how gvfs-helper responds to HTTP errors/problems. -# -################################################################# - -# See "enum gh__error_code" in gvfs-helper.c -# -GH__ERROR_CODE__HTTP_404=4 -GH__ERROR_CODE__HTTP_429=5 -GH__ERROR_CODE__HTTP_503=6 - -test_expect_success 'http-error: 503 Service Unavailable (with retry)' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_503 && - - test_expect_code $GH__ERROR_CODE__HTTP_503 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - test_grep "error: get: (http:503)" OUT.stderr && - verify_connection_count 3 -' - -test_expect_success 'http-error: 429 Service Unavailable (with retry)' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_429 && - - test_expect_code $GH__ERROR_CODE__HTTP_429 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - test_grep "error: get: (http:429)" OUT.stderr && - verify_connection_count 3 -' - -test_expect_success 'http-error: 404 Not Found (no retry)' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_404 && - - test_expect_code $GH__ERROR_CODE__HTTP_404 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - test_grep "error: get: (http:404)" OUT.stderr && - verify_connection_count 1 -' - -################################################################# -# Tests to confirm that gvfs-helper does silently recover when an -# HTTP request succeeds after a failure. -# -################################################################# - -test_expect_success 'successful retry after http-error: origin get' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_429_1 && - - # Connect to the origin server (w/o auth). - # Make a single-object GET request. - # Confirm that it succeeds without error. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OID_ONE_BLOB_FILE" >OUT.output && - - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && - verify_connection_count 2 -' - -################################################################# -# So far we have confirmed that gvfs-helper can recover from a network -# error (with retries, since the cache-server was disabled in all of -# the above tests). Try again with fallback turned on. -# -# With mayhem "http_503" turned on both the cache and origin server -# will always throw a 503 error. -# -# Confirm that we tried to make six connections: we should hit the -# cache-server 3 times (one initial attempt and two retries) and then -# try the origin server 3 times. -# -################################################################# - -test_expect_success 'http-error: 503 Service Unavailable (with retry and fallback)' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_503 && - - test_expect_code $GH__ERROR_CODE__HTTP_503 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --fallback \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - test_grep "error: get: (http:503)" OUT.stderr && - verify_connection_count 6 -' - -################################################################# -# Now repeat the above, but explicitly turn off fallback. -# -# Again, we use mayhem "http_503". However, with fallback turned -# off, we will only attempt the 3 connections to the cache server. -# We will not try to hit the origin server. -# -# So we should only see a total of 3 connections rather than the -# six in the previous test. -# -################################################################# - -test_expect_success 'http-error: 503 Service Unavailable (with retry and no-fallback)' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_503 && - - test_expect_code $GH__ERROR_CODE__HTTP_503 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --no-fallback \ - get \ - --max-retries=2 \ - <"$OIDS_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - test_grep "error: get: (http:503)" OUT.stderr && - verify_connection_count 3 -' - -################################################################# -# Test HTTP Auth -# -################################################################# - -test_lazy_prereq CURL_7_75_OR_NEWER ' - git gvfs-helper curl-version ">=" 7.75.0 -' - -test_expect_success 'HTTP GET Auth on Origin Server' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_401 && - - # Force server to require auth. - # Connect to the origin server without auth. - # Make a single-object GET request. - # Confirm that it gets a 401 and then retries with auth. - # - GIT_CONFIG_NOSYSTEM=1 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OID_ONE_BLOB_FILE" >OUT.output && - - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && - if test_have_prereq CURL_7_75_OR_NEWER - then - verify_connection_count 2 - fi -' - -test_expect_success 'HTTP POST Auth on Origin Server' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_401 && - - # Connect to the origin server and make multi-object POST - # request and verify that it automatically handles the 401. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" >OUT.output && - - # Stop the server to prevent the verification steps from faulting-in - # any missing objects. - # - stop_gvfs_protocol_server && - - # gvfs-helper prints a "packfile " message for each received - # packfile. We verify the number of expected packfile(s) and we - # individually verify that each requested object is present in the - # shared cache (and index-pack already verified the integrity of - # the packfile), so we do not bother to run "git verify-pack -v" - # and do an exact matchup here. - # - verify_received_packfile_count 1 && - - verify_objects_in_shared_cache "$OIDS_BLOBS_FILE" && - verify_connection_count 2 -' - -test_expect_success 'HTTP GET Auth on Cache Server' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem http_401 && - - # Try auth to cache-server. Note that gvfs-helper *ALWAYS* sends - # creds to cache-servers, so we will never see the "400 Bad Request" - # response. And we are using "trust" mode, so we only expect 1 - # connection to the server. - # - GIT_CONFIG_NOSYSTEM=1 \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - --max-retries=2 \ - <"$OID_ONE_BLOB_FILE" >OUT.output && - - stop_gvfs_protocol_server && - - # gvfs-helper prints a "loose " message for each received object. - # Verify that gvfs-helper received each of the requested objects. - # - sed "s/loose //" OUT.actual && - test_cmp "$OID_ONE_BLOB_FILE" OUT.actual && - - verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && - verify_connection_count 1 -' - -################################################################# -# Integration tests with Git.exe -# -# Now that we have confirmed that gvfs-helper works in isolation, -# run a series of tests using random Git commands that fault-in -# objects as needed. -# -# At this point, I'm going to stop verifying the shape of the ODB -# (loose vs packfiles) and the number of connections required to -# get them. The tests from here on are to verify that objects are -# magically fetched whenever required. -################################################################# - -test_expect_success 'integration: explicit commit/trees, implicit blobs: diff 2 commits' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # We have a very empty repo. Seed it with all of the commits - # and trees. The purpose of this test is to demand-load the - # needed blobs only, so we prefetch the commits and trees. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - get \ - <"$OIDS_CT_FILE" >OUT.output && - - # Confirm that we do not have the blobs locally. - # With gvfs-helper turned off, we should fail. - # - test_must_fail \ - git -C "$REPO_T1" -c core.useGVFSHelper=false \ - diff $(cat m1.branch)..$(cat m3.branch) \ - >OUT.output 2>OUT.stderr && - - # Turn on gvfs-helper and retry. This should implicitly fetch - # any needed blobs. - # - git -C "$REPO_T1" -c core.useGVFSHelper=true \ - diff $(cat m1.branch)..$(cat m3.branch) \ - >OUT.output 2>OUT.stderr && - - # Verify that gvfs-helper wrote the fetched the blobs to the - # local ODB, such that a second attempt with gvfs-helper - # turned off should succeed. - # - git -C "$REPO_T1" -c core.useGVFSHelper=false \ - diff $(cat m1.branch)..$(cat m3.branch) \ - >OUT.output 2>OUT.stderr -' - -trace_has_queue_oid () { - oid=$1 - grep "gh_client__queue_oid: $oid" -} - -trace_has_immediate_oid () { - oid=$1 - grep "gh_client__get_immediate: $oid" -} - -test_expect_success 'integration: fully implicit: diff 2 commits' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - # Implicitly demand-load everything without any pre-seeding. - # - GIT_TRACE2_EVENT="$(pwd)/diff-trace.txt" \ - git -C "$REPO_T1" -c core.useGVFSHelper=true \ - diff $(cat m1.branch)..$(cat m3.branch) \ - >OUT.output 2>OUT.stderr && - - oid=$(git -C "$REPO_SRC" rev-parse main:file9.txt.t) && - trace_has_queue_oid $oid OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server -' - -test_expect_success 'integration: implicit-get: cache_http_503,no-fallback: diff 2 commits' ' - test_when_finished "per_test_cleanup" && - - # Tell cache server to send 503 and origin server to send 200. - start_gvfs_protocol_server_with_mayhem cache_http_503 && - - # Implicitly demand-load everything without any pre-seeding. - # This should fail because we do not allow fallback. - # - test_must_fail \ - git -C "$REPO_T2" \ - -c core.useGVFSHelper=true \ - -c gvfs.fallback=false \ - diff $(cat m1.branch)..$(cat m3.branch) \ - >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server -' - -test_expect_success 'integration: implicit-get: cache_http_503,with-fallback: diff 2 commits' ' - test_when_finished "per_test_cleanup" && - - # Tell cache server to send 503 and origin server to send 200. - start_gvfs_protocol_server_with_mayhem cache_http_503 && - - # Implicitly demand-load everything without any pre-seeding. - # - git -C "$REPO_T2" \ - -c core.useGVFSHelper=true \ - -c gvfs.fallback=true \ - diff $(cat m1.branch)..$(cat m3.branch) \ - >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server -' - -# T2 should be considered contaminated at this point. - - -################################################################# -# Duplicate packfile tests. -# -# If we request a fixed set of blobs, we should get a unique packfile -# of the form "vfs-.{pack,idx}". It we request that same set -# again, the server should create and send the exact same packfile. -# True web servers might build the custom packfile in random order, -# but our test web server should give us consistent results. -# -# Verify that we can handle the duplicate pack and idx file properly. -################################################################# - -test_expect_success 'duplicate: vfs- packfile' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && - verify_received_packfile_count 1 && - verify_vfs_packfile_count 1 && - - # Re-fetch the same packfile. We do not care if it replaces - # first one or if it silently fails to overwrite the existing - # one. We just confirm that afterwards we only have 1 packfile. - # - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && - verify_received_packfile_count 1 && - verify_vfs_packfile_count 1 && - - stop_gvfs_protocol_server -' - -test_expect_success 'duplicate and busy: vfs- packfile' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server && - - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" \ - >OUT.output \ - 2>OUT.stderr && - verify_received_packfile_count 1 && - verify_vfs_packfile_count 1 && - - # Re-fetch the same packfile, but hold the existing packfile - # open for writing on an obscure (and randomly-chosen) file - # descriptor. - # - # This should cause the replacement-install to fail (at least - # on Windows) with an EBUSY or EPERM or something. - # - # Verify that that error is eaten. We do not care if the - # replacement is retried or if gvfs-helper simply discards the - # second instance. We just confirm that afterwards we only - # have 1 packfile on disk and that the command "lies" and reports - # that it created the existing packfile. (We want the lie because - # in normal usage, gh-client has already built the packed-git list - # in memory and is using gvfs-helper to fetch missing objects; - # gh-client does not care who does the fetch, but it needs to - # update its packed-git list and restart the object lookup.) - # - PACK=$(first_received_packfile_pathname) && - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" \ - >OUT.output \ - 2>OUT.stderr \ - 9>>"$PACK" && - verify_received_packfile_count 1 && - verify_vfs_packfile_count 1 && - - stop_gvfs_protocol_server -' - -################################################################# -# Ensure that the SHA of the blob we received matches the SHA of -# the blob we requested. -################################################################# - -# Request a loose blob from the server. Verify that we received -# content matches the requested SHA. -# -test_expect_success 'catch corrupted loose object' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem corrupt_loose && - - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - # Verify corruption detected. - # Verify valid blob not included in response to client. - - grep "hash failed for received loose object" OUT.stderr && - - # Verify that we did not write the corrupted blob to the ODB. - - ! verify_objects_in_shared_cache "$OID_ONE_BLOB_FILE" && - git -C "$REPO_T1" fsck -' - -################################################################# -# Ensure that we can detect when we receive a corrupted packfile -# from the server. This is not concerned with network IO errors, -# but rather cases when the cache or origin server generates or -# sends an invalid packfile. -# -# For example, if the server throws an exception and writes the -# stack trace to the socket rather than or in addition to the -# packfile content. -# -# Or for example, if the packfile on the server's disk is corrupt -# and it sends it correctly, but the original data was already -# garbage, so the client still has garbage (and retrying won't -# help). -################################################################# - -# Send corrupt PACK files w/o IDX files (so that `gvfs-helper` -# must use `index-pack` to create it. (And as a side-effect, -# validate the PACK file is not corrupt.) -test_expect_success 'prefetch corrupt pack without idx' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem \ - bad_prefetch_pack_sha \ - no_prefetch_idx && - - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch \ - --max-retries=0 \ - --since="1000000000" \ - >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server && - - # Verify corruption detected in pack when building - # local idx file for it. - - test_grep "error: .* index-pack failed" OUT.stderr -' - -# Send corrupt PACK files with IDX files. Since the cache server -# sends both, `gvfs-helper` might fail to verify both of them. -test_expect_success 'prefetch corrupt pack with corrupt idx' ' - test_when_finished "per_test_cleanup" && - start_gvfs_protocol_server_with_mayhem \ - bad_prefetch_pack_sha && - - test_must_fail \ - git -C "$REPO_T1" gvfs-helper \ - --cache-server=disable \ - --remote=origin \ - --no-progress \ - prefetch \ - --max-retries=0 \ - --since="1000000000" \ - >OUT.output 2>OUT.stderr && - - stop_gvfs_protocol_server -' - -################################################################# -# Tests for gvfs..cache-server config. -# -# These tests verify that verb-specific cache-server overrides work -# correctly. We run two servers on different ports: -# - Server 0 (base port): configured as gvfs.cache-server (default) -# - Server 1 (base port + 1): configured as gvfs..cache-server -# -# For each verb (prefetch, get, post), we verify that: -# 1. When using the verb-specific override, the request goes to server 1 -# 2. When using a different verb, the request goes to server 0 -################################################################# - -test_expect_success 'verb-specific cache-server: prefetch uses gvfs.prefetch.cache-server' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - - # Configure server 0 as default cache-server and server 1 for prefetch. - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && - - # Run prefetch - should go to server 1. - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --no-progress \ - prefetch >OUT.output 2>OUT.stderr && - - # Verify server 1 was contacted (prefetch-specific). - verify_server_was_contacted 1 && - - # Verify server 0 was NOT contacted. - verify_server_was_not_contacted 0 -' - -test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch.cache-server' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - - # Configure server 0 as default cache-server and server 1 for prefetch. - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && - - # Run get - should go to server 0 (default), not server 1 (prefetch). - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && - - # Verify server 0 was contacted (default cache-server). - verify_server_was_contacted 0 && - - # Verify server 1 was NOT contacted (prefetch-specific). - verify_server_was_not_contacted 1 -' - -test_expect_success 'verb-specific cache-server: get uses gvfs.get.cache-server' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - - # Configure server 0 as default cache-server and server 1 for get. - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && - - # Run get - should go to server 1. - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && - - # Verify server 1 was contacted (get-specific). - verify_server_was_contacted 1 && - - # Verify server 0 was NOT contacted. - verify_server_was_not_contacted 0 -' - -test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get.cache-server' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - - # Configure server 0 as default cache-server and server 1 for get. - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && - - # Run prefetch - should go to server 0 (default), not server 1 (get). - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --no-progress \ - prefetch >OUT.output 2>OUT.stderr && - - # Verify server 0 was contacted (default cache-server). - verify_server_was_contacted 0 && - - # Verify server 1 was NOT contacted (get-specific). - verify_server_was_not_contacted 1 -' - -test_expect_success 'verb-specific cache-server: post uses gvfs.post.cache-server' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - - # Configure server 0 as default cache-server and server 1 for post. - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && - - # Run post - should go to server 1. - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && - - # Verify server 1 was contacted (post-specific). - verify_server_was_contacted 1 && - - # Verify server 0 was NOT contacted. - verify_server_was_not_contacted 0 -' - -test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cache-server' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - - # Configure server 0 as default cache-server and server 1 for post. - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && - - # Run get - should go to server 0 (default), not server 1 (post). - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && - - # Verify server 0 was contacted (default cache-server). - verify_server_was_contacted 0 && - - # Verify server 1 was NOT contacted (post-specific). - verify_server_was_not_contacted 1 -' - -test_expect_success 'verb-specific cache-server: all verbs with different servers' ' - test_when_finished "per_test_cleanup" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.cache-server" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && - test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && - start_gvfs_protocol_server 0 && - start_gvfs_protocol_server 1 && - start_gvfs_protocol_server 2 && - start_gvfs_protocol_server 3 && - - # Configure each verb to use a different server: - # - server 0: default (unused in this test) - # - server 1: prefetch - # - server 2: get - # - server 3: post - git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && - git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && - git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 2)" && - git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 3)" && - - # Run prefetch - should go to server 1. - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --no-progress \ - prefetch >OUT.output 2>OUT.stderr && - verify_server_was_contacted 1 && - verify_server_was_not_contacted 0 && - verify_server_was_not_contacted 2 && - verify_server_was_not_contacted 3 && - - # Clean up shared cache for next verb. - rm -rf "$SHARED_CACHE_T1"/pack/* && - - # Run get - should go to server 2. - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - get \ - <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && - verify_server_was_contacted 2 && - - # Clean up shared cache for next verb. - rm -rf "$SHARED_CACHE_T1"/[0-9a-f][0-9a-f]/ && - rm -rf "$SHARED_CACHE_T1"/pack/* && - - # Run post - should go to server 3. - git -C "$REPO_T1" gvfs-helper \ - --cache-server=trust \ - --remote=origin \ - --no-progress \ - post \ - <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && - verify_server_was_contacted 3 -' - -test_done