Merge pull request 'publish.sh: stop awscli2 sending a checksum Garage rejects' (#4) from fix/garage-checksum into main

publish.sh

@@ -80,6 +80,24 @@ if [ -n "$prefix" ]; then
   dest="$dest/${prefix#/}"
 fi
 
+# awscli2 2.23+ attaches a checksum algorithm to every PutObject by default.
+# Garage rejects it outright -- measured against a real bucket:
+#
+#     An error occurred (InvalidRequest) when calling the PutObject
+#     operation: Bad request: invalid checksum algorithm
+#
+# with aws-cli/2.34.24. Every upload fails, and it reads as a permissions
+# problem until you notice that reads succeed and that writes fail the same
+# way against a bucket the same credentials demonstrably can write.
+#
+# Set here rather than documented, because a caller should not have to know
+# about a client-side default to publish to the object store this script
+# exists for -- and leaving it to the environment means it works on one
+# machine and not another. `when_required` keeps checksums where the protocol
+# actually needs them, so this is narrower than switching them off.
+export AWS_REQUEST_CHECKSUM_CALCULATION="${AWS_REQUEST_CHECKSUM_CALCULATION:-when_required}"
+export AWS_RESPONSE_CHECKSUM_VALIDATION="${AWS_RESPONSE_CHECKSUM_VALIDATION:-when_required}"
+
 cmd=(bash "$here/with-nixpkgs.sh" awscli2 aws s3 sync "$local_dir" "$dest"
      --endpoint-url "$endpoint"
      --region "${AWS_DEFAULT_REGION:-us-east-1}"

test_publish.sh

@@ -11,6 +11,16 @@
 #   well-formed, without needing a real bucket or reaching beyond loopback.
 set -euo pipefail
 here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+# Captured BEFORE the fake credentials below are exported. Those fakes are
+# deliberate -- they keep the offline tests from ever reaching a real service
+# -- but they also overwrite the real ones for the rest of this file, so the
+# live round trip at the bottom has to hold on to the originals here or it
+# would authenticate as AKIAFAKETESTKEY and fail every time. A live test that
+# always fails gets disabled, which is worse than not having one.
+LIVE_KEY_ID="${AWS_ACCESS_KEY_ID:-}"
+LIVE_SECRET="${AWS_SECRET_ACCESS_KEY:-}"
+LIVE_REGION="${AWS_DEFAULT_REGION:-}"
 source "$here/test_lib.sh"
 
 work=$(mktemp -d)
@@ -54,4 +64,42 @@ if [ "${rc:-0}" -ne 0 ]; then pass "publish.sh against a closed port exits non-z
 fi
 assert_contains "the failure is a connection error, not an argument error" "$real_out" "Could not connect"
 
+# --- live round trip, opt-in ------------------------------------------------
+# Skipped unless a scratch bucket and credentials are present, so the default
+# offline run is unchanged. This is the only test here that talks to a real
+# server, and the only one that could have caught the Garage checksum
+# rejection (#3): everything above proves the command is well formed, which is
+# a different claim from a server accepting it.
+if [ -n "${FDROID_CI_LIVE_BUCKET:-}" ] && [ -n "${FDROID_CI_LIVE_ENDPOINT:-}" ] \
+   && [ -n "$LIVE_KEY_ID" ] && [ -n "$LIVE_SECRET" ]; then
+  echo "--- a real round trip against a real endpoint ---"
+  export AWS_ACCESS_KEY_ID="$LIVE_KEY_ID"
+  export AWS_SECRET_ACCESS_KEY="$LIVE_SECRET"
+  [ -n "$LIVE_REGION" ] && export AWS_DEFAULT_REGION="$LIVE_REGION"
+  live_dir=$(mktemp -d)
+  stamp="live-$(date -u +%s)-$$"
+  printf 'publish.sh live probe %s\n' "$stamp" > "$live_dir/probe.txt"
+
+  if bash "$here/publish.sh" "$live_dir" "$FDROID_CI_LIVE_BUCKET" \
+       "$FDROID_CI_LIVE_ENDPOINT" "$stamp" >/dev/null 2>&1; then
+    pass "publish.sh uploaded to a real bucket"
+  else
+    fail "publish.sh uploaded to a real bucket"
+  fi
+
+  got=$(bash "$here/with-nixpkgs.sh" awscli2 aws --endpoint-url "$FDROID_CI_LIVE_ENDPOINT" \
+          s3 cp "s3://$FDROID_CI_LIVE_BUCKET/$stamp/probe.txt" - 2>/dev/null || true)
+  if [ "$got" = "publish.sh live probe $stamp" ]; then
+    pass "the uploaded object reads back byte-identical"
+  else
+    fail "the uploaded object reads back byte-identical"
+  fi
+
+  bash "$here/with-nixpkgs.sh" awscli2 aws --endpoint-url "$FDROID_CI_LIVE_ENDPOINT" \
+    s3 rm --recursive "s3://$FDROID_CI_LIVE_BUCKET/$stamp/" >/dev/null 2>&1 || true
+  rm -rf "$live_dir"
+else
+  skip "live round trip (set FDROID_CI_LIVE_BUCKET, FDROID_CI_LIVE_ENDPOINT and AWS credentials)"
+fi
+
 report_and_exit