From 5e4b2fabc62bf8e34cfef95238c6642182465f87 Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 9 Sep 2026 06:58:46 +0000 Subject: [PATCH] publish.sh: stop awscli2 sending a checksum Garage rejects Measured against the real endpoint, not inferred: every PutObject failed with An error occurred (InvalidRequest) when calling the PutObject operation: Bad request: invalid checksum algorithm on aws-cli/2.34.24. Recent awscli2 attaches a checksum algorithm by default and Garage refuses it, so publish.sh could not upload anything -- after passing its entire test suite. It also reads as a permissions problem, which is the expensive part: reads succeed, writes fail, and it fails identically against a bucket the same credentials demonstrably can write. Set here rather than documented, because a caller should not need to know a client-side default to publish to the object store this script exists for, and leaving it to the environment makes it work on one machine and not another. `when_required` keeps checksums where the protocol needs them, so this is narrower than switching them off. Adds the test that would have caught it: an opt-in live round trip that uploads, reads back and deletes, skipped unless a scratch bucket and credentials are present so the offline run is unchanged. Everything else in this suite proves the command is well formed, which is a different claim from a server accepting it, and this failure lived exactly in that gap. The live block captures the real credentials at the top of the file, before the deliberate fakes (AKIAFAKETESTKEY) are exported for the offline tests -- otherwise it authenticates as the fake and fails every time, and a live test that always fails gets disabled, which is worse than not having one. Verified both directions against the real bucket: with the fix, 11 passed / 0 skipped; with it removed, the two live assertions fail. Offline, 9 passed / 1 skipped. Closes #3. Co-authored-by: bit --- publish.sh | 18 ++++++++++++++++++ test_publish.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/publish.sh b/publish.sh index 6e4d0fc..16e3be6 100755 --- a/publish.sh +++ b/publish.sh @@ -80,6 +80,24 @@ if [ -n "$prefix" ]; then dest="$dest/${prefix#/}" fi +# awscli2 2.23+ attaches a checksum algorithm to every PutObject by default. +# Garage rejects it outright -- measured against a real bucket: +# +# An error occurred (InvalidRequest) when calling the PutObject +# operation: Bad request: invalid checksum algorithm +# +# with aws-cli/2.34.24. Every upload fails, and it reads as a permissions +# problem until you notice that reads succeed and that writes fail the same +# way against a bucket the same credentials demonstrably can write. +# +# Set here rather than documented, because a caller should not have to know +# about a client-side default to publish to the object store this script +# exists for -- and leaving it to the environment means it works on one +# machine and not another. `when_required` keeps checksums where the protocol +# actually needs them, so this is narrower than switching them off. +export AWS_REQUEST_CHECKSUM_CALCULATION="${AWS_REQUEST_CHECKSUM_CALCULATION:-when_required}" +export AWS_RESPONSE_CHECKSUM_VALIDATION="${AWS_RESPONSE_CHECKSUM_VALIDATION:-when_required}" + cmd=(bash "$here/with-nixpkgs.sh" awscli2 aws s3 sync "$local_dir" "$dest" --endpoint-url "$endpoint" --region "${AWS_DEFAULT_REGION:-us-east-1}" diff --git a/test_publish.sh b/test_publish.sh index 240448e..eb3d088 100755 --- a/test_publish.sh +++ b/test_publish.sh @@ -11,6 +11,16 @@ # well-formed, without needing a real bucket or reaching beyond loopback. set -euo pipefail here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Captured BEFORE the fake credentials below are exported. Those fakes are +# deliberate -- they keep the offline tests from ever reaching a real service +# -- but they also overwrite the real ones for the rest of this file, so the +# live round trip at the bottom has to hold on to the originals here or it +# would authenticate as AKIAFAKETESTKEY and fail every time. A live test that +# always fails gets disabled, which is worse than not having one. +LIVE_KEY_ID="${AWS_ACCESS_KEY_ID:-}" +LIVE_SECRET="${AWS_SECRET_ACCESS_KEY:-}" +LIVE_REGION="${AWS_DEFAULT_REGION:-}" source "$here/test_lib.sh" work=$(mktemp -d) @@ -54,4 +64,42 @@ if [ "${rc:-0}" -ne 0 ]; then pass "publish.sh against a closed port exits non-z fi assert_contains "the failure is a connection error, not an argument error" "$real_out" "Could not connect" +# --- live round trip, opt-in ------------------------------------------------ +# Skipped unless a scratch bucket and credentials are present, so the default +# offline run is unchanged. This is the only test here that talks to a real +# server, and the only one that could have caught the Garage checksum +# rejection (#3): everything above proves the command is well formed, which is +# a different claim from a server accepting it. +if [ -n "${FDROID_CI_LIVE_BUCKET:-}" ] && [ -n "${FDROID_CI_LIVE_ENDPOINT:-}" ] \ + && [ -n "$LIVE_KEY_ID" ] && [ -n "$LIVE_SECRET" ]; then + echo "--- a real round trip against a real endpoint ---" + export AWS_ACCESS_KEY_ID="$LIVE_KEY_ID" + export AWS_SECRET_ACCESS_KEY="$LIVE_SECRET" + [ -n "$LIVE_REGION" ] && export AWS_DEFAULT_REGION="$LIVE_REGION" + live_dir=$(mktemp -d) + stamp="live-$(date -u +%s)-$$" + printf 'publish.sh live probe %s\n' "$stamp" > "$live_dir/probe.txt" + + if bash "$here/publish.sh" "$live_dir" "$FDROID_CI_LIVE_BUCKET" \ + "$FDROID_CI_LIVE_ENDPOINT" "$stamp" >/dev/null 2>&1; then + pass "publish.sh uploaded to a real bucket" + else + fail "publish.sh uploaded to a real bucket" + fi + + got=$(bash "$here/with-nixpkgs.sh" awscli2 aws --endpoint-url "$FDROID_CI_LIVE_ENDPOINT" \ + s3 cp "s3://$FDROID_CI_LIVE_BUCKET/$stamp/probe.txt" - 2>/dev/null || true) + if [ "$got" = "publish.sh live probe $stamp" ]; then + pass "the uploaded object reads back byte-identical" + else + fail "the uploaded object reads back byte-identical" + fi + + bash "$here/with-nixpkgs.sh" awscli2 aws --endpoint-url "$FDROID_CI_LIVE_ENDPOINT" \ + s3 rm --recursive "s3://$FDROID_CI_LIVE_BUCKET/$stamp/" >/dev/null 2>&1 || true + rm -rf "$live_dir" +else + skip "live round trip (set FDROID_CI_LIVE_BUCKET, FDROID_CI_LIVE_ENDPOINT and AWS credentials)" +fi + report_and_exit