fdroid-repo-ci/publish.sh

#!/usr/bin/env bash
#
# Upload a repo/ tree -- signed APKs plus the signed index -- to an
# S3-compatible bucket.
#
#     AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \
#       bash publish.sh <repo-root>/repo <bucket> <endpoint-url> [<prefix>]
#
# Bucket, endpoint and prefix are all arguments, never baked in here: this
# repo does not know, and per fdroid-repo-ci#1 must never come to know, which
# bucket or host any consumer publishes to. Credentials come from the
# standard AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN
# environment variables that `aws` itself reads -- nothing here parses or
# forwards them as arguments.
#
# Region is required by the `aws` CLI even against a non-AWS S3-compatible
# backend that ignores it; set AWS_DEFAULT_REGION if the endpoint cares,
# otherwise the placeholder below is never actually consulted.
#
# ## Ordering (fdroid-repo-ci#5)
#
# An F-Droid repo is a signed index plus the files it names. A partial
# upload can leave them disagreeing in two ways: the index lands first and
# names files that still 404 (breaks the repo for every client until the
# upload finishes or is rolled back), or the files land first and the index
# is not yet updated (harmless -- the new build is merely not visible yet).
# Only the first is dangerous, and `aws s3 sync` walking a tree in its own
# order gives no protection against it. So this script uploads in two
# passes, unconditionally: every non-index file first, the index itself
# last. The repo is therefore always internally consistent -- before a
# publish it still describes the old state, after it describes the new one,
# and an interruption anywhere in pass 1 simply leaves it describing the old
# state a little longer, never a mixture.
#
# "The index" here is index-v1.json(.asc), index-v2.json, entry.json and
# entry.jar -- fdroidserver's actual top-level index/entry file names,
# matched only at the top level of the tree (a nested path like
# diff/<ts>.json is not "the index" and travels with the ordinary files;
# it is only ever reached by a client that already has last publish's index
# and is willing to look for it).
#
# The two passes are also separately invokable, with `--files-only` and
# `--index-only`. This is not a test seam -- it is real, useful on its own:
# an interrupted publish can resume at the index step without re-uploading
# every APK, and a caller can stage files now and publish the index later.
# It is also what makes the ordering guarantee testable *by construction*:
# run `--files-only`, assert the bucket has the files and not the index, run
# `--index-only`, assert the index appears -- no synchronization hook and no
# `eval` of anything env-supplied needed in a script that runs with a
# signing key and S3 credentials in its environment, in a public repo. (An
# earlier version of this script had exactly such a hook; it was removed
# once these flags made it unnecessary.) With neither flag, both passes run,
# files then index, which is the safe default and is unconditional -- there
# is no way to select "index then files" short of invoking `--index-only`
# by hand before ever running `--files-only`, which is the deliberate
# misuse the two flags exist to make possible for cases that want it
# (resuming, staging) while never being the default.
#
# ## Credentials are verified before anything is uploaded (fdroid-repo-ci#5)
#
# Checking that AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY are merely *set* does
# not mean they are *usable* -- an unusable-but-present credential failing
# halfway through is exactly how a partial (and, per the ordering point
# above, potentially index-first) upload happens. So before pass 1 this
# script makes one real authenticated call (`head-bucket`) and aborts if it
# fails, rather than discovering that the credentials do not work partway
# through a sync.
#
# ## Refusing to overwrite an existing APK (fdroid-repo-ci#6)
#
# next-version-code.sh's clamp is monotonic against *observed* state, not
# monotonic by construction -- it does not close the race where two jobs
# start in the same minute, both read the same `published` value, both
# compute the same next versionCode, and both build and publish. The window
# is the whole build, not just the clock read, because the version code is
# baked into the APK before it exists. Nothing in this script can prevent
# that wasted build. What it CAN do -- and is required to do, fdroid-repo-ci#6
# -- is turn the collision from a silent, permanent wrong state (two
# different APKs claiming one versionCode, with clients pinned to whichever
# they happened to fetch first) into a loud failure of the second job, at
# the last moment when the information is finally reliable: immediately
# before that job would otherwise upload over the first job's APK. An
# F-Droid repo names an APK `<applicationId>_<versionCode>.apk`, so the
# collision this guards against is exactly a name collision -- every local
# *.apk file's destination key is checked with `head-object` before pass 1,
# and the whole publish aborts if any of them is already there. This runs
# for every *.apk regardless of whether it actually collides, because there
# is no cheaper way to know in advance which ones might.
#
# Note this also means retrying a publish that already got partway through
# uploading its own APKs (e.g. pass 1 died mid-sync from a network drop)
# will now itself be refused on retry, indistinguishable at the object level
# from a genuine concurrent collision. That is an accepted cost of the same
# trade: this check cannot tell "the same job resuming" from "a different
# job that computed the same versionCode" without hashing every object,
# which was judged not worth the added complexity here. Recovering from that
# case is a fresh publish with a new versionCode, or deleting the partial
# upload by hand.
#
# Uses `aws s3 sync`, not `cp --recursive`: re-publishing re-uploads only
# what changed. Does **not** pass `--delete` by default -- an F-Droid repo
# normally keeps every APK it has ever served (older index entries can still
# point at them), so silently pruning the remote on every publish is the
# wrong default. Pass `--delete` as an extra argument if a caller genuinely
# wants the remote pruned to match local; it is forwarded to *both* passes,
# which is safe because `aws s3 sync --delete` never deletes anything that
# was excluded by that pass's own --exclude/--include filters -- so the
# files-pass never touches an index object and the index-pass never touches
# a file object, regardless of --delete.
#
# `fdroid update` needs no network; this is the one script in this repo that
# does (fdroid-repo-ci#1), and is accordingly the one script whose happy path
# the test suite cannot exercise against a real bucket. `--dry-run` prints
# the command(s) this script would run for whichever pass(es) are selected,
# in order -- including the `with-nixpkgs.sh` pin -- and exits without
# touching the network at all: no credential check, no APK-existence check,
# no sync. That is what the test suite asserts against for command
# construction and ordering. (A PATH-stubbed `aws` cannot do this job: `nix
# shell` puts the packages it was asked for at the *front* of PATH, ahead of
# anything already there, so a stub earlier on the caller's PATH would
# simply be shadowed by the real `aws` from awscli2.) The credential check,
# the APK-existence check and the two-pass ordering are each proven
# separately: a real invocation against a closed local port for "fails on
# the network, and fails during the credential check specifically, before
# any upload is attempted"; a tiny local HEAD-only HTTP stub for "an
# existing APK is refused, and only an existing APK", offline; and the
# opt-in live round trip against a real bucket, run once as `--files-only`
# then `--index-only` to observe the ordering by construction, and once
# more as a real head-to-head APK collision.
#
# Pins awscli2 via with-nixpkgs.sh/nixpkgs-pin, next to this script.
set -euo pipefail

here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

usage() {
  cat >&2 <<'USAGE'
usage: publish.sh [--dry-run] [--files-only | --index-only] \
                   <local-repo-dir> <bucket> <endpoint-url> [<prefix>] [extra aws s3 sync args...]

With neither --files-only nor --index-only, both passes run: every
non-index file first, then the index -- the safe default, and the only way
to select "index before files" is to invoke --index-only by hand ahead of
--files-only, never the default. --files-only and --index-only are mutually
exclusive.

Credentials: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (and optionally
AWS_SESSION_TOKEN) must already be exported -- never passed as arguments.
USAGE
}

dry_run=false
files_only=false
index_only=false
while [ "$#" -gt 0 ]; do
  case "$1" in
    --dry-run) dry_run=true; shift ;;
    --files-only) files_only=true; shift ;;
    --index-only) index_only=true; shift ;;
    *) break ;;
  esac
done

if [ "$files_only" = true ] && [ "$index_only" = true ]; then
  echo "publish.sh: --files-only and --index-only are mutually exclusive" >&2
  exit 2
fi

if [ "$#" -lt 3 ]; then
  usage
  exit 2
fi

local_dir=$1
bucket=$2
endpoint=$3
shift 3

prefix=""
if [ "$#" -gt 0 ] && [[ $1 != -* ]]; then
  prefix=$1
  shift
fi

extra_args=("$@")

[ -d "$local_dir" ] || { echo "publish.sh: no such directory: $local_dir" >&2; exit 2; }
: "${AWS_ACCESS_KEY_ID:?publish.sh: AWS_ACCESS_KEY_ID is not set}"
: "${AWS_SECRET_ACCESS_KEY:?publish.sh: AWS_SECRET_ACCESS_KEY is not set}"

dest="s3://$bucket"
if [ -n "$prefix" ]; then
  dest="$dest/${prefix#/}"
fi
region="${AWS_DEFAULT_REGION:-us-east-1}"

# awscli2 2.23+ attaches a checksum algorithm to every PutObject by default.
# Garage rejects it outright -- measured against a real bucket:
#
#     An error occurred (InvalidRequest) when calling the PutObject
#     operation: Bad request: invalid checksum algorithm
#
# with aws-cli/2.34.24. Every upload fails, and it reads as a permissions
# problem until you notice that reads succeed and that writes fail the same
# way against a bucket the same credentials demonstrably can write.
#
# Set here rather than documented, because a caller should not have to know
# about a client-side default to publish to the object store this script
# exists for -- and leaving it to the environment means it works on one
# machine and not another. `when_required` keeps checksums where the protocol
# actually needs them, so this is narrower than switching them off.
export AWS_REQUEST_CHECKSUM_CALCULATION="${AWS_REQUEST_CHECKSUM_CALCULATION:-when_required}"
export AWS_RESPONSE_CHECKSUM_VALIDATION="${AWS_RESPONSE_CHECKSUM_VALIDATION:-when_required}"

aws_base=(bash "$here/with-nixpkgs.sh" awscli2 aws --endpoint-url "$endpoint" --region "$region")

# Top-level index/entry file names only -- see the "Ordering" comment above
# for why a nested path such as diff/<ts>.json is deliberately not matched.
index_exclude_filters=(--exclude "index*" --exclude "entry.*")
index_only_filters=(--exclude "*" --include "index*" --include "entry.*")

cmd_files=("${aws_base[@]}" s3 sync "$local_dir" "$dest"
           "${index_exclude_filters[@]}" "${extra_args[@]}")
cmd_index=("${aws_base[@]}" s3 sync "$local_dir" "$dest"
           "${index_only_filters[@]}" "${extra_args[@]}")

# Which pass(es) this invocation runs. Unconditional and exactly one of
# three shapes: both (the default), files alone, or index alone -- the
# mutual-exclusivity check above already ruled out both flags together.
run_files=true
run_index=true
[ "$index_only" = true ] && run_files=false
[ "$files_only" = true ] && run_index=false

if [ "$dry_run" = true ]; then
  if [ "$run_files" = true ]; then
    printf '%q ' "${cmd_files[@]}"
    printf '\n'
  fi
  if [ "$run_index" = true ]; then
    printf '%q ' "${cmd_index[@]}"
    printf '\n'
  fi
  exit 0
fi

echo "publish.sh: verifying credentials against $endpoint"
if ! verify_out=$("${aws_base[@]}" s3api head-bucket --bucket "$bucket" 2>&1); then
  echo "publish.sh: credentials rejected (or bucket unreachable) -- nothing was uploaded:" >&2
  echo "$verify_out" >&2
  exit 1
fi

if [ "$run_files" = true ]; then
  # Refuse to overwrite an existing APK -- see the comment block above for
  # why this exists and what it does and does not close. Every local *.apk
  # is checked, regardless of nesting, since fdroidserver's binary-repo
  # layout keeps them under repo/ directly but nothing here should assume
  # that. Skipped entirely for a plain --index-only invocation: the index
  # pass never uploads a *.apk (index_only_filters excludes everything but
  # the index/entry names), so there is nothing this check could catch.
  while IFS= read -r -d '' apk; do
    rel=${apk#"$local_dir"/}
    key="$rel"
    if [ -n "$prefix" ]; then
      key="${prefix#/}/$rel"
    fi
    if head_out=$("${aws_base[@]}" s3api head-object --bucket "$bucket" --key "$key" 2>&1); then
      echo "publish.sh: refusing to publish -- $key already exists in s3://$bucket (would overwrite an existing APK, fdroid-repo-ci#6)" >&2
      # Hint only, never a control-flow change (fdroid-repo-ci#13): a
      # colliding filename with no digits at all cannot be carrying a
      # versionCode, which is the single most common way to trip this
      # guard on every publish after the first -- Flutter's
      # app-<flavor>-release.apk being the concrete case that motivated
      # this. A filename that does contain digits still gets refused with
      # just the message above, exactly as before; it may yet be a genuine
      # collision (fdroid-repo-ci#6) and this heuristic cannot tell.
      base=$(basename "$key")
      if [[ $base != *[0-9]* ]]; then
        echo "publish.sh: $base does not appear to carry a versionCode -- F-Droid's convention is <applicationId>_<versionCode>.apk; a constant filename plus this guard means only the first publish into a prefix can ever succeed" >&2
      fi
      exit 1
    elif echo "$head_out" | grep -qi '404\|not found\|NoSuchKey'; then
      : # does not exist yet -- fine, this is the common case
    else
      echo "publish.sh: could not check whether $key already exists -- refusing to guess:" >&2
      echo "$head_out" >&2
      exit 1
    fi
  done < <(find "$local_dir" -type f -name '*.apk' -print0)

  echo "publish.sh: pass 1/2: uploading files (index deferred) $local_dir -> $dest"
  "${cmd_files[@]}"
fi

if [ "$run_index" = true ]; then
  echo "publish.sh: pass 2/2: uploading the index $local_dir -> $dest"
  "${cmd_index[@]}"
fi

echo "publish.sh: done"