fdroid-repo-ci/fdroid-update.sh

#!/usr/bin/env bash
#
# Run `fdroid update` over a `repo/` directory of already-signed APKs and
# produce the signed index.
#
# Binary-repo mode (fdroid-repo-ci#1): every APK under repo/ already carries
# its own signature -- from sign-apk.sh, or from whatever produced it -- so
# this step never touches an app's signing key. It only uses the separate
# *repo index* key, a different key entirely, which is what this script
# signs the index with.
#
#     bash fdroid-update.sh <repo-root> <index-keystore> <key-alias> \
#       <keystore-pass-env> [<key-pass-env>]
#
# <repo-root> is the directory *containing* repo/ -- fdroidserver's own
# layout, not repo/ itself. This script writes <repo-root>/config.yml itself
# on every run (see below), so an existing one is overwritten; nothing else
# under <repo-root> is touched beyond what `fdroid update` writes into repo/.
#
# <keystore-pass-env>/<key-pass-env> name environment variables holding the
# passwords, the same convention as sign-apk.sh and for the same reason: an
# argument is visible in the job log and in `ps`, a password must not be. If
# <key-pass-env> is omitted, the keystore password is reused as the key
# password.
#
# FDROID_REPO_URL and FDROID_REPO_NAME (environment variables, not
# arguments -- they are not secrets, but they are metadata that belongs
# beside the other config.yml fields this script already writes, not a sixth
# and seventh positional) are REQUIRED and this script refuses to run
# without them. Omitting them used to mean fdroidserver silently fell back to
# its own placeholder template ("MyFirstFDroidRepo.org", "My First F-Droid
# Repo Demo") -- a repo that publishes successfully while advertising
# someone else's URL and name, which is worse than failing loudly.
# FDROID_REPO_DESCRIPTION is optional. This is a breaking change to this
# script's interface (fdroid-repo-ci#11); acceptable only because
# weblib-viewer is this action's sole consumer and is updated in the same
# change.
#
# FDROID_REPO_URL MUST end in the literal suffix "/repo" -- fdroidserver
# enforces this itself (common.py: "repo_url needs to end with /repo"),
# unconditionally, whenever config.yml sets repo_url at all, and refuses to
# run otherwise. It is not this script's own rule and there is no way to
# turn it off, so the caller's publish step has to serve the tree under a
# URL path actually ending in /repo -- e.g. `publish.sh`'s own <prefix>
# argument -- or the value embedded here (verbatim, as repo.address in
# index-v2.json) will not be where the index actually is.
#
# FDROID_PREVIOUS_INDEX (optional) is a **local path** to the
# `index-v2.json` this repo published last time. Give it and every build
# that is still present keeps the date it was first published; omit it and
# `fdroid update` stamps *every* APK in repo/ with the current wall clock,
# which is weblib/weblib-viewer#95 -- a version list where five builds all
# claim to have been added within seconds of each other.
#
# The dates live in the published index and nowhere else. fdroidserver's
# `common.KnownApks` loads them from `repo/index-v2.json` (and
# `archive/index-v2.json`) relative to the repo root, and falls back to
# `datetime.now()` for any APK filename it has not seen -- its own docstring
# calls the index "currently the only way to permanently store the 'updated'
# date of APKs". A CI job that rebuilds repo/ on a fresh runner, as a
# retention scheme re-downloading the previous APKs does, therefore starts
# with an empty KnownApks. `preserve_added.py` (next to this script, and
# where the full citation lives) puts the previous index back before the
# run and checks afterwards that the dates really did survive; a build that
# is genuinely new is untouched and still gets "now". Nothing here invents
# or writes a date.
#
# A path that is set but does not exist is a hard failure, not a shrug: a
# consumer whose download silently produced nothing would otherwise flatten
# its whole history on every publish and look fine doing it. A genuine
# first-ever publish has no previous index, and expresses that by leaving
# the variable unset.
#
# Needs no network (fdroid-repo-ci#1) -- it only reads repo/ and signs.
# FDROID_PREVIOUS_INDEX is a local file for that reason: fetching last
# publish's index is the caller's job, since only the caller knows the
# bucket (and, per fdroid-repo-ci#1, must remain the only one that does).
# Pins fdroidserver, the JDK (fdroidserver shells out to jarsigner/keytool
# for the index signature) and apksigner (for the per-entry v1 signature in
# newer index formats) via with-nixpkgs.sh/nixpkgs-pin, next to this script.
set -euo pipefail

here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

if [ "$#" -lt 4 ]; then
  cat >&2 <<'USAGE'
usage: fdroid-update.sh <repo-root> <index-keystore> <key-alias> \
                         <keystore-pass-env> [<key-pass-env>] [extra fdroid update args...]

required environment variables: FDROID_REPO_URL, FDROID_REPO_NAME
optional environment variables: FDROID_REPO_DESCRIPTION,
                                FDROID_PREVIOUS_INDEX (local path to the
                                previously published index-v2.json, so
                                already-published builds keep the date they
                                were first added)
USAGE
  exit 2
fi

root=$1
keystore=$2
alias_name=$3
ks_pass_env=$4
shift 4

key_pass_env=$ks_pass_env
if [ "$#" -gt 0 ] && [[ $1 != -* ]]; then
  key_pass_env=$1
  shift
fi

extra_args=("$@")

[ -d "$root/repo" ] || { echo "fdroid-update.sh: no repo/ under $root" >&2; exit 2; }
[ -f "$keystore" ] || { echo "fdroid-update.sh: no such keystore: $keystore" >&2; exit 2; }
: "${!ks_pass_env:?fdroid-update.sh: environment variable $ks_pass_env (keystore password) is not set}"
: "${!key_pass_env:?fdroid-update.sh: environment variable $key_pass_env (key password) is not set}"

# Hard-required: an unset repo_url/repo_name used to mean fdroidserver fell
# back to its own placeholder template, which is a *successful* publish that
# advertises the wrong repo -- silently wrong, the worst outcome. Failing
# loudly, naming the variable, is strictly better than defaulting.
: "${FDROID_REPO_URL:?fdroid-update.sh: environment variable FDROID_REPO_URL (canonical repo URL, no trailing slash) is not set}"
: "${FDROID_REPO_NAME:?fdroid-update.sh: environment variable FDROID_REPO_NAME (repo display name) is not set}"

previous_index=${FDROID_PREVIOUS_INDEX:-}
if [ -n "$previous_index" ] && [ ! -f "$previous_index" ]; then
  echo "fdroid-update.sh: FDROID_PREVIOUS_INDEX is set to '$previous_index', which does not exist." >&2
  echo "fdroid-update.sh: leave it unset for a first-ever publish; do not point it at a file that was never fetched." >&2
  exit 2
fi

# YAML single-quoted scalar: doubling an embedded single quote is the only
# escape it needs, and (unlike double-quoted) it does not interpret
# backslashes -- so this is safe for an arbitrary URL/name/description
# without having to know what characters fdroidserver's own parser accepts.
yaml_squote() {
  local s=$1
  s=${s//\'/\'\'}
  printf "'%s'" "$s"
}

# fdroidserver's config.yml supports "{env: VARNAME}" in place of any string
# value specifically so a config that names a keystore and its passwords can
# still be free of secrets -- fdroidserver's own common.py: "it is a good
# idea to manage config.yml (WITHOUT PASSWORDS!) in git". Writing it here
# means a caller never has to hand-write fdroidserver's config format, and
# this repo -- not every consumer -- is the one place that has to track it if
# it changes.
{
  cat <<EOF
keystore: $keystore
keystorepass:
  env: $ks_pass_env
keypass:
  env: $key_pass_env
repo_keyalias: $alias_name
repo_url: $(yaml_squote "$FDROID_REPO_URL")
repo_name: $(yaml_squote "$FDROID_REPO_NAME")
EOF
  if [ -n "${FDROID_REPO_DESCRIPTION:-}" ]; then
    echo "repo_description: $(yaml_squote "$FDROID_REPO_DESCRIPTION")"
  fi
} > "$root/config.yml"
chmod 600 "$root/config.yml"

# Stash the previous index somewhere `fdroid update` cannot reach. install
# copies it *into* repo/, where the run then overwrites it -- so the "before"
# side of the verification has to be read from a copy taken beforehand. It
# also makes FDROID_PREVIOUS_INDEX=<root>/repo/index-v2.json (a caller
# re-running in place) work rather than being a copy onto itself.
previous_stash=
if [ -n "$previous_index" ]; then
  previous_stash=$(mktemp "${TMPDIR:-/tmp}/fdroid-previous-index.XXXXXX")
  trap 'rm -f "$previous_stash"' EXIT
  cp "$previous_index" "$previous_stash"
fi

echo "fdroid-update.sh: running fdroid update in $root"
# fdroidserver locates a JDK by globbing standard system install locations
# (or $JAVA_HOME) -- a nix store path matches neither, so JAVA_HOME has to be
# set explicitly from whichever jarsigner this nix shell actually put on
# PATH, rather than assumed.
#
# python3 is on the list for preserve_added.py: fdroidserver is a python
# application but its nix output only exposes `fdroid` on PATH, not the
# interpreter it runs on.
bash "$here/with-nixpkgs.sh" "fdroidserver jdk21 apksigner python3" bash -c '
  set -euo pipefail
  export JAVA_HOME="$(dirname "$(dirname "$(command -v jarsigner)")")"
  tools=$1; prev=$2; shift 2
  cd "$1"; shift
  if [ -n "$prev" ]; then
    python3 "$tools/preserve_added.py" install "$prev" repo
  fi
  fdroid update "$@"
  # After, not instead of: install only makes the dates *available* to
  # fdroidserver. This measures that they actually came out the other side,
  # so a future fdroidserver that stops consulting the old index fails the
  # publish here rather than silently flattening the history again
  # (weblib/weblib-viewer#95).
  if [ -n "$prev" ]; then
    python3 "$tools/preserve_added.py" verify "$prev" repo/index-v2.json
  fi
' bash "$here" "$previous_stash" "$root" "${extra_args[@]}"

echo "fdroid-update.sh: done; index at $root/repo/index-v2.json"