fdroid-repo-ci/preserve_added.py

#!/usr/bin/env python3
"""Carry an F-Droid repo's per-build "added" timestamps across a rebuild.

weblib/weblib-viewer#95: every version in the published index claimed it was
added at the moment of the *last* publish, so a client's version list showed
five builds all "added" within seconds of each other.

## Why that happens (fdroidserver 2.4.3, verified in the source)

`fdroid update` does not read an APK's mtime, and does not invent the date
either. It asks `common.KnownApks`, whose `__init__` loads the dates out of
an *already existing* index:

    for part in ('repo', 'archive'):
        path = os.path.join(part, 'index-v2.json')
        if os.path.isfile(path):
            ...
            filename = version["file"]["name"][1:]
            date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc)
            self.apks[filename] = date

and whose `recordapk` falls back to the wall clock for anything it has not
seen before:

    if apkName not in self.apks:
        if default_date is None:
            default_date = datetime.now(timezone.utc)
        self.apks[apkName] = default_date
    return self.apks[apkName]

(`default_date` is only non-None under `--use-date-from-apk`, which uses the
local file's mtime -- no better here, since a re-downloaded file's mtime is
the time it was downloaded.)

So the per-build `added` date is stored *in the published index and nowhere
else* -- `KnownApks`' own docstring says as much: "This is currently the only
way to permanently store the 'updated' date of APKs."

A CI job that rebuilds `repo/` from scratch on a fresh runner -- which is
exactly what a retention scheme re-downloading the previous APKs does --
starts with no `repo/index-v2.json`, so `KnownApks` starts empty and *every*
build, old ones included, gets `datetime.now()`. Hence "all apps show the
date of the last update".

## The fix

Put last publish's `index-v2.json` back into `repo/` before `fdroid update`
runs, and let fdroidserver's own mechanism do the rest. Nothing here writes
a date itself; `install` stages the previous index and `verify` afterwards
*measures* that the dates actually survived, so a future fdroidserver that
drops this behaviour (its own `read_added_date_from_all_apks` carries a TODO
about reading the dates differently) fails the publish loudly instead of
silently flattening the history again.

    preserve_added.py install <previous-index-v2.json> <repo-dir>
    preserve_added.py verify  <previous-index-v2.json> <new-index-v2.json>

Builds are keyed on the index's own `file.name`, because that is the key
`KnownApks` itself uses -- not the versionCode, which is one level deeper in
the JSON and is not what fdroidserver matches on.
"""

import argparse
import json
import os
import shutil
import sys


def load_index(path, what):
    """Parse an index-v2.json, rejecting anything KnownApks would choke on.

    KnownApks indexes straight into `index["packages"]` and into each
    version's `file.name`/`added` with no guards at all, so a truncated or
    wrong-shaped file does not degrade -- it aborts `fdroid update` from
    inside a library, with a traceback that says nothing about where the
    file came from. Checking the shape here costs nothing and turns that
    into one sentence naming the actual path.
    """
    try:
        with open(path, "r", encoding="utf-8") as f:
            index = json.load(f)
    except FileNotFoundError:
        sys.exit(f"preserve_added.py: no such {what}: {path}")
    except json.JSONDecodeError as e:
        sys.exit(f"preserve_added.py: {what} {path} is not valid JSON: {e}")

    if not isinstance(index, dict) or not isinstance(index.get("packages"), dict):
        sys.exit(
            f"preserve_added.py: {what} {path} has no top-level 'packages' object "
            "-- it does not look like an index-v2.json"
        )
    return index


def added_by_file(index):
    """Map {apk file name -> added (ms)} exactly as KnownApks keys it.

    Entries with no `added`, or no `file.name`, are skipped rather than
    defaulted: a missing date is not a date, and inventing one here would be
    the very bug this script exists to stop.
    """
    out = {}
    for pkg in index["packages"].values():
        if not isinstance(pkg, dict):
            continue
        for version in (pkg.get("versions") or {}).values():
            if not isinstance(version, dict):
                continue
            name = (version.get("file") or {}).get("name")
            added = version.get("added")
            if isinstance(name, str) and isinstance(added, int):
                out[name.lstrip("/")] = added
    return out


def cmd_install(args):
    previous = load_index(args.previous_index, "previous index")
    dates = added_by_file(previous)

    if not os.path.isdir(args.repo_dir):
        sys.exit(f"preserve_added.py: no such repo directory: {args.repo_dir}")

    target = os.path.join(args.repo_dir, "index-v2.json")
    shutil.copyfile(args.previous_index, target)
    print(
        f"preserve_added.py: staged {args.previous_index} at {target}; "
        f"fdroid update will carry the 'added' date of {len(dates)} known build(s)"
    )
    return 0


def cmd_verify(args):
    previous = added_by_file(load_index(args.previous_index, "previous index"))
    new = added_by_file(load_index(args.new_index, "new index"))

    carried, drifted = [], []
    for name, was in previous.items():
        if name not in new:
            # Aged out of the retention window, or otherwise no longer
            # published. Not this script's business.
            continue
        # Whole seconds, deliberately. fdroidserver stores `added` in
        # milliseconds but only ever *produces* whole seconds
        # (index.py: `int(calendar.timegm(obj.timetuple()) * 1000)`), and
        # KnownApks reads it back as `version["added"] // 1000`. A
        # hand-made previous index carrying a sub-second value would
        # therefore come back rounded, through no fault of the mechanism
        # being checked -- failing a publish over that would be a false
        # alarm, while every real regression here is seconds to hours.
        if new[name] // 1000 == was // 1000:
            carried.append(name)
        else:
            drifted.append((name, was, new[name]))

    for name, was, now in sorted(drifted):
        print(
            f"preserve_added.py: {name} was added at {was}, "
            f"the new index says {now}",
            file=sys.stderr,
        )
    fresh = sorted(set(new) - set(previous))
    for name in fresh:
        print(f"preserve_added.py: {name} is new in this index (added {new[name]})")

    if drifted:
        print(
            f"preserve_added.py: {len(drifted)} already-published build(s) lost their "
            "original 'added' timestamp (weblib/weblib-viewer#95)",
            file=sys.stderr,
        )
        return 1

    print(
        f"preserve_added.py: {len(carried)} already-published build(s) kept their "
        f"original 'added' timestamp, {len(fresh)} new"
    )
    return 0


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    sub = parser.add_subparsers(dest="command", required=True)

    p_install = sub.add_parser(
        "install",
        help="stage a previously published index-v2.json in repo/ so "
        "`fdroid update` reuses its per-build 'added' dates",
    )
    p_install.add_argument("previous_index")
    p_install.add_argument("repo_dir")
    p_install.set_defaults(func=cmd_install)

    p_verify = sub.add_parser(
        "verify",
        help="fail if any build present in both indexes changed its "
        "'added' timestamp",
    )
    p_verify.add_argument("previous_index")
    p_verify.add_argument("new_index")
    p_verify.set_defaults(func=cmd_verify)

    args = parser.parse_args(argv)
    return args.func(args)


if __name__ == "__main__":
    sys.exit(main())