From 061d8b266cedfb33c3df8670577d551c3b9a736d Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 11:25:33 +0000 Subject: [PATCH] Seed the shared CI tools Split out of the four repos per weblib-archive#44. All three files were byte-identical across every repo at this moment, which will not stay true -- they converged only because four twin PRs landed within hours today, and report_job_log.py had already drifted once before that. Taken from weblib-archive, verified identical to every other copy first: with-nixpkgs.sh ca43fa20 (cfbypass, archive, fs) report_job_log.py aaef8f62 (cfbypass, archive) sync_blocked_label.py e6ddb21d (all four) action.yml is included so the `uses:` question can be re-measured now the repo is public; it did not work while private. Co-authored-by: bit --- README.md | 51 ++++++++++++ action.yml | 22 ++++++ report_job_log.py | 162 +++++++++++++++++++++++++++++++++++++ sync_blocked_label.py | 180 ++++++++++++++++++++++++++++++++++++++++++ with-nixpkgs.sh | 56 +++++++++++++ 5 files changed, 471 insertions(+) create mode 100644 action.yml create mode 100755 report_job_log.py create mode 100755 sync_blocked_label.py create mode 100755 with-nixpkgs.sh diff --git a/README.md b/README.md index 9504b98..b5d0043 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,53 @@ # weblib-ci +The CI scripts shared by [cfbypass], [weblib-archive], [weblib-fs] and +[weblib-viewer]. Split out per weblib-archive#44, where they had been +hand-copied into each repo and had already drifted once. + +**Public deliberately.** Nothing here is a secret or specific to the archive's +contents: a nixpkgs-pinning wrapper, a log poster and a label reconciler. +Public means a consumer needs no deploy key, no ssh setup and no secret to +fetch it — which was measured to be the difference between one step and three. + +## What is here + +| file | what it does | +|---|---| +| `with-nixpkgs.sh` | Runs a command with one nixpkgs package on PATH, pinned to the *consuming* repo's `flake.lock`. Avoids `nix shell nixpkgs#x`, which re-resolves the registry and refetches a channel tarball whenever the branch moves. | +| `report_job_log.py` | Posts the tail of a build log as a PR comment. Exists because `actions/jobs/{id}/logs` returns 500 for every id on Gitea 1.25.2, so a red job otherwise says only that it failed. | +| `sync_blocked_label.py` | Keeps `Status/Blocked` in step with Gitea's dependency graph. | + +All three are standard library / plain bash only. They are *run*, not built, so +this repo has no flake. + +## Using it + +`with-nixpkgs.sh` reads the **consuming** repo's `flake.lock` relative to the +working directory, so it keeps working from a subdirectory. + +```yaml +- uses: actions/checkout@v4 +- name: fetch the shared CI tools + run: git clone --depth 1 https://git.chaosbit.de/weblib/weblib-ci.git .ci +- run: bash .ci/with-nixpkgs.sh python3 python3 .ci/report_job_log.py /tmp/build.log +``` + +No credentials: the repo is public, which is the whole point of it being so. + +### Why not `uses:` + +`uses:` pointing at a repo on this instance was measured on weblib-archive#44 +and did not work, in either the bare `weblib/weblib-ci@main` form or with a +full URL — while a plain clone with the same access did. `action.yml` is kept +here so the question can be re-checked cheaply if the instance changes; the +README records the answer so nobody has to re-derive it. + +### Why not a flake input + +These are scripts a workflow runs, not derivations. A flake input would cost a +`flake.lock` bump in four repos every time one changes, and buys nothing. + +[cfbypass]: https://git.chaosbit.de/weblib/cfbypass +[weblib-archive]: https://git.chaosbit.de/weblib/weblib-archive +[weblib-fs]: https://git.chaosbit.de/weblib/weblib-fs +[weblib-viewer]: https://git.chaosbit.de/weblib/weblib-viewer diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..c8b2063 --- /dev/null +++ b/action.yml @@ -0,0 +1,22 @@ +# Makes the shared tools available to a workflow and reports where they are. +# +# Whether this is usable at all depends on `uses:` resolving a repo on this +# Gitea, which is instance configuration rather than something a repo controls. +# Measured on weblib-archive#44 while weblib-ci was still private: it did not +# work. Re-measured once the repo was public - see the README for the outcome +# and for the fallback, which is a plain clone and always works. +name: weblib CI tools +description: Shared CI scripts for the weblib repos. + +outputs: + path: + description: Directory holding with-nixpkgs.sh and the python tools. + value: ${{ github.action_path }} + +runs: + using: composite + steps: + - shell: bash + run: | + echo "weblib-ci tools at ${{ github.action_path }}" + test -f "${{ github.action_path }}/with-nixpkgs.sh" diff --git a/report_job_log.py b/report_job_log.py new file mode 100755 index 0000000..2ff5a18 --- /dev/null +++ b/report_job_log.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Post the tail of a build log as a pull request comment. + +Gitea 1.25.2 returns 500 from `actions/jobs/{id}/logs` for **every** id, so a +red job reports `Failing after 48s` and nothing else. Every hypothesis about +why then costs a push, a wait, and a one-bit answer. This puts the log where it +can be read, next to the change that caused it. + +Standard library only, like `sync_blocked_label.py`, so it needs nothing but an +interpreter. + +Usage, from a workflow step guarded by `if: failure()`: + + report_job_log.py /tmp/build.log + +Everything else comes from the Actions environment: `GITHUB_REPOSITORY`, +`GITHUB_EVENT_PATH` (for the PR number), `GITEA_TOKEN`/`GITHUB_TOKEN`, and +`GITHUB_SERVER_URL`. + +Remove this once the instance is updated and logs can be read directly. +""" + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.request + +DEFAULT_HOST = "https://git.chaosbit.de" + +# Gitea renders a comment body wholesale; a full nix log is megabytes and the +# failure is always at the end. +DEFAULT_TAIL = 12000 + +# nix colours its output, and the raw escapes are noise in Markdown. +ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") + + +def pull_request_number(event_path): + """The PR this job is running for, from the event payload, or None.""" + if not event_path or not os.path.exists(event_path): + return None + with open(event_path) as fh: + event = json.load(fh) + pr = event.get("pull_request") or {} + return pr.get("number") + + +def pull_request_for_branch(host, repo, token, branch): + """The open PR whose head is `branch`, or None. + + Only a `pull_request` event carries the PR in its payload. A + `workflow_dispatch` or `schedule` run does not, and the first version of + this script simply printed "not a pull request build" and exited 0 -- so a + dispatched run that failed stayed exactly as silent as the one this script + exists to fix. Looking the branch up covers those. + + A push to a branch with no PR still has nowhere to comment, which is + honest: there is no thread for it. + """ + if not branch: + return None + url = (f"{host.rstrip('/')}/api/v1/repos/{repo}/pulls" + f"?state=open&limit=50") + req = urllib.request.Request(url) + req.add_header("Authorization", f"token {token}") + try: + with urllib.request.urlopen(req, timeout=30) as res: + pulls = json.loads(res.read() or b"[]") + except (urllib.error.URLError, OSError, ValueError): + # A reporter that raises reports nothing. HTTPError is a URLError, and + # ValueError covers a body that is not JSON. + return None + for pull in pulls: + if ((pull.get("head") or {}).get("ref")) == branch: + return pull.get("number") + return None + + +def tail(path, limit): + with open(path, "rb") as fh: + try: + fh.seek(0, os.SEEK_END) + size = fh.tell() + fh.seek(max(0, size - limit * 4)) + except OSError: # not seekable; read it all + size = None + raw = fh.read() + text = ANSI.sub("", raw.decode("utf-8", "replace")) + if len(text) > limit: + text = text[-limit:] + # Do not start mid-line; it reads as corruption rather than truncation. + text = text.split("\n", 1)[-1] + text = "[…truncated…]\n" + text + return text + + +def comment(host, repo, number, token, body): + url = f"{host.rstrip('/')}/api/v1/repos/{repo}/issues/{number}/comments" + req = urllib.request.Request(url, data=json.dumps({"body": body}).encode(), + method="POST") + req.add_header("Content-Type", "application/json") + req.add_header("Authorization", f"token {token}") + with urllib.request.urlopen(req, timeout=30) as res: + return json.loads(res.read() or b"null") + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("log", help="path to the captured build log") + ap.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY")) + ap.add_argument("--host", default=os.environ.get("GITEA_HOST") + or os.environ.get("GITHUB_SERVER_URL") or DEFAULT_HOST) + ap.add_argument("--token", default=os.environ.get("GITEA_TOKEN") + or os.environ.get("GITHUB_TOKEN")) + ap.add_argument("--pr", type=int, default=None) + ap.add_argument("--job", default=os.environ.get("GITHUB_JOB") or "job") + ap.add_argument("--tail", type=int, default=DEFAULT_TAIL) + args = ap.parse_args(argv) + + if not args.repo or not args.token: + print("no repo or no token; set GITHUB_REPOSITORY and GITEA_TOKEN") + return 0 + + number = args.pr or pull_request_number(os.environ.get("GITHUB_EVENT_PATH")) + if number is None: + branch = (os.environ.get("GITHUB_HEAD_REF") + or os.environ.get("GITHUB_REF_NAME")) + number = pull_request_for_branch(args.host, args.repo, args.token, branch) + if number is None: + print("no pull request for this run: nothing to comment on") + return 0 + + if not os.path.exists(args.log): + text = ("The step produced no log file at " + f"`{args.log}` — it failed before the build started.") + else: + text = "```\n" + tail(args.log, args.tail).rstrip() + "\n```" + + sha = (os.environ.get("GITHUB_SHA") or "")[:8] + body = (f"**`{args.job}` failed**{f' on `{sha}`' if sha else ''}.\n\n" + "Job logs return 500 on this Gitea, so here is the tail of the " + "build output, posted by `tools/report_job_log.py`.\n\n" + f"{text}\n") + + # A failure to report a failure must not itself be silent, but it also must + # not mask the real one: the step is already `if: failure()`. + try: + result = comment(args.host, args.repo, number, args.token, body) + except urllib.error.HTTPError as e: + print(f"could not comment: {e.code} " + f"{e.read()[:300].decode('utf-8', 'replace')}", file=sys.stderr) + return 0 + print(f"reported to {args.repo}#{number}: {(result or {}).get('html_url')}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sync_blocked_label.py b/sync_blocked_label.py new file mode 100755 index 0000000..891cbd2 --- /dev/null +++ b/sync_blocked_label.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Keep `Status/Blocked` in step with Gitea's dependency graph. + +Gitea knows which issues block which, and will grey out a pull request's merge +button accordingly, but it does not surface that as a label -- so a list view +gives no hint that half of it cannot be started. This reconciles the two. + +Standard library only, so it runs in any runner image and can be driven by +hand. + +## The one rule worth arguing about + +**An issue with no dependencies at all is never touched.** Only issues that +have at least one dependency are managed: + +* any blocker not positively closed -> ensure the label is present +* every blocker closed -> ensure the label is absent + +The asymmetry is deliberate. A dependency in *another* repo is not fully +visible to the Actions token, which is scoped to one repo, so "not open" and +"closed" are not the same statement. + +That matters because `Status/Blocked` is also applied by hand for reasons the +graph knows nothing about -- weblib-archive#29 and #30 are blocked on a +decision, not on an issue. A rule of "no open blockers means not blocked" +would strip those the first time it ran. Anything with a dependency edge is +fair game because the edge is the statement of intent; anything without one is +somebody's judgement and stays. + +Usage: + sync_blocked_label.py --repo weblib/cfbypass [--repo weblib/weblib-archive] + [--dry-run] [--label Status/Blocked] + +Reads the token from --token, else $GITEA_TOKEN, else $GITHUB_TOKEN. +""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +DEFAULT_LABEL = "Status/Blocked" +DEFAULT_HOST = "https://git.chaosbit.de" + + +class Forge: + def __init__(self, base, token): + self.base = base.rstrip("/") + "/api/v1" + self.token = token + + def _call(self, method, path, body=None): + url = f"{self.base}/{path.lstrip('/')}" + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Content-Type", "application/json") + req.add_header("Authorization", f"token {self.token}") + try: + with urllib.request.urlopen(req, timeout=30) as res: + raw = res.read() + return json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + raise SystemExit(f"{method} {path} failed: {e.code} {e.read()[:200].decode('utf-8', 'replace')}") + + def get(self, path): + return self._call("GET", path) + + def post(self, path, body): + return self._call("POST", path, body) + + def put(self, path, body): + return self._call("PUT", path, body) + + +def label_id(forge, repo, name): + """The label's id, or None if this repo has no such label. + + Returning None rather than exiting matters when several repos are passed: + aborting on the third would leave the first two already modified, which is + a worse state than doing nothing. weblib-viewer has no labels at all. + """ + for label in forge.get(f"repos/{repo}/labels?limit=100"): + if label["name"] == name: + return label["id"] + return None + + +def open_items(forge, repo): + """Open issues *and* pull requests. They share the /issues endpoint.""" + out = [] + page = 1 + while True: + batch = forge.get(f"repos/{repo}/issues?state=open&limit=50&page={page}") + if not batch: + break + out.extend(batch) + if len(batch) < 50: + break + page += 1 + return out + + +def reconcile(forge, repo, label_name, dry_run): + lid = label_id(forge, repo, label_name) + if lid is None: + print(f" skipped: no {label_name!r} label in this repo", flush=True) + return [] + changed = [] + for item in open_items(forge, repo): + number = item["number"] + deps = forge.get(f"repos/{repo}/issues/{number}/dependencies") or [] + if not deps: + # No edges, no opinion. See the docstring. + continue + # Fail safe: only an *explicitly closed* dependency counts as done. + # A cross-repo blocker is not fully visible to the Actions token, which + # is scoped to one repo -- weblib-archive#30 is blocked by + # weblib/cfbypass#1, and the runner read that entry as not-open and + # took the label off while cfbypass#1 was still open. Anything whose + # state we cannot positively confirm keeps the issue blocked. + blockers = [d for d in deps if d.get("state") != "closed"] + labelled = any(l["name"] == label_name for l in item.get("labels") or []) + if blockers and not labelled: + action, detail = "add", f"blocked by {', '.join('#%d' % d['number'] for d in blockers)}" + if not dry_run: + forge.post(f"repos/{repo}/issues/{number}/labels", {"labels": [lid]}) + elif not blockers and labelled: + action, detail = "remove", "every blocker is closed" + if not dry_run: + # PUT the whole set minus this label, rather than + # DELETE .../labels/{id}. Under Gitea Actions the automatic + # token can *add* an issue label but not delete one -- the + # DELETE fails whatever `permissions:` the workflow declares, + # while the replace succeeds. Measured on 1.25.2; a run by + # hand with a personal token works either way, so this only + # ever showed up in CI. + keep = [l["id"] for l in item.get("labels") or [] + if l["name"] != label_name] + forge.put(f"repos/{repo}/issues/{number}/labels", {"labels": keep}) + else: + continue + changed.append((number, action, detail)) + print(f" {'would ' if dry_run else ''}{action:<6} {label_name} on {repo}#{number}" + f" ({detail})", flush=True) + return changed + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repo", action="append", default=[], + help="owner/name; repeatable. Defaults to $GITHUB_REPOSITORY.") + ap.add_argument("--host", default=os.environ.get("GITEA_HOST") or DEFAULT_HOST) + ap.add_argument("--token", default=os.environ.get("GITEA_TOKEN") + or os.environ.get("GITHUB_TOKEN")) + ap.add_argument("--label", default=DEFAULT_LABEL) + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args(argv) + + repos = args.repo or ([os.environ["GITHUB_REPOSITORY"]] + if os.environ.get("GITHUB_REPOSITORY") else []) + if not repos: + ap.error("no --repo given and GITHUB_REPOSITORY is not set") + if not args.token: + ap.error("no token: pass --token or set GITEA_TOKEN/GITHUB_TOKEN") + + forge = Forge(args.host, args.token) + total = 0 + for repo in repos: + print(f"{repo}:", flush=True) + total += len(reconcile(forge, repo, args.label, args.dry_run)) + print(f"--> {total} change(s){' (dry run, nothing applied)' if args.dry_run else ''}", + flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/with-nixpkgs.sh b/with-nixpkgs.sh new file mode 100755 index 0000000..eb5f5a7 --- /dev/null +++ b/with-nixpkgs.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Run a command with one nixpkgs package on PATH, taken from *this repo's* +# flake.lock. +# +# bash .gitea/with-nixpkgs.sh python3 python3 tools/sync_blocked_label.py … +# bash .gitea/with-nixpkgs.sh openssh nix build .#checks.x86_64-linux.tests +# +# **Call it as `bash