From 099a48f2b54050f5da1ac4675274aba4ab113239 Mon Sep 17 00:00:00 2001 From: Dave Kowalski Date: Sat, 29 Aug 2026 21:21:47 +0000 Subject: [PATCH 1/2] feat(delivery): publish multiple surfaces Authored-By: OpenAI (GPT-5) --- .gitea/workflows/test.yaml | 14 ++ README.md | 207 ++++++++++-------- action.yaml | 8 +- scripts/build.py | 8 +- scripts/deploy.py | 267 +++++++++++++---------- scripts/setup.py | 5 +- scripts/utils.py | 369 ++++++++++++++++++++++++-------- templates/ingress.yaml.j2 | 14 +- templates/kustomization.yaml.j2 | 6 +- templates/service.yaml.j2 | 8 +- tests/__init__.py | 1 + tests/fixtures/v2-site.yaml | 36 ++++ tests/test_site_publish.py | 192 +++++++++++++++++ 13 files changed, 826 insertions(+), 309 deletions(-) create mode 100644 .gitea/workflows/test.yaml create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/v2-site.yaml create mode 100644 tests/test_site_publish.py diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml new file mode 100644 index 0000000..0fb71e7 --- /dev/null +++ b/.gitea/workflows/test.yaml @@ -0,0 +1,14 @@ +name: Test +on: + push: + branches: [main] + pull_request: +jobs: + test: + runs-on: fritzlab + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: python3 -m pip install --quiet jinja2 pyyaml awscli + - name: Test + run: python3 -m unittest discover -v diff --git a/README.md b/README.md index 6873418..6941f14 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,117 @@ # action/site-publish -Composite Gitea Action that publishes a **static-content** website to the -fritzlab k8s cluster. Supports `static`, `hugo`, and `mkdocs`. Content goes -to a Garage S3 bucket; Traefik fronts the bucket via an `ExternalName` -Service with cert-manager TLS. +Composite Gitea Action that publishes static, Hugo, and MkDocs output to Garage +and writes the matching Argo CD, Kubernetes, Traefik, and cert-manager resources +to `fritzlab/apps`. -> **Containerized web apps (Dockerfile-based) are NOT handled here.** Use the -> standard image-producer chain instead: -> [`action/image-build`](https://code.fritzlab.net/action/image-build) + -> [`action/image-push`](https://code.fritzlab.net/action/image-push) + -> [`action/image-deploy`](https://code.fritzlab.net/action/image-deploy). -> Hand-author the apps-repo manifests once (Deployment, Service, Ingress, -> Certificate, kustomization with `images:` block) and let `image-deploy` -> pin the tag on every push. See `sjc001/websites/rainsounds.vino.network/` -> for the canonical example. site-publish errors out explicitly if -> `site.yaml` has `type: docker`. +Containerized applications use `action/image-build`, `action/image-push`, and +`action/image-deploy`. `type: docker` fails validation. -## Convention +## Delivery contracts -Bucket name = repo name = canonical domain. Sibling hostnames (e.g. `www.`, -`ipv6.`) are declared as `aliases:` in `site.yaml` — the action registers each -as a Garage `globalAlias` on the bucket and adds it to the Ingress + Certificate -on every deploy. Manual edits to manifests in the apps repo are clobbered; -edit `site.yaml` instead. +There are two explicit internal modes: -## Usage +- `single-surface-v1` is selected only when `schema`, `artifacts`, and `routes` are all + absent. It preserves the original one-repository/one-bucket/one-`/` route + behavior, existing action inputs, aliases, middleware, and revalidated cache. +- `v2` is selected by `schema: v2`. It requires named artifacts and routes and + validates the entire access/storage graph before any upload or manifest write. -Scaffold a new site (handles repo creation + Garage bucket): +Never mix the modes. A partial v2 declaration fails closed. -```sh -./new-site.sh --name my-site.vino.network --domain my-site.vino.network --type static +### Multi-surface schema + +This example publishes immutable releases and revalidated channel pointers +anonymously while protecting the portal catch-all with Authentik: + +```yaml +schema: v2 +domain: baseline.fritzlab.net +type: static +content_dir: output + +artifacts: + releases: + source: dist/releases + bucket: baseline-releases + credentials: + access_key_env: BASELINE_RELEASES_S3_ACCESS_KEY + secret_key_env: BASELINE_RELEASES_S3_SECRET_KEY + cache: immutable-release + channels: + source: dist/channels + bucket: baseline-channels + credentials: + access_key_env: BASELINE_CHANNELS_S3_ACCESS_KEY + secret_key_env: BASELINE_CHANNELS_S3_SECRET_KEY + cache: revalidated-channel + portal: + source: portal + bucket: baseline-portal + credentials: + access_key_env: BASELINE_PORTAL_S3_ACCESS_KEY + secret_key_env: BASELINE_PORTAL_S3_SECRET_KEY + cache: private + +routes: + - path: /dist/releases + artifact: releases + access: public + - path: /dist/channels + artifact: channels + access: public + - path: / + artifact: portal + access: authenticated + middlewares: [authentik-forwardauth] ``` -Or do it manually. `site.yaml`: +Artifact `source` is relative to `build/html` after the build phase. Credential +fields name inherited environment variables; secret values never belong in +`site.yaml`. Callers expose those variables to the composite action through the +workflow `env` contract. + +Cache policies are intentionally closed: + +| Policy | Header | Publication behavior | +|---|---|---| +| `immutable-release` | `public, max-age=31536000, immutable` | write once; identical SHA-256 retry skips; changed key fails | +| `revalidated-channel` | `public, max-age=0, must-revalidate` | replace and delete stale keys | +| `private` | `private, no-store` | replace and delete stale keys | + +Validation rejects unknown keys or references, unsafe paths, duplicate route +prefixes, public `/`, authenticated routes without middleware, private/public +cache mismatches, unrouted artifacts, an artifact routed twice, and bucket or +publication credential reuse between artifacts. V2 also requires an explicit +`/` access policy. Routes render deterministically by +longest prefix. + +Each route receives a distinct ExternalName Service and Ingress. The Service +targets `.web.sjc001.fritzlab.net` with Traefik host forwarding disabled, +so Garage selects that route's bucket. Objects are stored under their route +prefix, avoiding a path-rewrite middleware. Legacy Services directly target the +live `garage-s3.storage.svc.k8s.sjc001.fritzlab.net:80` endpoint. All routes share +one Certificate. + +## Legacy usage + +Existing callers remain valid: ```yaml domain: my-site.vino.network -type: static # static | hugo | mkdocs -# content_dir: html # subdirectory containing content (default: repo root) -# aliases: # additional hostnames (each gets a globalAlias on the bucket) -# - www.my-site.vino.network -# tidy: true # set false to skip HTML tidy -# enabled: true # set false to decommission -# excludes: # paths/patterns to skip during sync (relative to bucket root). -# - welcome/welcome.pdf -# # These are passed verbatim to `aws s3 sync --exclude`, -# # so they're both un-uploaded AND un-deleted. Use this -# # for large assets managed out-of-band via aws-cli -# # (e.g. media files updated more often than the site code). -# middlewares: # extra Traefik FILE-PROVIDER middleware names appended to the -# - authentik-forwardauth # Ingress annotation (after https-redirect,retry-upstream). -# # The middleware must already exist in the traefik-dynamic -# # ConfigMap. Use authentik-forwardauth to auth-gate a site -# # (also requires an Authentik proxy provider + app for the host). +type: static +content_dir: html +# aliases: [www.my-site.vino.network] +# tidy: true +# enabled: true +# excludes: [welcome/welcome.pdf] +# middlewares: [authentik-forwardauth] ``` -`.gitea/workflows/publish.yaml`: +Legacy bucket name is the repository name. Aliases are reconciled as Garage +global aliases and require `garage-admin-token`; reconciliation errors stop the +deploy. Disabling a site removes generated Apps manifests but never deletes its +bucket or objects. ```yaml name: Publish @@ -75,58 +131,33 @@ jobs: garage-admin-token: ${{ secrets.GARAGE_ADMIN_TOKEN }} ``` -DNS: subdomains of `vino.network` are covered by the wildcard CNAME to -`traefik.edge.svc…`. For other zones, add an explicit CNAME: - -``` -my-site.fritzlab.net 300 IN CNAME traefik.edge.svc.k8s.sjc001.fritzlab.net. -``` - ## Inputs | Input | Required | Default | Description | |---|---|---|---| -| `token` | yes | | Gitea token for apps repo push | -| `s3-access-key` | yes | | Garage `ci-deploy-key` access key id | -| `s3-secret-key` | yes | | Garage `ci-deploy-key` secret key | -| `s3-endpoint` | no | `http://garage.storage.svc:3900` | Garage S3 endpoint | -| `garage-admin-token` | only if site has `aliases` | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) | -| `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint | +| `token` | yes | | Gitea token used for the Apps repository | +| `s3-access-key` | single-surface-v1 | | Legacy Garage access key | +| `s3-secret-key` | single-surface-v1 | | Legacy Garage secret key | +| `s3-endpoint` | no | `http://garage-s3.storage.svc:3900` | Garage S3 API | +| `garage-admin-token` | with legacy aliases | | Garage admin token | +| `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API | | `username` | no | `ci-bot` | Gitea username | -Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`, -`GARAGE_S3_SECRET_KEY`, `GARAGE_ADMIN_TOKEN`. +Git HTTPS authentication is supplied to Git through a short-lived inherited +file descriptor. The token isn't placed in argv, command output, or a clone URL. -## Tools +## Generated topology -- **`new-site.sh`** — create a new site: Gitea repo, Garage bucket, web hosting enabled. -- **`scripts/publish.py decommission `** — remove a site's manifests from apps repo. Bucket purge is manual. - -## Architecture - -``` -push to websites/ - → CI runs site-publish action - → reads site.yaml, builds content (static copy / hugo / mkdocs), runs tidy - → aws s3 sync → Garage bucket named after the repo - → admin API: ensures every alias from site.yaml is a globalAlias on the bucket - → renders manifests in fritzlab/apps from templates: ExternalName Service → - garage.storage.svc, Traefik Ingress (canonical + aliases), cert-manager - Certificate (canonical + aliases as SANs), kustomization - → commits + pushes apps repo only if diff is non-empty - → ArgoCD syncs → site live with TLS +```text +push + -> build/html + -> validate complete site.yaml graph + -> publish each artifact with its cache/write policy and credential pair + -> one Certificate + -> N bucket-specific Services + longest-prefix Ingresses + -> commit changed manifests to fritzlab/apps + -> Argo CD reconciliation ``` -The Ingress + Certificate are re-rendered on every deploy from `site.yaml`. -There is no "first-deploy vs. update" branching — every deploy is idempotent. - -No nginx pods, no per-site Docker images. Garage matches `Host:` header to -bucket name (or any of its globalAliases), so every site shares a single -ExternalName target. - -## History - -- 2026-05-06: removed `type: docker` support. The single docker site - (`rainsounds.vino.network`) migrated to the `image-*` chain. site-publish - is now scoped strictly to static-content sites. -- 2026-05-06: renamed from `fritzlab/publish-site` → `action/site-publish`. +Manual changes to generated Apps resources are overwritten. `new-site.sh` +continues to scaffold legacy one-surface sites. diff --git a/action.yaml b/action.yaml index 505170e..86cd8e3 100644 --- a/action.yaml +++ b/action.yaml @@ -5,11 +5,11 @@ inputs: description: Gitea token (ci-bot) for apps repo push and API operations required: true s3-access-key: - description: Garage ci-deploy-key access key id - required: true + description: Garage access key id (required by single-surface-v1; v2 uses artifact credential env selectors) + required: false s3-secret-key: - description: Garage ci-deploy-key secret access key - required: true + description: Garage secret key (required by single-surface-v1; v2 uses artifact credential env selectors) + required: false s3-endpoint: # Targets garage-s3 (data-only Service) so requests do not round-robin onto # the gateway pod, whose emptyDir-backed metadata view intermittently diff --git a/scripts/build.py b/scripts/build.py index 14d933a..9cec420 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -34,11 +34,11 @@ def build_static(site_dir, cfg): elif cfg["type"] == "hugo": print(f"Building Hugo site from {src}") - run(f"hugo --source {src} --destination {html_dir}") + run(["hugo", "--source", str(src), "--destination", str(html_dir)]) elif cfg["type"] == "mkdocs": print(f"Building MkDocs site from {src}") - run(f"cd {src} && mkdocs build -d {html_dir}") + run(["mkdocs", "build", "-d", str(html_dir)], cwd=src) if cfg.get("tidy", True): print("Running tidy on HTML files...") @@ -56,7 +56,9 @@ def build_static(site_dir, cfg): def cmd_build(): site_dir = Path(env("SITE_DIR")) - cfg = parse_site_yaml(site_dir) + site_repo = env("SITE_REPO", "") + site_name = site_repo.split("/", 1)[1] if "/" in site_repo else None + cfg = parse_site_yaml(site_dir, site_name) if not cfg["enabled"]: print("Site disabled — skipping build") diff --git a/scripts/deploy.py b/scripts/deploy.py index 7b6cbc7..b74a721 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -1,71 +1,132 @@ -"""Deploy phase — S3 sync, manifest rendering, alias reconcile.""" +"""Deploy phase — artifact publication and split-route manifest rendering.""" +import fnmatch +import hashlib import json +import mimetypes import os -import shlex import shutil -import tempfile +import subprocess from pathlib import Path from urllib.error import HTTPError, URLError +from urllib.parse import quote from urllib.request import Request, urlopen +try: + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials +except ModuleNotFoundError: # aws-cli v1's bundled botocore layout + from awscli.botocore.auth import S3SigV4Auth + from awscli.botocore.awsrequest import AWSRequest + from awscli.botocore.credentials import Credentials + from utils import ( - DEFAULT_S3_ENDPOINT, - GITEA_HOST, - NAMESPACE, - clone_apps, - commit_and_push, - die, - env, - k8s_name, - parse_site_yaml, - render_templates, - run, + DEFAULT_S3_ENDPOINT, GARAGE_WEBSITE_HOST, NAMESPACE, clone_apps, + commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run, ) GARAGE_ADMIN_ENDPOINT = os.environ.get( - "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903" -) + "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903") -CACHE_CONTROL = "public, max-age=0, must-revalidate" +def _aws_env(artifact): + credentials = artifact["credentials"] + access = env(credentials["access_key_env"]) + secret = env(credentials["secret_key_env"]) + child = os.environ.copy() + child.update({"AWS_ACCESS_KEY_ID": access, "AWS_SECRET_ACCESS_KEY": secret, + "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001")}) + return child -def s3_sync(site_name, site_dir, excludes=None): +def _aws(endpoint, operation): + return ["aws", "--endpoint-url", endpoint, *operation] + + +def _excluded(relative, patterns): + return any(fnmatch.fnmatch(relative, pattern) for pattern in patterns) + + +def _conditional_put(endpoint, artifact, key, path, content_type, digest): + """Atomically create one S3 object without exposing signing credentials.""" + selectors = artifact["credentials"] + credentials = Credentials(env(selectors["access_key_env"]), + env(selectors["secret_key_env"])) + region = os.environ.get("AWS_DEFAULT_REGION", "sjc001") + url = f"{endpoint.rstrip('/')}/{quote(artifact['bucket'], safe='')}/{quote(key)}" + data = path.read_bytes() + request = AWSRequest(method="PUT", url=url, data=data, headers={ + "Content-Type": content_type, + "Cache-Control": artifact["cache_control"], + "x-amz-meta-sha256": digest, + "If-None-Match": "*", + }) + S3SigV4Auth(credentials, "s3", region).add_auth(request) + signed = Request(url, data=data, method="PUT", headers=dict(request.headers.items())) + try: + with urlopen(signed) as response: + response.read() + except HTTPError as error: + if error.code == 412: + die(f"immutable artifact {artifact['name']}: concurrent object creation: {key}; " + "rerun to verify identical content") + die(f"immutable artifact {artifact['name']}: upload failed for {key}: {error}") + + +def _immutable_sync(source, artifact, endpoint, excludes): + """Publish write-once objects; identical retries are no-ops.""" + aws_env = _aws_env(artifact) + bucket = artifact["bucket"] + for path in sorted(item for item in source.rglob("*") if item.is_file()): + relative = path.relative_to(source).as_posix() + if _excluded(relative, excludes): + continue + key = "/".join(part for part in (artifact.get("key_prefix"), relative) if part) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + head = subprocess.run( + _aws(endpoint, ["s3api", "head-object", "--bucket", bucket, "--key", key]), + env=aws_env, check=False, text=True, capture_output=True) + if head.returncode == 0: + metadata = json.loads(head.stdout) + if (metadata.get("Metadata") or {}).get("sha256") != digest: + die(f"immutable artifact {artifact['name']}: object changed: {key}") + if metadata.get("CacheControl") != artifact["cache_control"]: + die(f"immutable artifact {artifact['name']}: cache metadata changed: {key}") + print(f" Immutable object unchanged: s3://{bucket}/{key}") + continue + missing = head.stderr.lower() + if not any(marker in missing for marker in ("404", "not found", "nosuchkey")): + die(f"immutable artifact {artifact['name']}: head-object failed for {key}: " + f"{head.stderr.strip()}") + content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + _conditional_put(endpoint, artifact, key, path, content_type, digest) + + +def _replaceable_sync(source, artifact, endpoint, excludes): + aws_env = _aws_env(artifact) + prefix = artifact.get("key_prefix") + destination = f"s3://{artifact['bucket']}/{prefix + '/' if prefix else ''}" + exclude_args = [part for pattern in excludes for part in ("--exclude", pattern)] + common = ["--only-show-errors", "--cache-control", artifact["cache_control"], + *exclude_args] + run(_aws(endpoint, ["s3", "sync", f"{source}/", destination, + "--delete", *common]), env=aws_env) + run(_aws(endpoint, ["s3", "cp", f"{source}/", destination, + "--recursive", *common]), env=aws_env) + + +def publish_artifact(build_root, artifact, excludes=None): endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT) - html_dir = site_dir / "build" / "html" - if not html_dir.exists(): - die(f"build/html not found — did the build step run? ({html_dir})") - env("AWS_ACCESS_KEY_ID") - env("AWS_SECRET_ACCESS_KEY") - os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001") - # `excludes` are patterns (site.yaml `excludes:` list) that should never - # be uploaded *and* should never be deleted from the bucket — escape hatch - # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli). - exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or [])) - if excludes: - print(f"Excluding patterns: {excludes}") - print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}") - # `sync --delete` handles new/changed/orphaned files. `cp --recursive` - # then re-uploads everything to refresh metadata (cache-control, - # content-type) on objects sync skipped because nothing changed. - # Cost: a no-op deploy still re-uploads every byte. Sites here are - # small enough that that's free; correctness wins over throughput. - # AWS CLI guesses Content-Type from file extension on local→S3 uploads, - # so a fresh upload always carries the right MIME type. - run( - f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ " - f"--delete --only-show-errors " - f"--cache-control '{CACHE_CONTROL}' " - f"{exclude_flags}".rstrip() - ) - print("Re-stamping metadata on all objects...") - run( - f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ " - f"--recursive --only-show-errors " - f"--cache-control '{CACHE_CONTROL}' " - f"{exclude_flags}".rstrip() - ) + source = (build_root / artifact["source"]).resolve() + if not source.is_relative_to(build_root.resolve()) or not source.is_dir(): + die(f"artifact {artifact['name']}: source directory not found: {source}") + excludes = excludes or [] + print(f"Publishing {artifact['name']} from {source} to s3://{artifact['bucket']}") + if artifact["immutable"]: + _immutable_sync(source, artifact, endpoint, excludes) + else: + _replaceable_sync(source, artifact, endpoint, excludes) def garage_admin(method, path, token, body=None): @@ -74,92 +135,83 @@ def garage_admin(method, path, token, body=None): headers = {"Authorization": f"Bearer {token}"} if data is not None: headers["Content-Type"] = "application/json" - req = Request(url, data=data, method=method, headers=headers) - with urlopen(req) as resp: - raw = resp.read() + request = Request(url, data=data, method=method, headers=headers) + with urlopen(request) as response: + raw = response.read() return json.loads(raw) if raw else {} def ensure_bucket_aliases(site_name, aliases, admin_token): - """Add cfg['aliases'] as Garage globalAliases on the site bucket. - - Idempotent: skips aliases already present. Never removes aliases not in - the desired set (safety — orphan removal is manual). - """ + """Fail closed while reconciling legacy Garage global aliases.""" if not aliases: return if not admin_token: - print(" (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)") - return - + die("GARAGE_ADMIN_TOKEN is required when aliases are configured") try: - info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}", + info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={quote(site_name)}", admin_token) - except (HTTPError, URLError) as e: - print(f" WARNING: bucket lookup failed: {e}") - return - + except (HTTPError, URLError) as error: + die(f"bucket lookup failed: {error}") bucket_id = info.get("id") + if not bucket_id: + die(f"bucket lookup returned no id for {site_name}") existing = set(info.get("globalAliases") or []) - print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}") - for alias in aliases: - if alias in existing: - continue - print(f" Adding globalAlias: {alias}") - try: + if alias not in existing: garage_admin("POST", "/v2/AddBucketAlias", admin_token, {"bucketId": bucket_id, "globalAlias": alias}) - except HTTPError as e: - body = e.read().decode(errors="replace") if hasattr(e, "read") else "" - print(f" ERROR adding alias {alias}: {e} {body}") - raise def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg): - """Always re-render manifests from current site.yaml. Templates own - domain + aliases, so changes propagate without manual edits.""" - manifests_dir.mkdir(parents=True, exist_ok=True) - template_vars = { + if manifests_dir.exists(): + shutil.rmtree(manifests_dir) + routes = [] + for route in cfg["routes"]: + artifact = cfg["artifacts"][route["artifact"]] + routes.append({ + **route, + "artifact_config": artifact, + "resource_name": k8s_name(site_name, route["artifact"]), + "backend_host": (GARAGE_WEBSITE_HOST if cfg["compatibility"] else + f"{artifact['bucket']}.web.sjc001.fritzlab.net"), + "pass_host_header": cfg["compatibility"], + }) + render_templates(action_dir, { "site": site_name, "site_k8s": k8s_name(site_name), "domain": cfg["domain"], "aliases": cfg["aliases"], "namespace": NAMESPACE, - "middlewares": cfg["middlewares"], - } - render_templates(action_dir, template_vars, app_dir, manifests_dir) + "website_host": GARAGE_WEBSITE_HOST, + "routes": routes, + }, app_dir, manifests_dir) def deploy_static(site_name, site_dir, action_dir, token, cfg): - s3_sync(site_name, site_dir, excludes=cfg.get("excludes")) - ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")) + build_root = (site_dir / "build" / "html").resolve() + if not build_root.is_dir(): + die(f"build/html not found — did the build step run? ({build_root})") + for artifact in cfg["artifacts"].values(): + publish_artifact(build_root, artifact, cfg.get("excludes")) + if cfg["compatibility"]: + ensure_bucket_aliases(site_name, cfg["aliases"], + os.environ.get("GARAGE_ADMIN_TOKEN")) apps_dir = clone_apps(token) app_dir = apps_dir / "sjc001" / "websites" / site_name - manifests_dir = app_dir / "manifests" - - render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg) - - commit_and_push(apps_dir, f"Deploy {site_name}") + render_site_manifests(site_name, action_dir, app_dir, app_dir / "manifests", cfg) + commit_and_push(apps_dir, f"Deploy {site_name}", token) def decommission(site_name, token): - """Remove manifests from apps repo.""" - user = env("CI_BOT_USER", "ci-bot") - with tempfile.TemporaryDirectory() as tmp: - apps_dir = Path(tmp) - run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}") - site_path = apps_dir / "sjc001" / "websites" / site_name - if not site_path.exists(): - print(f"No manifests for {site_name} — nothing to remove") - return - shutil.rmtree(site_path) - run(f"git -C {apps_dir} config user.name {user}") - run(f"git -C {apps_dir} config user.email {user}@fritzlab.net") - commit_and_push(apps_dir, f"Decommission {site_name}") + apps_dir = clone_apps(token) + site_path = apps_dir / "sjc001" / "websites" / site_name + if not site_path.exists(): + print(f"No manifests for {site_name} — nothing to remove") + return + shutil.rmtree(site_path) + commit_and_push(apps_dir, f"Decommission {site_name}", token) print(f"Bucket {site_name} and its objects are NOT purged automatically.") - print(f" garage bucket delete {site_name} --yes") def cmd_deploy(): @@ -168,12 +220,9 @@ def cmd_deploy(): action_dir = Path(env("ACTION_DIR")) token = env("CI_BOT_TOKEN") site_name = site_repo.split("/", 1)[1] - - cfg = parse_site_yaml(site_dir) - + cfg = parse_site_yaml(site_dir, site_name) if not cfg["enabled"]: print("Site disabled — running decommission...") decommission(site_name, token) return - deploy_static(site_name, site_dir, action_dir, token, cfg) diff --git a/scripts/setup.py b/scripts/setup.py index 701ae1d..9871554 100644 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -24,9 +24,10 @@ def ensure_aws(): subprocess.run(["aws", "--version"], check=True) -def ensure_jinja2(): +def ensure_python_dependencies(): try: import jinja2 + import yaml except ImportError: print("Installing jinja2 + pyyaml...") subprocess.run( @@ -36,6 +37,6 @@ def ensure_jinja2(): if __name__ == "__main__": - ensure_jinja2() + ensure_python_dependencies() ensure_aws() print("Setup complete") diff --git a/scripts/utils.py b/scripts/utils.py index d61a36b..4eb18b7 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,154 +1,343 @@ """Shared utilities for the site-publish action.""" +import hashlib import os +import re import shutil import subprocess import sys -from pathlib import Path +import tempfile +from contextlib import contextmanager +from pathlib import Path, PurePosixPath import yaml -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader, StrictUndefined APPS_REPO = "fritzlab/apps" GITEA_HOST = "code.fritzlab.net" NAMESPACE = "websites" DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900" - +GARAGE_WEBSITE_HOST = "garage-s3.storage.svc.k8s.sjc001.fritzlab.net" EXCLUDE_FILES = { - ".git", ".gitea", ".gitignore", "site.yaml", - "build", "Makefile", "README.md", "CLAUDE.md", - "Dockerfile", ".dockerignore", "go.mod", "go.sum", + ".git", ".gitea", ".gitignore", "site.yaml", "build", "Makefile", + "README.md", "CLAUDE.md", "Dockerfile", ".dockerignore", "go.mod", "go.sum", } - VALID_TYPES = {"static", "hugo", "mkdocs"} +CACHE_POLICIES = { + "immutable-release": ("public, max-age=31536000, immutable", True), + "revalidated-channel": ("public, max-age=0, must-revalidate", False), + "private": ("private, no-store", False), +} +ENV_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") +NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") +BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$") +HOST_RE = BUCKET_RE +MIDDLEWARE_RE = NAME_RE DOCKER_DEPRECATION_MSG = """\ type: docker is no longer supported by action/site-publish. -site-publish handles only static-content sites (static, hugo, mkdocs) -that ship to Garage S3. For containerized web apps, use the standard -image-producer chain: - - - uses: action/image-build@v1 # build + smoke-test - - uses: action/image-push@v1 # push + prune - - uses: action/image-deploy@v1 # apps repo image-pin - -Hand-author your apps-repo manifests once (Deployment, Service, Ingress, -Certificate, kustomization with images: block) under -sjc001/websites//manifests/. image-deploy will pin the tag on -every CI run. See action/image-deploy README and -sjc001/websites/rainsounds.vino.network/manifests/ for the canonical -example.\ +site-publish handles only static-content sites (static, hugo, mkdocs). +Use action/image-build, action/image-push, and action/image-deploy for images.\ """ -def k8s_name(name): - """Sanitize for DNS-1035 label (dots → dashes).""" - return name.replace(".", "-") +def k8s_name(*parts): + raw = re.sub(r"[^a-z0-9-]", "-", "-".join(parts).replace(".", "-").lower()) + raw = raw.strip("-") + if len(raw) <= 63: + return raw + return f"{raw[:54].rstrip('-')}-{hashlib.sha256(raw.encode()).hexdigest()[:8]}" def env(key, default=None): - val = os.environ.get(key, default) - if val is None: + value = os.environ.get(key, default) + if value is None or value == "": die(f"Missing required env var: {key}") - return val + return value -def die(msg): - print(f"ERROR: {msg}", file=sys.stderr) - sys.exit(1) +def die(message): + print(f"ERROR: {message}", file=sys.stderr) + raise SystemExit(1) -def run(cmd, **kwargs): - print(f" $ {cmd}") - return subprocess.run(cmd, shell=True, check=True, **kwargs) +def run(cmd, *, display=None, **kwargs): + """Run argv without a shell and print only a credential-free display.""" + if isinstance(cmd, str): + raise TypeError("run() requires an argv sequence") + print(f" $ {display or ' '.join(str(part) for part in cmd)}") + return subprocess.run(cmd, check=True, **kwargs) -def parse_site_yaml(site_dir): - path = Path(site_dir) / "site.yaml" - if not path.exists(): - die("site.yaml not found in repo root") +def _string_list(cfg, key): + value = cfg.get(key) or [] + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + die(f"{key} must be a list of strings") + return value - with open(path) as f: - cfg = yaml.safe_load(f) - - if not cfg.get("domain"): - die("domain is required in site.yaml") +def _common(cfg): + domain = cfg.get("domain") + if not isinstance(domain, str) or not HOST_RE.fullmatch(domain): + die("domain must be a lowercase hostname") + aliases = _string_list(cfg, "aliases") + if any(not HOST_RE.fullmatch(alias) for alias in aliases): + die("aliases must contain lowercase hostnames") site_type = cfg.get("type", "static") - if site_type == "docker": die(DOCKER_DEPRECATION_MSG) - if site_type not in VALID_TYPES: die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})") - - excludes = cfg.get("excludes") or [] - if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes): - die("excludes must be a list of string patterns") - - middlewares = cfg.get("middlewares") or [] - if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares): - die("middlewares must be a list of Traefik file-provider middleware names") - - site = { - "domain": cfg["domain"], + middlewares = _string_list(cfg, "middlewares") + if any(not MIDDLEWARE_RE.fullmatch(item) for item in middlewares): + die("middlewares must contain Traefik file-provider names") + if not isinstance(cfg.get("enabled", True), bool): + die("enabled must be a boolean") + if not isinstance(cfg.get("tidy", True), bool): + die("tidy must be a boolean") + content_dir = cfg.get("content_dir", "") + content_path = PurePosixPath(content_dir) + if not isinstance(content_dir, str) or content_path.is_absolute() or ".." in content_path.parts: + die("content_dir must be a relative path inside the repository") + return { + "domain": domain, "type": site_type, "enabled": cfg.get("enabled", True), - "aliases": cfg.get("aliases") or [], - "content_dir": cfg.get("content_dir", ""), + "aliases": aliases, + "content_dir": content_dir, "tidy": cfg.get("tidy", True), - "excludes": excludes, + "excludes": _string_list(cfg, "excludes"), "middlewares": middlewares, } - print("Site config:") - for k, v in site.items(): - print(f" {k}: {v}") + +def _source(value, name): + if not isinstance(value, str) or not value: + die(f"artifact {name}: source must be a non-empty relative path") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts: + die(f"artifact {name}: source must stay under build/html") + return value + + +def _v2(cfg, site): + if cfg.get("schema") != "v2": + die("multi-surface configs must set schema: v2") + raw_artifacts, raw_routes = cfg.get("artifacts"), cfg.get("routes") + if not isinstance(raw_artifacts, dict) or not raw_artifacts: + die("artifacts must be a non-empty mapping") + if not isinstance(raw_routes, list) or not raw_routes: + die("routes must be a non-empty list") + + artifacts = {} + for name, raw in raw_artifacts.items(): + if not isinstance(name, str) or not NAME_RE.fullmatch(name): + die("artifact names must be lowercase kebab-case") + if not isinstance(raw, dict): + die(f"artifact {name}: definition must be a mapping") + unknown = set(raw) - {"source", "bucket", "credentials", "cache"} + if unknown: + die(f"artifact {name}: unknown keys: {', '.join(sorted(unknown))}") + bucket = raw.get("bucket") + if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket): + die(f"artifact {name}: bucket must be a lowercase Garage bucket name") + credentials = raw.get("credentials") + if not isinstance(credentials, dict) or set(credentials) != { + "access_key_env", "secret_key_env"}: + die(f"artifact {name}: credentials require access_key_env and secret_key_env") + if any(not isinstance(value, str) or not ENV_RE.fullmatch(value) + for value in credentials.values()): + die(f"artifact {name}: credential selectors must be environment variable names") + cache = raw.get("cache") + if cache not in CACHE_POLICIES: + die(f"artifact {name}: unknown cache policy {cache!r}") + cache_control, immutable = CACHE_POLICIES[cache] + artifacts[name] = { + "name": name, "source": _source(raw.get("source"), name), + "bucket": bucket, "credentials": credentials, "cache": cache, + "cache_control": cache_control, "immutable": immutable, + } + + routes, paths, used = [], set(), set() + buckets, credentials = {}, {} + for raw in raw_routes: + if not isinstance(raw, dict): + die("each route must be a mapping") + unknown = set(raw) - {"path", "artifact", "access", "middlewares"} + if unknown: + die(f"route has unknown keys: {', '.join(sorted(unknown))}") + path = raw.get("path") + if (not isinstance(path, str) or not path.startswith("/") or "//" in path + or (path != "/" and path.endswith("/"))): + die("route paths must be normalized absolute prefixes") + if path in paths: + die(f"duplicate route path: {path}") + paths.add(path) + artifact_name = raw.get("artifact") + if artifact_name not in artifacts: + die(f"route {path}: unknown artifact {artifact_name!r}") + if artifact_name in used: + die(f"artifact {artifact_name} may be routed only once") + used.add(artifact_name) + access = raw.get("access") + if access not in {"public", "authenticated"}: + die(f"route {path}: access must be public or authenticated") + if path == "/" and access == "public": + die("public catch-all route / is forbidden in schema v2") + middlewares = raw.get("middlewares") or [] + if not isinstance(middlewares, list) or any( + not isinstance(item, str) or not MIDDLEWARE_RE.fullmatch(item) + for item in middlewares): + die(f"route {path}: middlewares must contain file-provider names") + if access == "authenticated" and not middlewares: + die(f"route {path}: authenticated access requires middleware") + artifact = artifacts[artifact_name] + if access == "authenticated" and artifact["cache"] != "private": + die(f"route {path}: authenticated artifacts must use private cache") + if access == "public" and artifact["cache"] == "private": + die(f"route {path}: public artifacts cannot use private cache") + prior_access = buckets.setdefault(artifact["bucket"], access) + if prior_access != access: + die(f"route {path}: bucket is reused across access classes") + credential_pair = tuple(artifact["credentials"].values()) + prior_artifact = credentials.setdefault(credential_pair, artifact_name) + if prior_artifact != artifact_name: + die(f"route {path}: publication credentials are reused by artifacts " + f"{prior_artifact} and {artifact_name}") + artifact["access"] = access + routes.append({"path": path, "artifact": artifact_name, + "access": access, "middlewares": middlewares}) + artifact["key_prefix"] = path.lstrip("/") + + missing = set(artifacts) - used + if missing: + die(f"unrouted artifacts: {', '.join(sorted(missing))}") + if "/" not in paths: + die("schema v2 requires an explicit / access policy") + artifact_list = list(artifacts.values()) + for index, left in enumerate(artifact_list): + left_parts = PurePosixPath(left["source"]).parts + for right in artifact_list[index + 1:]: + right_parts = PurePosixPath(right["source"]).parts + overlaps = (left_parts == right_parts[:len(left_parts)] or + right_parts == left_parts[:len(right_parts)]) + if overlaps and left["access"] != right["access"]: + die(f"artifact sources overlap across access classes: " + f"{left['name']} and {right['name']}") + site.update({"schema": "v2", "compatibility": False, "artifacts": artifacts}) + site["routes"] = sorted(routes, key=lambda route: (-len(route["path"]), route["path"])) return site +def parse_site_yaml(site_dir, site_name=None): + path = Path(site_dir) / "site.yaml" + if not path.exists(): + die("site.yaml not found in repo root") + with open(path, encoding="utf-8") as handle: + cfg = yaml.safe_load(handle) + if not isinstance(cfg, dict): + die("site.yaml must contain a mapping") + allowed = {"domain", "type", "enabled", "aliases", "content_dir", "tidy", + "excludes", "middlewares", "schema", "artifacts", "routes"} + unknown = set(cfg) - allowed + if unknown: + die(f"unknown site.yaml keys: {', '.join(sorted(unknown))}") + site = _common(cfg) + has_v2 = any(key in cfg for key in ("schema", "artifacts", "routes")) + if has_v2: + if not all(key in cfg for key in ("schema", "artifacts", "routes")): + die("schema, artifacts, and routes must be declared together") + if site["middlewares"]: + die("schema v2 middlewares belong on individual routes") + site = _v2(cfg, site) + else: + site_name = site_name or site["domain"] + cache_control, immutable = CACHE_POLICIES["revalidated-channel"] + site.update({ + "schema": "single-surface-v1", "compatibility": True, + "artifacts": {"site": { + "name": "site", "source": ".", "bucket": site_name, + "credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", + "secret_key_env": "AWS_SECRET_ACCESS_KEY"}, + "cache": "revalidated-channel", "cache_control": cache_control, + "immutable": immutable, "key_prefix": "", + }}, + "routes": [{"path": "/", "artifact": "site", "access": "legacy", + "middlewares": site["middlewares"]}], + }) + print(f"Site config: {site['schema']} {site['domain']} ({len(site['routes'])} route(s))") + return site + + +@contextmanager +def git_auth(token, user): + """Provide an HTTPS token through an inherited FD, never argv or output.""" + read_fd, write_fd = os.pipe() + with tempfile.TemporaryDirectory() as temp_dir: + askpass = Path(temp_dir) / "askpass.sh" + askpass.write_text( + "#!/bin/sh\ncase \"$1\" in\n" + " *Username*) printf '%s\\n' \"$SITE_PUBLISH_GIT_USER\" ;;\n" + f" *Password*) cat <&{read_fd} ;;\nesac\n", encoding="utf-8") + askpass.chmod(0o700) + os.write(write_fd, token.encode()) + os.close(write_fd) + child_env = os.environ.copy() + child_env.update({"GIT_ASKPASS": str(askpass), "GIT_TERMINAL_PROMPT": "0", + "SITE_PUBLISH_GIT_USER": user}) + try: + yield {"env": child_env, "pass_fds": (read_fd,)} + finally: + os.close(read_fd) + + def clone_apps(token): user = env("CI_BOT_USER", "ci-bot") apps_dir = Path("/tmp/apps-deploy") if apps_dir.exists(): shutil.rmtree(apps_dir) - run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}") - run(f"git -C {apps_dir} config user.name {user}") - run(f"git -C {apps_dir} config user.email {user}@fritzlab.net") + url = f"https://{GITEA_HOST}/{APPS_REPO}.git" + with git_auth(token, user) as auth: + run(["git", "clone", "--depth", "1", url, str(apps_dir)], **auth) + run(["git", "-C", str(apps_dir), "config", "user.name", user]) + run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"]) return apps_dir def render_templates(action_dir, template_vars, app_dir, manifests_dir): - """Render Jinja2 templates for a static-content site.""" - templates_dir = Path(action_dir) / "templates" - jinja_env = Environment( - loader=FileSystemLoader(str(templates_dir)), - keep_trailing_newline=True, - ) - - tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2", - "kustomization.yaml.j2", "service.yaml.j2"] - - for tmpl_name in tmpl_names: - tmpl = jinja_env.get_template(tmpl_name) - rendered = tmpl.render(**template_vars) - out_name = tmpl_name.replace(".j2", "") - dest = app_dir / out_name if tmpl_name == "app.yaml.j2" else manifests_dir / out_name - dest.write_text(rendered) - print(f" Rendered {tmpl_name} -> {dest}") + templates = Path(action_dir) / "templates" + jinja = Environment(loader=FileSystemLoader(str(templates)), + keep_trailing_newline=True, undefined=StrictUndefined) + app_dir.mkdir(parents=True, exist_ok=True) + manifests_dir.mkdir(parents=True, exist_ok=True) + (app_dir / "app.yaml").write_text( + jinja.get_template("app.yaml.j2").render(**template_vars), encoding="utf-8") + (manifests_dir / "certificate.yaml").write_text( + jinja.get_template("certificate.yaml.j2").render(**template_vars), encoding="utf-8") + resources = ["certificate.yaml"] + for route in template_vars["routes"]: + values = {**template_vars, "route": route} + for kind in ("service", "ingress"): + filename = f"{kind}-{route['resource_name']}.yaml" + (manifests_dir / filename).write_text( + jinja.get_template(f"{kind}.yaml.j2").render(**values), encoding="utf-8") + resources.append(filename) + (manifests_dir / "kustomization.yaml").write_text( + jinja.get_template("kustomization.yaml.j2").render( + **template_vars, resources=resources), encoding="utf-8") -def commit_and_push(apps_dir, message): - run(f"git -C {apps_dir} add -A") - result = subprocess.run( - f"git -C {apps_dir} diff --cached --quiet", - shell=True, check=False, - ) - if result.returncode == 0: +def commit_and_push(apps_dir, message, token): + user = env("CI_BOT_USER", "ci-bot") + run(["git", "-C", str(apps_dir), "add", "-A"]) + clean = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], + check=False).returncode == 0 + if clean: print("No manifest changes to commit") return False - run(f"git -C {apps_dir} commit -m '{message}'") - run(f"git -C {apps_dir} push") + run(["git", "-C", str(apps_dir), "commit", "-m", message]) + with git_auth(token, user) as auth: + run(["git", "-C", str(apps_dir), "push"], **auth) print("Manifests pushed — ArgoCD will sync") return True diff --git a/templates/ingress.yaml.j2 b/templates/ingress.yaml.j2 index e8b56c4..f22a750 100644 --- a/templates/ingress.yaml.j2 +++ b/templates/ingress.yaml.j2 @@ -1,12 +1,10 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ site_k8s }} + name: {{ route.resource_name }} namespace: {{ namespace }} -{%- if site_type != "docker" %} annotations: - traefik.ingress.kubernetes.io/router.middlewares: https-redirect@file,retry-upstream@file{% for m in middlewares %},{{ m }}@file{% endfor %} -{%- endif %} + traefik.ingress.kubernetes.io/router.middlewares: https-redirect@file,retry-upstream@file{% for middleware in route.middlewares %},{{ middleware }}@file{% endfor %} spec: ingressClassName: traefik tls: @@ -20,22 +18,22 @@ spec: - host: {{ domain }} http: paths: - - path: / + - path: {{ route.path }} pathType: Prefix backend: service: - name: {{ site_k8s }} + name: {{ route.resource_name }} port: number: 80 {%- for alias in aliases %} - host: {{ alias }} http: paths: - - path: / + - path: {{ route.path }} pathType: Prefix backend: service: - name: {{ site_k8s }} + name: {{ route.resource_name }} port: number: 80 {%- endfor %} diff --git a/templates/kustomization.yaml.j2 b/templates/kustomization.yaml.j2 index 14ebd47..77859b2 100644 --- a/templates/kustomization.yaml.j2 +++ b/templates/kustomization.yaml.j2 @@ -1,6 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- service.yaml -- ingress.yaml -- certificate.yaml +{%- for resource in resources %} +- {{ resource }} +{%- endfor %} diff --git a/templates/service.yaml.j2 b/templates/service.yaml.j2 index d425207..d5f4e77 100644 --- a/templates/service.yaml.j2 +++ b/templates/service.yaml.j2 @@ -1,11 +1,15 @@ apiVersion: v1 kind: Service metadata: - name: {{ site_k8s }} + name: {{ route.resource_name }} namespace: {{ namespace }} +{%- if not route.pass_host_header %} + annotations: + traefik.ingress.kubernetes.io/service.passhostheader: "false" +{%- endif %} spec: type: ExternalName - externalName: garage.storage.svc.k8s.sjc001.fritzlab.net + externalName: {{ route.backend_host }} ports: - port: 80 targetPort: 80 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..26be4f4 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""site-publish test package.""" diff --git a/tests/fixtures/v2-site.yaml b/tests/fixtures/v2-site.yaml new file mode 100644 index 0000000..84e93bf --- /dev/null +++ b/tests/fixtures/v2-site.yaml @@ -0,0 +1,36 @@ +schema: v2 +domain: baseline.fritzlab.net +type: static +artifacts: + releases: + source: dist/releases + bucket: baseline-releases + credentials: + access_key_env: RELEASES_ACCESS_KEY + secret_key_env: RELEASES_SECRET_KEY + cache: immutable-release + channels: + source: dist/channels + bucket: baseline-channels + credentials: + access_key_env: CHANNELS_ACCESS_KEY + secret_key_env: CHANNELS_SECRET_KEY + cache: revalidated-channel + portal: + source: portal + bucket: baseline-portal + credentials: + access_key_env: PORTAL_ACCESS_KEY + secret_key_env: PORTAL_SECRET_KEY + cache: private +routes: + - path: /dist/releases + artifact: releases + access: public + - path: /dist/channels + artifact: channels + access: public + - path: / + artifact: portal + access: authenticated + middlewares: [authentik-forwardauth] diff --git a/tests/test_site_publish.py b/tests/test_site_publish.py new file mode 100644 index 0000000..2310a78 --- /dev/null +++ b/tests/test_site_publish.py @@ -0,0 +1,192 @@ +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import deploy +import utils + + +def write_config(directory, value): + (Path(directory) / "site.yaml").write_text(yaml.safe_dump(value), encoding="utf-8") + + +def v2_config(): + return yaml.safe_load((ROOT / "tests" / "fixtures" / "v2-site.yaml").read_text()) + + +class ConfigTests(unittest.TestCase): + def parse(self, config): + with tempfile.TemporaryDirectory() as directory: + write_config(directory, config) + return utils.parse_site_yaml(directory, "baseline.fritzlab.net") + + def reject(self, config, message): + with self.assertRaisesRegex(SystemExit, "1"): + with contextlib.redirect_stderr(io.StringIO()) as stderr: + self.parse(config) + self.assertIn(message, stderr.getvalue()) + + def test_legacy_config_uses_explicit_compatibility_translation(self): + config = self.parse({"domain": "example.fritzlab.net", "type": "static", + "middlewares": ["authentik-forwardauth"]}) + self.assertEqual("single-surface-v1", config["schema"]) + self.assertTrue(config["compatibility"]) + self.assertEqual("baseline.fritzlab.net", config["artifacts"]["site"]["bucket"]) + self.assertEqual("/", config["routes"][0]["path"]) + + def test_v2_sorts_longest_prefix_and_assigns_cache_contracts(self): + config = self.parse(v2_config()) + self.assertEqual(["/dist/channels", "/dist/releases", "/"], + [route["path"] for route in config["routes"]]) + self.assertEqual("public, max-age=31536000, immutable", + config["artifacts"]["releases"]["cache_control"]) + self.assertEqual("public, max-age=0, must-revalidate", + config["artifacts"]["channels"]["cache_control"]) + + def test_rejects_partial_v2(self): + self.reject({"domain": "x.example", "schema": "v2", "routes": []}, + "declared together") + + def test_rejects_public_catch_all(self): + config = v2_config() + config["routes"][0] = {"path": "/", "artifact": "portal", "access": "public"} + config["artifacts"]["portal"]["cache"] = "revalidated-channel" + self.reject(config, "public catch-all") + + def test_rejects_duplicate_paths(self): + config = v2_config() + config["routes"][1]["path"] = "/dist/releases" + self.reject(config, "duplicate route path") + + def test_rejects_bucket_reuse_across_access(self): + config = v2_config() + config["artifacts"]["portal"]["bucket"] = "baseline-releases" + self.reject(config, "bucket is reused across access classes") + + def test_rejects_credential_reuse_across_artifacts(self): + config = v2_config() + config["artifacts"]["portal"]["credentials"] = { + "access_key_env": "RELEASES_ACCESS_KEY", "secret_key_env": "RELEASES_SECRET_KEY"} + self.reject(config, "publication credentials are reused") + + def test_requires_explicit_root_policy(self): + config = v2_config() + config["routes"] = [route for route in config["routes"] if route["path"] != "/"] + del config["artifacts"]["portal"] + self.reject(config, "explicit / access policy") + + def test_rejects_overlapping_sources_across_access(self): + config = v2_config() + config["artifacts"]["portal"]["source"] = "." + self.reject(config, "sources overlap across access classes") + + def test_rejects_unknown_top_level_key(self): + config = v2_config() + config["surprise"] = True + self.reject(config, "unknown site.yaml keys") + + +class ManifestTests(unittest.TestCase): + def test_split_routes_render_one_certificate_and_distinct_backends(self): + with tempfile.TemporaryDirectory() as config_dir, tempfile.TemporaryDirectory() as out: + write_config(config_dir, v2_config()) + config = utils.parse_site_yaml(config_dir, "baseline.fritzlab.net") + app_dir = Path(out) / "app" + deploy.render_site_manifests("baseline.fritzlab.net", ROOT, app_dir, + app_dir / "manifests", config) + manifests = app_dir / "manifests" + self.assertEqual(1, len(list(manifests.glob("certificate*.yaml")))) + self.assertEqual(3, len(list(manifests.glob("service-*.yaml")))) + self.assertEqual(3, len(list(manifests.glob("ingress-*.yaml")))) + for path in manifests.glob("*.yaml"): + list(yaml.safe_load_all(path.read_text(encoding="utf-8"))) + services = "".join(path.read_text() for path in manifests.glob("service-*.yaml")) + self.assertIn("baseline-releases.web.sjc001.fritzlab.net", services) + self.assertIn('service.passhostheader: "false"', services) + portal_ingress = next(manifests.glob("ingress-*-portal.yaml")).read_text() + self.assertIn("authentik-forwardauth@file", portal_ingress) + + +class PublicationTests(unittest.TestCase): + def artifact(self, immutable=False): + return { + "name": "release", "bucket": "release-bucket", + "credentials": {"access_key_env": "TEST_ACCESS", + "secret_key_env": "TEST_SECRET"}, + "cache_control": "public, max-age=31536000, immutable" if immutable + else "public, max-age=0, must-revalidate", + "immutable": immutable, + } + + @mock.patch.dict(os.environ, {"TEST_ACCESS": "access", "TEST_SECRET": "secret"}) + @mock.patch("deploy.urlopen") + def test_immutable_put_is_conditional_and_signed_in_process(self, urlopen_mock): + response = urlopen_mock.return_value.__enter__.return_value + response.read.return_value = b"" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "bundle.js" + path.write_bytes(b"release") + deploy._conditional_put("http://garage", self.artifact(True), + "dist/releases/bundle.js", path, + "application/javascript", "digest") + request = urlopen_mock.call_args.args[0] + self.assertEqual("*", request.headers["If-none-match"]) + self.assertNotIn("secret", request.full_url) + + @mock.patch.dict(os.environ, {"TEST_ACCESS": "access", "TEST_SECRET": "secret"}) + @mock.patch("deploy.run") + def test_revalidated_publication_sets_cache_on_sync_and_restamp(self, run_mock): + with tempfile.TemporaryDirectory() as directory: + deploy._replaceable_sync(Path(directory), self.artifact(), "http://garage", []) + commands = [call.args[0] for call in run_mock.call_args_list] + self.assertEqual(["sync", "cp"], [command[4] for command in commands]) + self.assertTrue(all("public, max-age=0, must-revalidate" in command + for command in commands)) + + @mock.patch.dict(os.environ, {"TEST_ACCESS": "access", "TEST_SECRET": "secret"}) + @mock.patch("deploy._conditional_put") + @mock.patch("deploy.subprocess.run") + def test_immutable_identical_retry_skips_and_changed_key_fails(self, subprocess_mock, + put_mock): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + content = b"release" + (source / "bundle.js").write_bytes(content) + digest = __import__("hashlib").sha256(content).hexdigest() + subprocess_mock.return_value = subprocess.CompletedProcess( + [], 0, stdout=json.dumps({ + "Metadata": {"sha256": digest}, + "CacheControl": "public, max-age=31536000, immutable"}), stderr="") + deploy._immutable_sync(source, self.artifact(True), "http://garage", []) + put_mock.assert_not_called() + subprocess_mock.return_value = subprocess.CompletedProcess( + [], 0, stdout='{"Metadata":{"sha256":"different"}}', stderr="") + with self.assertRaises(SystemExit): + deploy._immutable_sync(source, self.artifact(True), "http://garage", []) + + +class CredentialSafetyTests(unittest.TestCase): + def test_git_askpass_reads_token_only_from_inherited_fd(self): + token = "credential-sentinel" + with utils.git_auth(token, "ci-bot") as auth: + self.assertNotIn(token, str(auth["env"])) + result = subprocess.run( + [auth["env"]["GIT_ASKPASS"], "Password"], + capture_output=True, text=True, check=True, **auth) + self.assertEqual(token, result.stdout) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 49fcd8dafc992ba4b7c5d63b523e68448f986d0e Mon Sep 17 00:00:00 2001 From: Dave Kowalski Date: Sat, 29 Aug 2026 21:24:05 +0000 Subject: [PATCH 2/2] fix(ci): allow managed runner dependencies Authored-By: OpenAI (GPT-5) --- .gitea/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml index 0fb71e7..ec2bb68 100644 --- a/.gitea/workflows/test.yaml +++ b/.gitea/workflows/test.yaml @@ -9,6 +9,6 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install dependencies - run: python3 -m pip install --quiet jinja2 pyyaml awscli + run: python3 -m pip install --quiet --break-system-packages jinja2 pyyaml awscli - name: Test run: python3 -m unittest discover -v -- 2.54.0