From 5c2630b972f6842c792aff874e50faa7d00700fe Mon Sep 17 00:00:00 2001 From: Evelyn Chen Date: Sat, 29 Aug 2026 21:22:04 +0000 Subject: [PATCH 1/4] feat(site-publish): add split-surface publishing Authored-By: OpenAI (GPT-5) --- .gitea/workflows/test.yaml | 14 + README.md | 162 ++++++++-- action.yaml | 12 +- new-site.sh | 4 +- scripts/build.py | 47 +-- scripts/deploy.py | 281 +++++++++++++---- scripts/git-askpass.sh | 5 + scripts/setup.py | 5 +- scripts/utils.py | 514 ++++++++++++++++++++++++++----- templates/ingress.yaml.j2 | 14 +- templates/kustomization.yaml.j2 | 5 +- templates/service.yaml.j2 | 8 +- tests/fixtures/legacy-site.yaml | 9 + tests/fixtures/split-site.yaml | 42 +++ tests/test_contract.py | 521 ++++++++++++++++++++++++++++++++ 15 files changed, 1450 insertions(+), 193 deletions(-) create mode 100644 .gitea/workflows/test.yaml create mode 100755 scripts/git-askpass.sh create mode 100644 tests/fixtures/legacy-site.yaml create mode 100644 tests/fixtures/split-site.yaml create mode 100644 tests/test_contract.py diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml new file mode 100644 index 0000000..16cbb61 --- /dev/null +++ b/.gitea/workflows/test.yaml @@ -0,0 +1,14 @@ +name: Test +on: + pull_request: + push: + branches: [main] +jobs: + contract: + runs-on: fritzlab + steps: + - uses: actions/checkout@v4 + - name: Install test dependencies + run: python3 -m pip install --quiet --break-system-packages jinja2 pyyaml + - name: Run contract tests + run: python3 -m unittest discover -s tests -v diff --git a/README.md b/README.md index 6873418..d4c7b49 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # action/site-publish -Composite Gitea Action that publishes a **static-content** website to the -fritzlab k8s cluster. Supports `static`, `hugo`, and `mkdocs`. Content goes -to a Garage S3 bucket; Traefik fronts the bucket via an `ExternalName` -Service with cert-manager TLS. +Composite Gitea Action that publishes one or more **static-content** artifacts +to one hostname. Each split surface owns its build input, Garage bucket, +publication credential environment variables, cache rules, Service, Ingress, +route prefix, and access middleware. The hostname shares one Certificate. +`static`, `hugo`, and `mkdocs` builds are supported; a prebuilt Docusaurus +output is a `static` artifact. > **Containerized web apps (Dockerfile-based) are NOT handled here.** Use the > standard image-producer chain instead: @@ -16,9 +18,10 @@ Service with cert-manager TLS. > for the canonical example. site-publish errors out explicitly if > `site.yaml` has `type: docker`. -## Convention +## Single-surface compatibility -Bucket name = repo name = canonical domain. Sibling hostnames (e.g. `www.`, +Existing `site.yaml` files remain the `single-surface-v1` compatibility +contract. Bucket name = repo name = canonical domain. Sibling hostnames (e.g. `www.`, `ipv6.`) are declared as `aliases:` in `site.yaml` — the action registers each as a Garage `globalAlias` on the bucket and adds it to the Ingress + Certificate on every deploy. Manual edits to manifests in the apps repo are clobbered; @@ -32,6 +35,121 @@ Scaffold a new site (handles repo creation + Garage bucket): ./new-site.sh --name my-site.vino.network --domain my-site.vino.network --type static ``` +The compatibility path still writes `build/html`, uploads with +`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, and renders `service.yaml` plus +`ingress.yaml`. Its only behavior change is the required root-cause repair: +the website Service now targets the data-only `garage-s3` Service. + +## Split-surface contract + +Use `artifacts` and `routes` together. This example expresses an authenticated +prebuilt portal at `/` and public bundles at `/dist`; it is illustrative and +the schema has no Baseline-specific field. + +```yaml +domain: baseline.fritzlab.net +artifacts: + - name: distributions + type: static + content_dir: dist + publish: + bucket: baseline-dist + credentials: + access_key_env: DIST_S3_ACCESS_KEY + secret_key_env: DIST_S3_SECRET_KEY + cache: + rules: + - path: / + cache_control: public, max-age=0, must-revalidate + - path: releases + cache_control: public, max-age=31536000, immutable + - path: channels + cache_control: public, max-age=0, must-revalidate + - name: portal + type: static + content_dir: portal/build + publish: + bucket: baseline-portal + credentials: + access_key_env: PORTAL_S3_ACCESS_KEY + secret_key_env: PORTAL_S3_SECRET_KEY + cache: + rules: + - path: / + cache_control: private, no-store +routes: + - name: distributions + path: /dist + artifact: distributions + access: + mode: public + - name: portal + path: / + artifact: portal + access: + mode: protected + middleware: authentik-forwardauth +``` + +The caller supplies each declared credential name as an environment variable +on the action step. Names must be matched `_S3_ACCESS_KEY` and +`_S3_SECRET_KEY` pairs; arbitrary environment variables cannot become +publication credentials. Values pass to `aws` only through its environment and +never appear in a logged command or process argument. + +```yaml + - uses: https://code.fritzlab.net/action/site-publish@v1 + with: + token: ${{ secrets.CI_BOT_TOKEN }} + env: + DIST_S3_ACCESS_KEY: ${{ secrets.DIST_S3_ACCESS_KEY }} + DIST_S3_SECRET_KEY: ${{ secrets.DIST_S3_SECRET_KEY }} + PORTAL_S3_ACCESS_KEY: ${{ secrets.PORTAL_S3_ACCESS_KEY }} + PORTAL_S3_SECRET_KEY: ${{ secrets.PORTAL_S3_SECRET_KEY }} +``` + +Routes are normalized and rendered longest-prefix first. Split mode requires +one `/` catch-all so unmatched paths have an explicit access policy. If any +route is protected, that catch-all must also be protected. Every artifact must +belong to exactly one route and bucket; protected and public routes cannot +reuse a bucket. A protected route requires an existing file-provider access +middleware. Public routes cannot declare one. + +Every cache policy requires a `/` default. More-specific cache paths override +it, are reapplied in deterministic prefix order, and must exist in the built +artifact. Contradictory directives (`public` plus `private`, `immutable` plus +revalidation, or `no-store` plus a positive max-age) are rejected. Protected +artifacts require `private` or `no-store` and cannot emit `public`. +Metadata restamping transfers each artifact once even on a no-op publication; +that is the cost of making policy changes effective on unchanged Garage objects. +An immutable cache path is excluded from sync and deletion. Every object key in +that path must contain exactly one full publication SHA-256, calculated over its +cache policy, content type, and bytes. That content address makes concurrent +writes identical even though Garage v2.2.0 has no conditional destination +write. An identical retry converges; a changed object, missing digest metadata, +wrong address, or nested policy under that immutable prefix fails publication. + +Artifact input directories must be pairwise disjoint after filesystem +resolution. Publication stops before build or upload if one contains another or +escapes the repository, preventing protected input from entering a public +artifact. Split storage endpoints are pinned to Garage, and each website +authority is derived from its bucket; a site cannot expose an arbitrary backend. + +Each split route gets a bucket-specific `.web.sjc001.fritzlab.net` +ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route +Ingresses share the hostname's certificate Secret. The access middleware and +Garage bucket/key must already exist; the publisher doesn't create identity +providers or credentials. + +### Migrating a site + +Leave an existing single-surface file unchanged until a real second surface +exists. Then build every artifact before this action, move the old fields into +an artifact, declare a route for every artifact, give each bucket a separately +scoped key, and set the route access/cache contract. Run the repository tests +and inspect generated Apps changes. Removing a route removes its generated +Service and Ingress on the next render; bucket deletion remains manual. + Or do it manually. `site.yaml`: ```yaml @@ -75,11 +193,11 @@ jobs: garage-admin-token: ${{ secrets.GARAGE_ADMIN_TOKEN }} ``` -DNS: subdomains of `vino.network` are covered by the wildcard CNAME to -`traefik.edge.svc…`. For other zones, add an explicit CNAME: +DNS: subdomains of `vino.network` are covered by the wildcard CNAME to the +public gateway. For other zones, add an explicit CNAME: ``` -my-site.fritzlab.net 300 IN CNAME traefik.edge.svc.k8s.sjc001.fritzlab.net. +my-site.fritzlab.net 300 IN CNAME gateway.sjc001.fritzlab.net. ``` ## Inputs @@ -87,10 +205,10 @@ my-site.fritzlab.net 300 IN CNAME traefik.edge.svc.k8s.sjc001.fritzlab.net. | Input | Required | Default | Description | |---|---|---|---| | `token` | yes | | Gitea token for apps repo push | -| `s3-access-key` | yes | | Garage `ci-deploy-key` access key id | -| `s3-secret-key` | yes | | Garage `ci-deploy-key` secret key | -| `s3-endpoint` | no | `http://garage.storage.svc:3900` | Garage S3 endpoint | -| `garage-admin-token` | only if site has `aliases` | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) | +| `s3-access-key` | legacy only | | Garage access key id for single-surface sites | +| `s3-secret-key` | legacy only | | Garage secret key for single-surface sites | +| `s3-endpoint` | no | `http://garage-s3.storage.svc:3900` | Legacy Garage S3 endpoint | +| `garage-admin-token` | legacy aliases only | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) | | `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint | | `username` | no | `ci-bot` | Gitea username | @@ -107,12 +225,11 @@ Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`, ``` push to websites/ → CI runs site-publish action - → reads site.yaml, builds content (static copy / hugo / mkdocs), runs tidy - → aws s3 sync → Garage bucket named after the repo - → admin API: ensures every alias from site.yaml is a globalAlias on the bucket - → renders manifests in fritzlab/apps from templates: ExternalName Service → - garage.storage.svc, Traefik Ingress (canonical + aliases), cert-manager - Certificate (canonical + aliases as SANs), kustomization + → reads and validates all of site.yaml before publication + → independently builds each static / Hugo / MkDocs artifact + → syncs each artifact to its route-owned Garage bucket and prefix + → reapplies the artifact's default and longest-prefix cache headers + → renders one Service + Ingress per route and one shared Certificate → commits + pushes apps repo only if diff is non-empty → ArgoCD syncs → site live with TLS ``` @@ -120,9 +237,10 @@ push to websites/ The Ingress + Certificate are re-rendered on every deploy from `site.yaml`. There is no "first-deploy vs. update" branching — every deploy is idempotent. -No nginx pods, no per-site Docker images. Garage matches `Host:` header to -bucket name (or any of its globalAliases), so every site shares a single -ExternalName target. +No nginx pods, no per-site Docker images. Compatibility sites pass the public +host to the shared data-only Garage website Service. Split routes disable host +passing on their Service so Garage receives that artifact's bucket-specific +website authority. ## History diff --git a/action.yaml b/action.yaml index 505170e..7d287b2 100644 --- a/action.yaml +++ b/action.yaml @@ -1,15 +1,15 @@ name: Publish Site -description: Build and deploy a static-content site (static, hugo, mkdocs) to Garage S3 with Traefik + cert-manager. Containerized apps should use action/image-build + action/image-push + action/image-deploy. +description: Build and deploy one or more routed static-content artifacts to Garage S3 with Traefik and cert-manager. inputs: token: description: Gitea token (ci-bot) for apps repo push and API operations required: true s3-access-key: - description: Garage ci-deploy-key access key id - required: true + description: Garage access key id (required by the legacy single-surface contract) + required: false s3-secret-key: - description: Garage ci-deploy-key secret access key - required: true + description: Garage secret access key (required by the legacy single-surface contract) + required: false s3-endpoint: # Targets garage-s3 (data-only Service) so requests do not round-robin onto # the gateway pod, whose emptyDir-backed metadata view intermittently @@ -18,7 +18,7 @@ inputs: required: false default: http://garage-s3.storage.svc:3900 garage-admin-token: - description: Garage admin API token (required only when site.yaml has aliases — used to reconcile bucket globalAliases) + description: Garage admin API token (required only for legacy aliases — used to reconcile bucket globalAliases) required: false garage-admin-endpoint: description: Garage admin API endpoint URL diff --git a/new-site.sh b/new-site.sh index 0d0f7cd..4375159 100755 --- a/new-site.sh +++ b/new-site.sh @@ -165,6 +165,6 @@ echo echo "Site created: ${ORG}/${NAME}" echo "First build will trigger on push." echo -echo "DNS: ${DOMAIN} is covered by the *.vino.network wildcard (→ traefik.edge)." +echo "DNS: ${DOMAIN} is covered by the *.vino.network wildcard (→ public gateway)." echo "For a domain outside vino.network, add an explicit CNAME:" -echo " ${DOMAIN} 300 IN CNAME traefik.edge.svc.k8s.sjc001.fritzlab.net." +echo " ${DOMAIN} 300 IN CNAME gateway.sjc001.fritzlab.net." diff --git a/scripts/build.py b/scripts/build.py index 14d933a..6530e28 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -1,25 +1,25 @@ -"""Build phase — content prep for static-content sites.""" +"""Build each declared static-content artifact independently.""" import shutil import subprocess import tempfile from pathlib import Path -from utils import EXCLUDE_FILES, env, parse_site_yaml, run +from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs -def build_static(site_dir, cfg): - build_dir = site_dir / "build" - html_dir = build_dir / "html" +def build_artifact(site_dir, artifact): + html_dir = site_dir / artifact["build_dir"] + if html_dir.parent.exists(): + shutil.rmtree(html_dir.parent) - if build_dir.exists(): - shutil.rmtree(build_dir) - - content_dir = cfg["content_dir"] + content_dir = artifact["content_dir"] src = site_dir / content_dir if content_dir else site_dir + if not src.exists(): + raise FileNotFoundError(f"artifact {artifact['name']} content_dir not found: {src}") - if cfg["type"] == "static": - print(f"Copying static content from {src}") + if artifact["type"] == "static": + print(f"Copying artifact {artifact['name']} from {src}") with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) / "html" shutil.copytree(src, tmp_path, dirs_exist_ok=True) @@ -29,18 +29,18 @@ def build_static(site_dir, cfg): shutil.rmtree(p) elif p.exists(): p.unlink() - build_dir.mkdir(parents=True) + html_dir.parent.mkdir(parents=True) shutil.move(str(tmp_path), str(html_dir)) - elif cfg["type"] == "hugo": - print(f"Building Hugo site from {src}") - run(f"hugo --source {src} --destination {html_dir}") + elif artifact["type"] == "hugo": + print(f"Building Hugo artifact {artifact['name']} from {src}") + run(["hugo", "--source", str(src), "--destination", str(html_dir)]) - elif cfg["type"] == "mkdocs": - print(f"Building MkDocs site from {src}") - run(f"cd {src} && mkdocs build -d {html_dir}") + elif artifact["type"] == "mkdocs": + print(f"Building MkDocs artifact {artifact['name']} from {src}") + run(["mkdocs", "build", "-d", str(html_dir)], cwd=src) - if cfg.get("tidy", True): + if artifact["tidy"]: print("Running tidy on HTML files...") for html_file in html_dir.rglob("*.html"): subprocess.run( @@ -51,7 +51,9 @@ def build_static(site_dir, cfg): check=False, ) - print(f"Build complete — content at {html_dir}") + if not any(path.is_file() for path in html_dir.rglob("*")): + raise FileNotFoundError(f"artifact {artifact['name']} produced no files in {html_dir}") + print(f"Artifact {artifact['name']} complete — content at {html_dir}") def cmd_build(): @@ -62,4 +64,7 @@ def cmd_build(): print("Site disabled — skipping build") return - build_static(site_dir, cfg) + validate_artifact_inputs(site_dir, cfg) + + for artifact in cfg["artifacts"]: + build_artifact(site_dir, artifact) diff --git a/scripts/deploy.py b/scripts/deploy.py index 7b6cbc7..d7354c9 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -1,17 +1,18 @@ """Deploy phase — S3 sync, manifest rendering, alias reconcile.""" +import fnmatch +import hashlib import json +import mimetypes import os -import shlex +import re import shutil -import tempfile +import subprocess from pathlib import Path from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen from utils import ( - DEFAULT_S3_ENDPOINT, - GITEA_HOST, NAMESPACE, clone_apps, commit_and_push, @@ -21,6 +22,7 @@ from utils import ( parse_site_yaml, render_templates, run, + validate_artifact_inputs, ) GARAGE_ADMIN_ENDPOINT = os.environ.get( @@ -28,44 +30,194 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get( ) -CACHE_CONTROL = "public, max-age=0, must-revalidate" +def validate_artifact_output(site_dir, artifact): + """Prove every artifact and declared cache prefix exists before publishing any.""" + html_dir = site_dir / artifact["build_dir"] + if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")): + die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}") + for rule in artifact["cache_rules"]: + if not rule["path"]: + continue + cache_root = html_dir / rule["path"] + if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")): + die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files") -def s3_sync(site_name, site_dir, excludes=None): - endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT) - html_dir = site_dir / "build" / "html" - if not html_dir.exists(): - die(f"build/html not found — did the build step run? ({html_dir})") - env("AWS_ACCESS_KEY_ID") - env("AWS_SECRET_ACCESS_KEY") - os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001") +def validate_publication_environment(cfg): + """Resolve every declared credential before the first bucket is changed.""" + for artifact in cfg["artifacts"]: + env(artifact["credentials"]["access_key_env"]) + env(artifact["credentials"]["secret_key_env"]) + if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"): + die("GARAGE_ADMIN_TOKEN is required when aliases are declared") + + +def _is_immutable(rule): + return "immutable" in { + part.strip().lower().split("=", 1)[0] + for part in rule["cache_control"].split(",") + } + + +def _aws_capture(args, aws_env): + """Run a non-streaming AWS request without exposing environment credentials.""" + print(f" $ {' '.join(str(part) for part in args)}") + return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False) + + +def _immutable_head(endpoint, bucket, key, aws_env): + args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object", + "--bucket", bucket, "--key", key, "--output", "json"] + result = _aws_capture(args, aws_env) + if result.returncode == 0: + return json.loads(result.stdout) + error = f"{result.stdout}\n{result.stderr}" + if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")): + return None + raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}") + + +def _immutable_digests(source, cache_control, content_type): + with source.open("rb") as stream: + content_digest = hashlib.file_digest(stream, "sha256").hexdigest() + publication = hashlib.sha256() + publication.update(cache_control.encode()) + publication.update(b"\0") + publication.update(content_type.encode()) + publication.update(b"\0") + with source.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + publication.update(block) + return content_digest, publication.hexdigest() + + +def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type): + metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()} + return ( + metadata.get("sha256") == content_digest + and metadata.get("publication-sha256") == publication_digest + and info.get("CacheControl") == cache_control + and info.get("ContentType") == content_type + ) + + +def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env): + """Publish a content-addressed key; identical retries converge.""" + content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream" + content_digest, publication_digest = _immutable_digests(source, cache_control, content_type) + address_digests = re.findall(r"(? 253 or value.endswith("."): + raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot") + labels = value.split(".") + if len(labels) < 2 or any(not NAME_RE.fullmatch(part) for part in labels): + raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot") + return value + + +def _aliases(value, domain): + aliases = _strings(value or [], "aliases") + aliases = [_hostname(alias, "aliases entry") for alias in aliases] + if len(aliases) != len(set(aliases)) or domain in aliases: + raise ConfigError("aliases must be unique and cannot repeat domain") + return aliases + + +def _relative_path(value, label, *, allow_root=False): + value = "" if value is None else value + if not isinstance(value, str): + raise ConfigError(f"{label} must be a string") + if value == "/" and allow_root: + return "" + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts: + raise ConfigError(f"{label} must be a relative path without '..'") + normalized = str(path).strip("/") + return "" if normalized == "." else normalized + + +def _route_path(value, label): + if not isinstance(value, str) or not value.startswith("/"): + raise ConfigError(f"{label} must start with '/'") + if "//" in value or "?" in value or "#" in value or ".." in value.split("/"): + raise ConfigError(f"{label} is not a canonical URL path") + normalized = value.rstrip("/") or "/" + if not re.fullmatch(r"/[A-Za-z0-9._~/-]*", normalized): + raise ConfigError(f"{label} contains unsupported URL path characters") + return normalized + + +def _middlewares(value, label): + names = _strings(value, label) + if any(not MIDDLEWARE_RE.fullmatch(name) for name in names): + raise ConfigError(f"{label} contains an invalid middleware name") + if len(names) != len(set(names)): + raise ConfigError(f"{label} contains duplicate middleware names") + return names + + +def _cache_control(value, label): + if not isinstance(value, str) or not value.strip(): + raise ConfigError(f"{label} must be a non-empty Cache-Control value") + directives = [part.strip().lower() for part in value.split(",")] + names = [part.split("=", 1)[0] for part in directives] + if len(names) != len(set(names)): + raise ConfigError(f"{label} repeats a Cache-Control directive") + present = set(names) + if {"public", "private"} <= present: + raise ConfigError(f"{label} cannot be both public and private") + ages = {} + for part in directives: + name = part.split("=", 1)[0] + if name in {"max-age", "s-maxage"}: + raw = part.split("=", 1)[1] if "=" in part else "" + if not raw.isdigit(): + raise ConfigError(f"{label} {name} must be a non-negative integer") + ages[name] = int(raw) + max_age = ages.get("max-age") + if "immutable" in present and (not max_age or present & {"no-store", "no-cache", "must-revalidate"}): + raise ConfigError(f"{label} immutable requires positive max-age without revalidation") + if "no-store" in present and any(ages.values()): + raise ConfigError(f"{label} no-store contradicts positive caching") + if "private" in present and ages.get("s-maxage"): + raise ConfigError(f"{label} private contradicts shared-cache max-age") + return ", ".join(part.strip() for part in value.split(",")) + + +def _site_type(value, label): + site_type = value or "static" if site_type == "docker": - die(DOCKER_DEPRECATION_MSG) - + raise ConfigError(DOCKER_DEPRECATION_MSG) if site_type not in VALID_TYPES: - die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})") + raise ConfigError(f"Unknown {label}: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})") + return site_type - excludes = cfg.get("excludes") or [] - if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes): - die("excludes must be a list of string patterns") - middlewares = cfg.get("middlewares") or [] - if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares): - die("middlewares must be a list of Traefik file-provider middleware names") +def _endpoint(value, label): + parsed = urlparse(value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.path not in {"", "/"}: + raise ConfigError(f"{label} must be an http(s) origin without a path") + return value.rstrip("/") - site = { - "domain": cfg["domain"], - "type": site_type, - "enabled": cfg.get("enabled", True), - "aliases": cfg.get("aliases") or [], - "content_dir": cfg.get("content_dir", ""), - "tidy": cfg.get("tidy", True), - "excludes": excludes, + +def _legacy_config(raw, site_name): + if not isinstance(raw.get("tidy", True), bool): + raise ConfigError("tidy must be a boolean") + if not isinstance(raw.get("enabled", True), bool): + raise ConfigError("enabled must be a boolean") + artifact = { + "name": "site", + "type": _site_type(raw.get("type", "static"), "site type"), + "content_dir": _relative_path(raw.get("content_dir", ""), "content_dir"), + "tidy": raw.get("tidy", True), + "excludes": _strings(raw.get("excludes") or [], "excludes"), + "build_dir": "build/html", + "bucket": site_name, + "s3_endpoint": os.environ.get("GARAGE_S3_ENDPOINT") or DEFAULT_S3_ENDPOINT, + "website_authority": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net", + "credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", "secret_key_env": "AWS_SECRET_ACCESS_KEY"}, + "cache_rules": [{"path": "", "cache_control": DEFAULT_CACHE_CONTROL}], + } + return { + "version": 1, + "compatibility": "single-surface-v1", + "domain": raw["domain"], + "aliases": _aliases(raw.get("aliases"), raw["domain"]), + "enabled": raw.get("enabled", True), + "artifacts": [artifact], + "routes": [{ + "name": "site", "path": "/", "artifact": "site", "access": "legacy", + "access_middleware": None, + "middlewares": _middlewares(raw.get("middlewares") or [], "middlewares"), + }], + } + + +def _artifact(item, index): + label = f"artifacts[{index}]" + item = _mapping(item, label) + _known_keys(item, {"name", "type", "content_dir", "tidy", "excludes", "publish", "cache"}, label) + name = item.get("name") + if not isinstance(name, str) or not NAME_RE.fullmatch(name): + raise ConfigError(f"{label}.name must be a DNS label") + publish = _mapping(item.get("publish"), f"{label}.publish") + _known_keys(publish, {"bucket", "credentials"}, f"{label}.publish") + bucket = publish.get("bucket") + if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket): + raise ConfigError(f"{label}.publish.bucket is not a valid bucket name") + credentials = _mapping(publish.get("credentials"), f"{label}.publish.credentials") + _known_keys(credentials, {"access_key_env", "secret_key_env"}, f"{label}.publish.credentials") + normalized_credentials = {} + for key in ("access_key_env", "secret_key_env"): + value = credentials.get(key) + if not isinstance(value, str) or not ENV_RE.fullmatch(value): + raise ConfigError(f"{label}.publish.credentials.{key} must name an environment variable") + normalized_credentials[key] = value + access_match = ACCESS_KEY_ENV_RE.fullmatch(normalized_credentials["access_key_env"]) + expected_secret = ( + f"{access_match.group(1)}_S3_SECRET_KEY" if access_match else None + ) + if normalized_credentials["secret_key_env"] != expected_secret: + raise ConfigError( + f"{label}.publish.credentials must be a matched " + "_S3_ACCESS_KEY and _S3_SECRET_KEY pair" + ) + cache = _mapping(item.get("cache"), f"{label}.cache") + _known_keys(cache, {"rules"}, f"{label}.cache") + rules = _list(cache.get("rules"), f"{label}.cache.rules") + if not rules: + raise ConfigError(f"{label}.cache.rules must declare a '/' default") + cache_rules, paths = [], set() + for rule_index, rule in enumerate(rules): + rule_label = f"{label}.cache.rules[{rule_index}]" + rule = _mapping(rule, rule_label) + _known_keys(rule, {"path", "cache_control"}, rule_label) + path = _relative_path(rule.get("path"), f"{rule_label}.path", allow_root=True) + if path in paths: + raise ConfigError(f"{label}.cache.rules has duplicate path /{path}") + paths.add(path) + cache_rules.append({"path": path, "cache_control": _cache_control( + rule.get("cache_control"), f"{rule_label}.cache_control" + )}) + if "" not in paths: + raise ConfigError(f"{label}.cache.rules must declare a '/' default") + cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"])) + immutable_paths = [rule["path"] for rule in cache_rules + if "immutable" in {part.strip().lower().split("=", 1)[0] + for part in rule["cache_control"].split(",")}] + if "" in immutable_paths: + raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'") + for path in immutable_paths: + if any(other.startswith(f"{path}/") for other in paths): + raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}") + authority = f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}" + if not isinstance(item.get("tidy", True), bool): + raise ConfigError(f"{label}.tidy must be a boolean") + return { + "name": name, + "type": _site_type(item.get("type", "static"), f"{label}.type"), + "content_dir": _relative_path(item.get("content_dir", ""), f"{label}.content_dir"), + "tidy": item.get("tidy", True), + "excludes": _strings(item.get("excludes") or [], f"{label}.excludes"), + "build_dir": f".site-publish/{name}/html", + "bucket": bucket, + "s3_endpoint": DEFAULT_S3_ENDPOINT, + "website_authority": authority, + "credentials": normalized_credentials, + "cache_rules": cache_rules, + } + + +def _route(item, index): + label = f"routes[{index}]" + item = _mapping(item, label) + _known_keys(item, {"name", "path", "artifact", "access", "middlewares"}, label) + name = item.get("name") + if not isinstance(name, str) or not NAME_RE.fullmatch(name): + raise ConfigError(f"{label}.name must be a DNS label") + access = _mapping(item.get("access"), f"{label}.access") + _known_keys(access, {"mode", "middleware"}, f"{label}.access") + mode, middleware = access.get("mode"), access.get("middleware") + if mode not in {"public", "protected"}: + raise ConfigError(f"{label}.access.mode must be public or protected") + if mode == "protected" and (not isinstance(middleware, str) or not MIDDLEWARE_RE.fullmatch(middleware)): + raise ConfigError(f"{label}.access.middleware is required for protected access") + if mode == "public" and middleware is not None: + raise ConfigError(f"{label}.access.middleware is forbidden for public access") + middlewares = _middlewares(item.get("middlewares") or [], f"{label}.middlewares") + if middleware in middlewares: + raise ConfigError(f"{label} repeats its access middleware") + artifact = item.get("artifact") + if not isinstance(artifact, str): + raise ConfigError(f"{label}.artifact must name an artifact") + return { + "name": name, "path": _route_path(item.get("path"), f"{label}.path"), + "artifact": artifact, "access": mode, "access_middleware": middleware, "middlewares": middlewares, } + +def _validate_multi(cfg): + artifacts, routes = cfg["artifacts"], cfg["routes"] + artifact_names = [item["name"] for item in artifacts] + route_names = [item["name"] for item in routes] + route_paths = [item["path"] for item in routes] + if len(artifact_names) != len(set(artifact_names)): + raise ConfigError("artifact names must be unique") + if len(route_names) != len(set(route_names)): + raise ConfigError("route names must be unique") + if len(route_paths) != len(set(route_paths)): + raise ConfigError("route paths are ambiguous after normalization") + if "/" not in route_paths: + raise ConfigError("routes must declare a '/' catch-all") + artifact_by_name = {item["name"]: item for item in artifacts} + references = {name: [] for name in artifact_by_name} + for route in routes: + if route["artifact"] not in artifact_by_name: + raise ConfigError(f"route {route['name']} references unknown artifact {route['artifact']}") + references[route["artifact"]].append(route) + for name, used_by in references.items(): + if len(used_by) != 1: + raise ConfigError(f"artifact {name} must be referenced by exactly one route") + buckets, authorities, credential_owners = {}, {}, {} + for route in routes: + artifact = artifact_by_name[route["artifact"]] + if artifact["bucket"] in buckets: + other_access, other_name = buckets[artifact["bucket"]] + if other_access != route["access"]: + raise ConfigError(f"bucket {artifact['bucket']} cannot be reused by protected and public routes") + raise ConfigError(f"bucket {artifact['bucket']} must belong to one artifact ({other_name})") + buckets[artifact["bucket"]] = (route["access"], artifact["name"]) + if artifact["website_authority"] in authorities: + raise ConfigError( + f"website authority {artifact['website_authority']} must belong to one artifact" + ) + authorities[artifact["website_authority"]] = artifact["name"] + for variable in artifact["credentials"].values(): + if variable in credential_owners: + raise ConfigError( + f"publication credential {variable} is reused by artifacts " + f"{credential_owners[variable]} and {artifact['name']}" + ) + credential_owners[variable] = artifact["name"] + for cache_rule in artifact["cache_rules"]: + directives = {part.strip().lower().split("=", 1)[0] + for part in cache_rule["cache_control"].split(",")} + if route["access"] == "protected" and "public" in directives: + raise ConfigError(f"protected route {route['name']} cannot use public cache policy") + if route["access"] == "protected" and not directives & {"private", "no-store"}: + raise ConfigError(f"protected route {route['name']} cache policy must be private or no-store") + if route["access"] == "protected" and "s-maxage" in directives: + raise ConfigError(f"protected route {route['name']} cannot use shared-cache max-age") + if route["access"] == "public" and "private" in directives: + raise ConfigError(f"public route {route['name']} cannot use private cache policy") + root = next(route for route in routes if route["path"] == "/") + if any(route["access"] == "protected" for route in routes) and root["access"] == "public": + raise ConfigError("a public '/' catch-all would expose unmatched protected content") + + +def normalize_site_config(raw, site_name): + raw = _mapping(raw, "site.yaml") + domain = _hostname(raw.get("domain"), "domain") + if not isinstance(raw.get("enabled", True), bool): + raise ConfigError("enabled must be a boolean") + if ("artifacts" in raw) != ("routes" in raw): + raise ConfigError("artifacts and routes must be declared together") + if "artifacts" not in raw: + return _legacy_config(raw, site_name) + legacy_fields = {"type", "content_dir", "tidy", "excludes", "middlewares"} & set(raw) + if legacy_fields: + raise ConfigError("legacy fields cannot be mixed with artifacts/routes: " + ", ".join(sorted(legacy_fields))) + _known_keys(raw, {"domain", "aliases", "enabled", "artifacts", "routes"}, "site.yaml") + artifacts = [_artifact(item, index) for index, item in enumerate(_list(raw["artifacts"], "artifacts"))] + routes = [_route(item, index) for index, item in enumerate(_list(raw["routes"], "routes"))] + if not artifacts or not routes: + raise ConfigError("artifacts and routes must not be empty") + cfg = { + "version": 2, "compatibility": None, "domain": domain, + "aliases": _aliases(raw.get("aliases"), domain), + "enabled": raw.get("enabled", True), + "artifacts": sorted(artifacts, key=lambda item: item["name"]), + "routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])), + } + _validate_multi(cfg) + for route in cfg["routes"]: + if len(f"{k8s_name(site_name)}-{route['name']}") > 63: + raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters") + return cfg + + +def validate_artifact_inputs(site_dir, cfg): + """Reject source containment before a public or protected build starts.""" + root = Path(site_dir).resolve() + sources = [] + for artifact in cfg["artifacts"]: + source = (root / artifact["content_dir"]).resolve() + if source != root and root not in source.parents: + raise ConfigError( + f"artifact {artifact['name']} content_dir resolves outside the repository" + ) + sources.append((artifact["name"], source)) + for index, (name, source) in enumerate(sources): + for other_name, other_source in sources[index + 1:]: + if source == other_source or source in other_source.parents or other_source in source.parents: + raise ConfigError( + f"artifact build inputs overlap after resolution: {name} and {other_name}" + ) + + +def parse_site_yaml(site_dir, site_name=None): + path = Path(site_dir) / "site.yaml" + if not path.exists(): + die("site.yaml not found in repo root") + if site_name is None: + repo = os.environ.get("SITE_REPO", "") + site_name = repo.split("/", 1)[-1] if "/" in repo else Path(site_dir).name + try: + with open(path) as stream: + cfg = normalize_site_config(yaml.safe_load(stream), site_name) + except (ConfigError, yaml.YAMLError) as exc: + die(str(exc)) print("Site config:") - for k, v in site.items(): - print(f" {k}: {v}") - return site + print(f" domain: {cfg['domain']}") + print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}") + print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}") + print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}") + return cfg def clone_apps(token): + """Clone Apps without placing the credential in argv or output.""" user = env("CI_BOT_USER", "ci-bot") apps_dir = Path("/tmp/apps-deploy") if apps_dir.exists(): shutil.rmtree(apps_dir) - run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}") - run(f"git -C {apps_dir} config user.name {user}") - run(f"git -C {apps_dir} config user.email {user}@fritzlab.net") + clone_env = git_auth_env(token) + url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git" + run(["git", "clone", "--depth", "1", url, str(apps_dir)], + display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env) + run(["git", "-C", str(apps_dir), "config", "user.name", user]) + run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"]) return apps_dir def render_templates(action_dir, template_vars, app_dir, manifests_dir): - """Render Jinja2 templates for a static-content site.""" - templates_dir = Path(action_dir) / "templates" - jinja_env = Environment( - loader=FileSystemLoader(str(templates_dir)), - keep_trailing_newline=True, - ) - - tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2", - "kustomization.yaml.j2", "service.yaml.j2"] - - for tmpl_name in tmpl_names: - tmpl = jinja_env.get_template(tmpl_name) - rendered = tmpl.render(**template_vars) - out_name = tmpl_name.replace(".j2", "") - dest = app_dir / out_name if tmpl_name == "app.yaml.j2" else manifests_dir / out_name - dest.write_text(rendered) - print(f" Rendered {tmpl_name} -> {dest}") + """Render one certificate and deterministic per-route resources.""" + jinja_env = Environment(loader=FileSystemLoader(str(Path(action_dir) / "templates")), + keep_trailing_newline=True, undefined=StrictUndefined) + manifests_dir.mkdir(parents=True, exist_ok=True) + for child in manifests_dir.iterdir(): + if child.is_file() and child.suffix in {".yaml", ".yml"}: + child.unlink() + route_files = [] + for route in template_vars["routes"]: + stem = "" if template_vars["compatibility"] else f"-{route['name']}" + for kind in ("service", "ingress"): + out_name = f"{kind}{stem}.yaml" + destination = manifests_dir / out_name + destination.write_text(jinja_env.get_template(f"{kind}.yaml.j2").render( + **template_vars, route=route + )) + route_files.append(out_name) + print(f" Rendered {kind}.yaml.j2 -> {destination}") + common_vars = {**template_vars, "route_files": route_files} + for out_name, destination in { + "certificate.yaml": manifests_dir / "certificate.yaml", + "kustomization.yaml": manifests_dir / "kustomization.yaml", + "app.yaml": app_dir / "app.yaml", + }.items(): + destination.write_text(jinja_env.get_template(f"{out_name}.j2").render(**common_vars)) + print(f" Rendered {out_name}.j2 -> {destination}") -def commit_and_push(apps_dir, message): - run(f"git -C {apps_dir} add -A") - result = subprocess.run( - f"git -C {apps_dir} diff --cached --quiet", - shell=True, check=False, - ) +def git_auth_env(token): + """Return credential-safe Git authentication shared by clone and push.""" + auth_env = os.environ.copy() + auth_env["CI_BOT_TOKEN"] = token + auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh")) + auth_env["GIT_TERMINAL_PROMPT"] = "0" + return auth_env + + +def commit_and_push(apps_dir, message, token=None): + run(["git", "-C", str(apps_dir), "add", "-A"]) + result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False) if result.returncode == 0: print("No manifest changes to commit") return False - run(f"git -C {apps_dir} commit -m '{message}'") - run(f"git -C {apps_dir} push") + run(["git", "-C", str(apps_dir), "commit", "-m", message]) + push_env = git_auth_env(token) if token else None + run(["git", "-C", str(apps_dir), "push"], env=push_env) print("Manifests pushed — ArgoCD will sync") return True diff --git a/templates/ingress.yaml.j2 b/templates/ingress.yaml.j2 index e8b56c4..30ab0b9 100644 --- a/templates/ingress.yaml.j2 +++ b/templates/ingress.yaml.j2 @@ -1,12 +1,10 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ site_k8s }} + name: {{ route.resource_name }} namespace: {{ namespace }} -{%- if site_type != "docker" %} annotations: - traefik.ingress.kubernetes.io/router.middlewares: https-redirect@file,retry-upstream@file{% for m in middlewares %},{{ m }}@file{% endfor %} -{%- endif %} + traefik.ingress.kubernetes.io/router.middlewares: https-redirect@file,retry-upstream@file{% if route.access_middleware %},{{ route.access_middleware }}@file{% endif %}{% for m in route.middlewares %},{{ m }}@file{% endfor %} spec: ingressClassName: traefik tls: @@ -20,22 +18,22 @@ spec: - host: {{ domain }} http: paths: - - path: / + - path: {{ route.path }} pathType: Prefix backend: service: - name: {{ site_k8s }} + name: {{ route.resource_name }} port: number: 80 {%- for alias in aliases %} - host: {{ alias }} http: paths: - - path: / + - path: {{ route.path }} pathType: Prefix backend: service: - name: {{ site_k8s }} + name: {{ route.resource_name }} port: number: 80 {%- endfor %} diff --git a/templates/kustomization.yaml.j2 b/templates/kustomization.yaml.j2 index 14ebd47..1c71b4b 100644 --- a/templates/kustomization.yaml.j2 +++ b/templates/kustomization.yaml.j2 @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- service.yaml -- ingress.yaml +{% for route_file in route_files -%} +- {{ route_file }} +{% endfor -%} - certificate.yaml diff --git a/templates/service.yaml.j2 b/templates/service.yaml.j2 index d425207..c66a6bd 100644 --- a/templates/service.yaml.j2 +++ b/templates/service.yaml.j2 @@ -1,11 +1,15 @@ apiVersion: v1 kind: Service metadata: - name: {{ site_k8s }} + name: {{ route.resource_name }} namespace: {{ namespace }} +{%- if not compatibility %} + annotations: + traefik.ingress.kubernetes.io/service.passhostheader: "false" +{%- endif %} spec: type: ExternalName - externalName: garage.storage.svc.k8s.sjc001.fritzlab.net + externalName: {{ route.artifact_config.website_authority }} ports: - port: 80 targetPort: 80 diff --git a/tests/fixtures/legacy-site.yaml b/tests/fixtures/legacy-site.yaml new file mode 100644 index 0000000..f891e32 --- /dev/null +++ b/tests/fixtures/legacy-site.yaml @@ -0,0 +1,9 @@ +domain: example.fritzlab.net +type: static +content_dir: html +aliases: + - www.example.fritzlab.net +middlewares: + - response-headers +excludes: + - media/* diff --git a/tests/fixtures/split-site.yaml b/tests/fixtures/split-site.yaml new file mode 100644 index 0000000..bbb8ec8 --- /dev/null +++ b/tests/fixtures/split-site.yaml @@ -0,0 +1,42 @@ +domain: baseline.fritzlab.net +artifacts: + - name: portal + type: static + content_dir: portal/build + publish: + bucket: baseline-portal + credentials: + access_key_env: PORTAL_S3_ACCESS_KEY + secret_key_env: PORTAL_S3_SECRET_KEY + cache: + rules: + - path: / + cache_control: private, no-store + - name: distributions + type: static + content_dir: dist + publish: + bucket: baseline-dist + credentials: + access_key_env: DIST_S3_ACCESS_KEY + secret_key_env: DIST_S3_SECRET_KEY + cache: + rules: + - path: / + cache_control: public, max-age=0, must-revalidate + - path: releases + cache_control: public, max-age=31536000, immutable + - path: channels + cache_control: public, max-age=0, must-revalidate +routes: + - name: portal + path: / + artifact: portal + access: + mode: protected + middleware: authentik-forwardauth + - name: distributions + path: /dist + artifact: distributions + access: + mode: public diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..ee45428 --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,521 @@ +import base64 +import copy +import hashlib +import io +import json +import os +import socket +import subprocess +import sys +import tempfile +import threading +import unittest +from contextlib import redirect_stderr, redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urlsplit +from unittest.mock import patch + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import deploy +import build +import utils +from utils import ConfigError, normalize_site_config, validate_artifact_inputs + + +def fixture(name): + return yaml.safe_load((ROOT / "tests" / "fixtures" / name).read_text()) + + +class IPv6GitHTTPServer(ThreadingHTTPServer): + address_family = socket.AF_INET6 + + +class AuthenticatedGitHandler(BaseHTTPRequestHandler): + project_root = None + expected_authorization = None + + def log_message(self, _format, *_args): + pass + + def do_GET(self): + self._git_backend() + + def do_POST(self): + self._git_backend() + + def _git_backend(self): + if self.headers.get("Authorization") != self.expected_authorization: + self.send_response(401) + self.send_header("WWW-Authenticate", 'Basic realm="test"') + self.end_headers() + return + parsed = urlsplit(self.path) + length = int(self.headers.get("Content-Length", "0")) + request_body = self.rfile.read(length) if length else b"" + backend_env = os.environ.copy() + backend_env.update({ + "GIT_PROJECT_ROOT": str(self.project_root), + "GIT_HTTP_EXPORT_ALL": "1", + "PATH_INFO": parsed.path, + "QUERY_STRING": parsed.query, + "REQUEST_METHOD": self.command, + "CONTENT_TYPE": self.headers.get("Content-Type", ""), + "CONTENT_LENGTH": str(length), + "REMOTE_USER": "ci-bot", + "REMOTE_ADDR": "::1", + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_PROTOCOL": "HTTP/1.1", + }) + result = subprocess.run( + ["git", "http-backend"], input=request_body, env=backend_env, + capture_output=True, check=True, + ) + raw_headers, response_body = result.stdout.split(b"\r\n\r\n", 1) + headers, status = [], 200 + for line in raw_headers.decode().split("\r\n"): + name, value = line.split(":", 1) + if name.lower() == "status": + status = int(value.strip().split(" ", 1)[0]) + else: + headers.append((name, value.strip())) + self.send_response(status) + for name, value in headers: + self.send_header(name, value) + self.end_headers() + self.wfile.write(response_body) + + +class ConfigContractTests(unittest.TestCase): + def setUp(self): + self.raw = fixture("split-site.yaml") + + def assert_invalid(self, mutate, message): + raw = copy.deepcopy(self.raw) + mutate(raw) + with self.assertRaisesRegex(ConfigError, message): + normalize_site_config(raw, "baseline.fritzlab.net") + + def test_legacy_normalizes_to_explicit_compatibility_surface(self): + cfg = normalize_site_config(fixture("legacy-site.yaml"), "example.fritzlab.net") + self.assertEqual("single-surface-v1", cfg["compatibility"]) + self.assertEqual("build/html", cfg["artifacts"][0]["build_dir"]) + self.assertEqual("example.fritzlab.net", cfg["artifacts"][0]["bucket"]) + self.assertEqual("/", cfg["routes"][0]["path"]) + self.assertEqual("legacy", cfg["routes"][0]["access"]) + + def test_routes_are_sorted_longest_prefix_first(self): + cfg = normalize_site_config(self.raw, "baseline.fritzlab.net") + self.assertEqual(["/dist", "/"], [route["path"] for route in cfg["routes"]]) + credentials = {artifact["name"]: artifact["credentials"] for artifact in cfg["artifacts"]} + self.assertNotEqual(credentials["distributions"], credentials["portal"]) + + def test_equivalent_route_paths_are_ambiguous(self): + self.assert_invalid(lambda raw: raw["routes"].append({ + "name": "duplicate", "path": "/dist/", "artifact": "portal", + "access": {"mode": "protected", "middleware": "authentik-forwardauth"}, + }), "ambiguous") + + def test_catch_all_is_required(self): + self.assert_invalid(lambda raw: raw["routes"].__setitem__(0, { + **raw["routes"][0], "path": "/portal" + }), "catch-all") + + def test_public_catch_all_is_rejected_when_any_route_is_protected(self): + def mutate(raw): + raw["routes"][0]["access"] = {"mode": "public"} + raw["routes"][1]["access"] = { + "mode": "protected", "middleware": "authentik-forwardauth" + } + raw["artifacts"][0]["cache"]["rules"][0]["cache_control"] = ( + "public, max-age=0, must-revalidate" + ) + for rule in raw["artifacts"][1]["cache"]["rules"]: + rule["cache_control"] = "private, no-store" + self.assert_invalid(mutate, "public '/' catch-all") + + def test_protected_route_requires_middleware(self): + self.assert_invalid( + lambda raw: raw["routes"][0].__setitem__("access", {"mode": "protected"}), + "middleware is required", + ) + + def test_bucket_cannot_cross_access_boundary(self): + self.assert_invalid( + lambda raw: raw["artifacts"][0]["publish"].__setitem__("bucket", "baseline-dist"), + "protected and public", + ) + + def test_publication_credentials_cannot_be_reused(self): + def mutate(raw): + raw["artifacts"][0]["publish"]["credentials"] = copy.deepcopy( + raw["artifacts"][1]["publish"]["credentials"] + ) + self.assert_invalid(mutate, "publication credential DIST_S3_ACCESS_KEY is reused") + + def test_cache_directive_contradiction_is_rejected(self): + self.assert_invalid( + lambda raw: raw["artifacts"][1]["cache"]["rules"][1].__setitem__( + "cache_control", "public, max-age=31536000, immutable, must-revalidate" + ), + "immutable requires", + ) + + def test_cache_policy_cannot_nest_below_immutable_path(self): + def mutate(raw): + raw["artifacts"][1]["cache"]["rules"].append({ + "path": "releases/candidates", + "cache_control": "public, max-age=0, must-revalidate", + }) + self.assert_invalid(mutate, "cannot nest another policy under immutable /releases") + + def test_public_cache_is_rejected_on_protected_route(self): + self.assert_invalid( + lambda raw: raw["artifacts"][0]["cache"]["rules"][0].__setitem__( + "cache_control", "public, max-age=0, must-revalidate" + ), + "protected route portal", + ) + + def test_legacy_and_split_fields_cannot_mix(self): + self.assert_invalid(lambda raw: raw.__setitem__("type", "static"), "cannot be mixed") + + def test_unknown_split_field_is_rejected(self): + self.assert_invalid( + lambda raw: raw["artifacts"][0].__setitem__("storage_bucket", "typo"), + "unknown fields: storage_bucket", + ) + + def test_backend_authority_and_endpoint_are_derived(self): + for field, value in ( + ("endpoint", "https://attacker.example"), + ("website_authority", "internal-api.default.svc.k8s.sjc001.fritzlab.net"), + ): + with self.subTest(field=field): + self.assert_invalid( + lambda raw, field=field, value=value: raw["artifacts"][0]["publish"].__setitem__(field, value), + f"unknown fields: {field}", + ) + + def test_publication_credentials_use_dedicated_matched_names(self): + self.assert_invalid( + lambda raw: raw["artifacts"][0]["publish"]["credentials"].__setitem__( + "access_key_env", "CI_BOT_TOKEN" + ), + "must be a matched _S3_ACCESS_KEY", + ) + + def test_resolved_build_inputs_must_be_pairwise_disjoint(self): + raw = copy.deepcopy(self.raw) + raw["artifacts"][1]["content_dir"] = "" + cfg = normalize_site_config(raw, "baseline.fritzlab.net") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "portal" / "build").mkdir(parents=True) + with self.assertRaisesRegex(ConfigError, "build inputs overlap after resolution"): + validate_artifact_inputs(root, cfg) + + +class GenerationTests(unittest.TestCase): + def render(self, raw): + cfg = normalize_site_config(raw, "baseline.fritzlab.net") + tmp = tempfile.TemporaryDirectory() + root = Path(tmp.name) + app_dir = root / "app" + manifests = app_dir / "manifests" + app_dir.mkdir(parents=True) + deploy.render_site_manifests( + "baseline.fritzlab.net", ROOT, app_dir, manifests, cfg + ) + files = {path.relative_to(app_dir).as_posix(): path.read_text() + for path in sorted(app_dir.rglob("*.yaml"))} + return tmp, app_dir, files + + def test_split_fixture_generates_per_route_resources_and_one_certificate(self): + tmp, _, files = self.render(fixture("split-site.yaml")) + self.addCleanup(tmp.cleanup) + self.assertEqual({ + "app.yaml", "manifests/certificate.yaml", "manifests/ingress-distributions.yaml", + "manifests/ingress-portal.yaml", "manifests/kustomization.yaml", + "manifests/service-distributions.yaml", "manifests/service-portal.yaml", + }, set(files)) + for content in files.values(): + self.assertIsNotNone(yaml.safe_load(content)) + self.assertIn("path: /dist", files["manifests/ingress-distributions.yaml"]) + self.assertNotIn("authentik-forwardauth", files["manifests/ingress-distributions.yaml"]) + self.assertIn("authentik-forwardauth@file", files["manifests/ingress-portal.yaml"]) + self.assertIn('service.passhostheader: "false"', files["manifests/service-portal.yaml"]) + self.assertNotIn("passhostheader", files["manifests/ingress-portal.yaml"]) + self.assertIn("baseline-dist.web.sjc001.fritzlab.net", files["manifests/service-distributions.yaml"]) + + def test_generation_is_deterministic_when_input_lists_are_reversed(self): + raw = fixture("split-site.yaml") + first_tmp, _, first = self.render(raw) + self.addCleanup(first_tmp.cleanup) + raw["artifacts"].reverse() + raw["routes"].reverse() + second_tmp, _, second = self.render(raw) + self.addCleanup(second_tmp.cleanup) + self.assertEqual(first, second) + + def test_stale_route_manifests_are_removed(self): + raw = fixture("split-site.yaml") + cfg = normalize_site_config(raw, "baseline.fritzlab.net") + with tempfile.TemporaryDirectory() as tmp: + app_dir = Path(tmp) / "app" + manifests = app_dir / "manifests" + manifests.mkdir(parents=True) + stale = manifests / "ingress-removed.yaml" + stale.write_text("stale\n") + deploy.render_site_manifests("baseline.fritzlab.net", ROOT, app_dir, manifests, cfg) + self.assertFalse(stale.exists()) + + def test_legacy_names_and_garage_s3_target_are_preserved(self): + tmp, _, files = self.render(fixture("legacy-site.yaml")) + self.addCleanup(tmp.cleanup) + self.assertIn("manifests/service.yaml", files) + self.assertIn("manifests/ingress.yaml", files) + self.assertIn("garage-s3.storage.svc.k8s.sjc001.fritzlab.net", files["manifests/service.yaml"]) + self.assertNotIn("passhostheader", files["manifests/ingress.yaml"]) + + +class BuildTests(unittest.TestCase): + def test_artifacts_build_independently_without_clobbering_siblings(self): + raw = fixture("split-site.yaml") + for artifact in raw["artifacts"]: + artifact["tidy"] = False + cfg = normalize_site_config(raw, "baseline.fritzlab.net") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "dist").mkdir() + (root / "dist" / "bundle.js").write_text("bundle") + (root / "portal" / "build").mkdir(parents=True) + (root / "portal" / "build" / "index.html").write_text("portal") + for artifact in cfg["artifacts"]: + build.build_artifact(root, artifact) + self.assertEqual( + "bundle", (root / ".site-publish/distributions/html/bundle.js").read_text() + ) + self.assertEqual( + "portal", (root / ".site-publish/portal/html/index.html").read_text() + ) + + +class PublishingTests(unittest.TestCase): + def test_apps_clone_never_places_token_in_argv_or_log(self): + calls = [] + secret = "clone-secret-must-not-appear" + with patch.dict(os.environ, {"CI_BOT_USER": "ci-bot"}, clear=False), \ + patch.object(utils, "run", side_effect=lambda command, **kwargs: calls.append((command, kwargs))), \ + patch.object(utils.shutil, "rmtree"), redirect_stdout(io.StringIO()) as output: + utils.clone_apps(secret) + self.assertNotIn(secret, output.getvalue()) + for command, kwargs in calls: + self.assertNotIn(secret, " ".join(command)) + self.assertNotIn(secret, kwargs.get("display", "")) + + def test_askpass_authenticates_clone_and_push_round_trip(self): + token = "round-trip-token" + expected = "Basic " + base64.b64encode(f"ci-bot:{token}".encode()).decode() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bare = root / "repo.git" + seed = root / "seed" + checkout = root / "checkout" + subprocess.run(["git", "init", "--bare", "--initial-branch=main", str(bare)], check=True, + stdout=subprocess.DEVNULL) + subprocess.run(["git", "-C", str(bare), "config", "http.receivepack", "true"], check=True) + subprocess.run(["git", "init", "--initial-branch=main", str(seed)], check=True, + stdout=subprocess.DEVNULL) + subprocess.run(["git", "-C", str(seed), "config", "user.name", "Test"], check=True) + subprocess.run(["git", "-C", str(seed), "config", "user.email", "test@example.invalid"], check=True) + (seed / "README.md").write_text("seed\n") + subprocess.run(["git", "-C", str(seed), "add", "README.md"], check=True) + subprocess.run(["git", "-C", str(seed), "commit", "-m", "seed"], check=True, + stdout=subprocess.DEVNULL) + subprocess.run(["git", "-C", str(seed), "push", str(bare), "main"], check=True, + stdout=subprocess.DEVNULL) + + handler = type("GitHandler", (AuthenticatedGitHandler,), { + "project_root": root, "expected_authorization": expected, + }) + server = IPv6GitHTTPServer(("::1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://ci-bot@[::1]:{server.server_port}/repo.git" + with patch.dict(os.environ, {"NO_PROXY": "::1,[::1]", "no_proxy": "::1,[::1]"}, clear=False): + subprocess.run( + ["git", "clone", url, str(checkout)], env=utils.git_auth_env(token), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Test"], check=True) + subprocess.run(["git", "-C", str(checkout), "config", "user.email", "test@example.invalid"], + check=True) + (checkout / "roundtrip.txt").write_text("authenticated\n") + utils.commit_and_push(checkout, "authenticated round trip", token) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + result = subprocess.run( + ["git", "--git-dir", str(bare), "show", "main:roundtrip.txt"], + check=True, text=True, capture_output=True, + ) + self.assertEqual("authenticated\n", result.stdout) + + def test_cache_headers_credentials_and_route_prefix_are_separate(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions") + route = next(item for item in cfg["routes"] if item["artifact"] == "distributions") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + html = root / artifact["build_dir"] + (html / "releases").mkdir(parents=True) + (html / "channels").mkdir() + (html / "releases" / "1.0.js").write_text("release") + (html / "channels" / "stable.json").write_text("channel") + commands, events = [], [] + + def capture(command, **kwargs): + commands.append((command, kwargs["env"])) + events.append("mutable") + + def publish_immutable(*_args): + events.append("immutable") + + secret = "secret-must-not-appear" + output = io.StringIO() + with patch.dict(os.environ, { + "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret + }, clear=False), patch.object(deploy, "run", side_effect=capture), \ + patch.object(deploy, "publish_immutable_rule", side_effect=publish_immutable) as immutable_publish, \ + redirect_stdout(output): + deploy.s3_sync(artifact, route, root) + + self.assertTrue(all(secret not in " ".join(command) for command, _ in commands)) + self.assertEqual("immutable", events[0]) + self.assertNotIn(secret, output.getvalue()) + self.assertTrue(all(call_env["AWS_ACCESS_KEY_ID"] == "dist-key" for _, call_env in commands)) + self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands)) + rendered = [" ".join(command) for command, _ in commands] + self.assertIn("s3://baseline-dist/dist/", rendered[0]) + self.assertIn("releases/*", rendered[0]) + self.assertNotIn("--delete", rendered[0]) + self.assertIn("--delete", rendered[1]) + self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:])) + self.assertTrue(any("channels/" in command and + "public, max-age=0, must-revalidate" in command + for command in rendered)) + immutable_publish.assert_called_once() + self.assertEqual( + "public, max-age=31536000, immutable", + immutable_publish.call_args.args[2]["cache_control"], + ) + + + def test_absent_artifact_is_detected_before_publish(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + with tempfile.TemporaryDirectory() as tmp, redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit): + deploy.validate_artifact_output(Path(tmp), cfg["artifacts"][0]) + + def test_absent_cache_prefix_is_detected(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions") + with tempfile.TemporaryDirectory() as tmp: + html = Path(tmp) / artifact["build_dir"] + html.mkdir(parents=True) + (html / "index.html").write_text("content") + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + deploy.validate_artifact_output(Path(tmp), artifact) + + +class ImmutablePublicationTests(unittest.TestCase): + CACHE = "public, max-age=31536000, immutable" + + def result(self, returncode, stdout="", stderr=""): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + def key_and_digests(self, source): + content_type = "text/javascript" + content_digest, publication_digest = deploy._immutable_digests( + source, self.CACHE, content_type, + ) + return f"dist/releases/release-{publication_digest}.js", content_digest, publication_digest + + def test_new_key_uses_content_address_and_digest_metadata(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "release.js" + source.write_text("fixed release") + key, content_digest, publication_digest = self.key_and_digests(source) + calls = [] + + def capture(args, _env): + calls.append(args) + return self.result(1, stderr="404 Not Found") if len(calls) == 1 else self.result(0, "{}") + + with patch.object(deploy, "_aws_capture", side_effect=capture): + created = deploy.publish_immutable_file( + "http://garage-s3.storage.svc:3900", "dist-bucket", + key, source, self.CACHE, {}, + ) + self.assertTrue(created) + put = calls[1] + self.assertEqual("put-object", put[4]) + self.assertNotIn("--if-none-match", put) + self.assertEqual( + f"sha256={content_digest},publication-sha256={publication_digest}", + put[put.index("--metadata") + 1], + ) + + def test_identical_retry_converges_without_put(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "release.js" + source.write_text("fixed release") + key, content_digest, publication_digest = self.key_and_digests(source) + head = json.dumps({ + "Metadata": {"sha256": content_digest, "publication-sha256": publication_digest}, + "CacheControl": self.CACHE, + "ContentType": "text/javascript", + }) + with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)) as request: + created = deploy.publish_immutable_file( + "http://garage-s3.storage.svc:3900", "dist-bucket", + key, source, self.CACHE, {}, + ) + self.assertFalse(created) + self.assertEqual(1, request.call_count) + + def test_changed_immutable_key_is_refused(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "release.js" + source.write_text("changed release") + key, _, _ = self.key_and_digests(source) + head = json.dumps({"Metadata": {"sha256": "different"}, "CacheControl": self.CACHE}) + with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)), \ + self.assertRaisesRegex(RuntimeError, "immutable object differs"): + deploy.publish_immutable_file( + "http://garage-s3.storage.svc:3900", "dist-bucket", + key, source, self.CACHE, {}, + ) + + def test_immutable_key_without_publication_digest_is_refused_before_s3(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "release.js" + source.write_text("fixed release") + with patch.object(deploy, "_aws_capture") as request, \ + self.assertRaisesRegex(RuntimeError, "must contain its one publication SHA-256"): + deploy.publish_immutable_file( + "http://garage-s3.storage.svc:3900", "dist-bucket", + "dist/releases/release.js", source, self.CACHE, {}, + ) + request.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 5261b9b99faa7f94f19cbd3ab585290ccd8c3fe5 Mon Sep 17 00:00:00 2001 From: Dave Kowalski Date: Sat, 29 Aug 2026 22:17:24 +0000 Subject: [PATCH 2/4] test(site-publish): prove immutable failure ordering Authored-By: OpenAI (GPT-5) --- README.md | 4 ++++ tests/test_contract.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/README.md b/README.md index d4c7b49..6cdc5a4 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ cache policy, content type, and bytes. That content address makes concurrent writes identical even though Garage v2.2.0 has no conditional destination write. An identical retry converges; a changed object, missing digest metadata, wrong address, or nested policy under that immutable prefix fails publication. +Every immutable target is validated and published before mutable objects change. +Mutable default and override partitions receive their final cache policy before +the matching prefix-scoped stale deletion, so publication never exposes a +provisional cache policy or a pointer to a missing immutable target. Artifact input directories must be pairwise disjoint after filesystem resolution. Publication stops before build or upload if one contains another or diff --git a/tests/test_contract.py b/tests/test_contract.py index ee45428..e9be6e9 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -417,6 +417,29 @@ class PublishingTests(unittest.TestCase): immutable_publish.call_args.args[2]["cache_control"], ) + def test_immutable_failure_stops_before_mutable_publication(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions") + route = next(item for item in cfg["routes"] if item["artifact"] == "distributions") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + html = root / artifact["build_dir"] + (html / "releases").mkdir(parents=True) + (html / "channels").mkdir() + (html / "releases" / "1.0.js").write_text("release") + (html / "channels" / "stable.json").write_text("channel") + + with patch.dict(os.environ, { + "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": "dist-secret" + }, clear=False), patch.object( + deploy, "publish_immutable_rule", side_effect=RuntimeError("immutable failed") + ), patch.object(deploy, "run") as mutable_run, self.assertRaisesRegex( + RuntimeError, "immutable failed" + ): + deploy.s3_sync(artifact, route, root) + + mutable_run.assert_not_called() + def test_absent_artifact_is_detected_before_publish(self): cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") From fb2e440bbb7df928d567a078b15ebaab9cf599c1 Mon Sep 17 00:00:00 2001 From: Evelyn Chen Date: Sat, 29 Aug 2026 22:23:51 +0000 Subject: [PATCH 3/4] fix(site-publish): preflight all immutable routes Authored-By: OpenAI (GPT-5) --- README.md | 3 ++- scripts/deploy.py | 34 +++++++++++++++++++++++++--------- tests/test_contract.py | 27 +++++++++++---------------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 6cdc5a4..e01dd87 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,8 @@ cache policy, content type, and bytes. That content address makes concurrent writes identical even though Garage v2.2.0 has no conditional destination write. An identical retry converges; a changed object, missing digest metadata, wrong address, or nested policy under that immutable prefix fails publication. -Every immutable target is validated and published before mutable objects change. +Every immutable target across every artifact is validated and published before +any route's mutable objects change. Mutable default and override partitions receive their final cache policy before the matching prefix-scoped stale deletion, so publication never exposes a provisional cache policy or a pointer to a missing immutable target. diff --git a/scripts/deploy.py b/scripts/deploy.py index d7354c9..025189d 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -149,9 +149,8 @@ def publish_immutable_rule(artifact, route, rule, html_dir, aws_env): ) -def s3_sync(artifact, route, site_dir, credential_env_names=None): - endpoint = artifact["s3_endpoint"] - html_dir = site_dir / artifact["build_dir"] +def publication_aws_env(artifact, credential_env_names=None): + """Build the route-scoped AWS environment without leaking other credentials.""" access_key = env(artifact["credentials"]["access_key_env"]) secret_key = env(artifact["credentials"]["secret_key_env"]) aws_env = os.environ.copy() @@ -165,6 +164,22 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None): "AWS_SECRET_ACCESS_KEY": secret_key, "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"), }) + return aws_env + + +def publish_route_immutables(artifact, route, site_dir, credential_env_names=None): + """Publish one route's immutable partitions during the global preflight.""" + html_dir = site_dir / artifact["build_dir"] + aws_env = publication_aws_env(artifact, credential_env_names) + for rule in artifact["cache_rules"]: + if _is_immutable(rule): + publish_immutable_rule(artifact, route, rule, html_dir, aws_env) + + +def s3_sync(artifact, route, site_dir, credential_env_names=None): + endpoint = artifact["s3_endpoint"] + html_dir = site_dir / artifact["build_dir"] + aws_env = publication_aws_env(artifact, credential_env_names) bucket = artifact["bucket"] object_prefix = route["path"].strip("/") destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}" @@ -176,12 +191,6 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None): exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)] if artifact["excludes"]: print(f"Excluding patterns: {artifact['excludes']}") - # Validate and publish every append-only target before a mutable channel can - # point at it. Partial immutable success is safe; partial mutable success is - # not. - for rule in artifact["cache_rules"]: - if _is_immutable(rule): - publish_immutable_rule(artifact, route, rule, html_dir, aws_env) print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}") # Upload with the final cache policy before cleanup. Sync and deletion are # scoped to the same current route prefix and cache partition. A route move @@ -296,6 +305,13 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg): validate_publication_environment(cfg) for artifact in cfg["artifacts"]: validate_artifact_output(site_dir, artifact) + # Complete immutable work across the whole publication before any route's + # mutable pointers can change. Partial immutable success is safe; mixing a + # new route with an old route after a later immutable failure is not. + for route in cfg["routes"]: + publish_route_immutables( + artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, + ) for route in cfg["routes"]: s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names) if cfg["compatibility"]: diff --git a/tests/test_contract.py b/tests/test_contract.py index e9be6e9..3452d9d 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -395,6 +395,7 @@ class PublishingTests(unittest.TestCase): }, clear=False), patch.object(deploy, "run", side_effect=capture), \ patch.object(deploy, "publish_immutable_rule", side_effect=publish_immutable) as immutable_publish, \ redirect_stdout(output): + deploy.publish_route_immutables(artifact, route, root) deploy.s3_sync(artifact, route, root) self.assertTrue(all(secret not in " ".join(command) for command, _ in commands)) @@ -417,28 +418,22 @@ class PublishingTests(unittest.TestCase): immutable_publish.call_args.args[2]["cache_control"], ) - def test_immutable_failure_stops_before_mutable_publication(self): + def test_later_route_immutable_failure_stops_all_mutable_publication(self): cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") - artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions") - route = next(item for item in cfg["routes"] if item["artifact"] == "distributions") with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) - html = root / artifact["build_dir"] - (html / "releases").mkdir(parents=True) - (html / "channels").mkdir() - (html / "releases" / "1.0.js").write_text("release") - (html / "channels" / "stable.json").write_text("channel") - - with patch.dict(os.environ, { - "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": "dist-secret" - }, clear=False), patch.object( - deploy, "publish_immutable_rule", side_effect=RuntimeError("immutable failed") - ), patch.object(deploy, "run") as mutable_run, self.assertRaisesRegex( + with patch.object(deploy, "validate_publication_environment"), \ + patch.object(deploy, "validate_artifact_output"), patch.object( + deploy, "publish_route_immutables", + side_effect=[None, RuntimeError("immutable failed")], + ) as immutable_publish, patch.object(deploy, "s3_sync") as mutable_sync, \ + self.assertRaisesRegex( RuntimeError, "immutable failed" ): - deploy.s3_sync(artifact, route, root) + deploy.deploy_static("baseline", root, root, "token", cfg) - mutable_run.assert_not_called() + self.assertEqual(2, immutable_publish.call_count) + mutable_sync.assert_not_called() def test_absent_artifact_is_detected_before_publish(self): From 7b824a61ca3784c5416a57d2f84c5c176d7dbf1c Mon Sep 17 00:00:00 2001 From: Evelyn Chen Date: Sat, 29 Aug 2026 22:30:43 +0000 Subject: [PATCH 4/4] fix(site-publish): reject split input symlinks Authored-By: OpenAI (GPT-5) --- README.md | 5 +++-- scripts/utils.py | 9 +++++++++ tests/test_contract.py | 11 +++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e01dd87..6a2c886 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,9 @@ provisional cache policy or a pointer to a missing immutable target. Artifact input directories must be pairwise disjoint after filesystem resolution. Publication stops before build or upload if one contains another or -escapes the repository, preventing protected input from entering a public -artifact. Split storage endpoints are pinned to Garage, and each website +escapes the repository. Descendant symlinks are also rejected, preventing +protected input from entering a public artifact through dereference. Split +storage endpoints are pinned to Garage, and each website authority is derived from its bucket; a site cannot expose an arbitrary backend. Each split route gets a bucket-specific `.web.sjc001.fritzlab.net` diff --git a/scripts/utils.py b/scripts/utils.py index a5a5ad9..8376cd5 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -427,6 +427,8 @@ def normalize_site_config(raw, site_name): def validate_artifact_inputs(site_dir, cfg): """Reject source containment before a public or protected build starts.""" + if cfg["compatibility"]: + return root = Path(site_dir).resolve() sources = [] for artifact in cfg["artifacts"]: @@ -435,6 +437,13 @@ def validate_artifact_inputs(site_dir, cfg): raise ConfigError( f"artifact {artifact['name']} content_dir resolves outside the repository" ) + if source.exists(): + symlink = next((path for path in source.rglob("*") if path.is_symlink()), None) + if symlink is not None: + raise ConfigError( + f"artifact {artifact['name']} build input contains symlink: " + f"{symlink.relative_to(root)}" + ) sources.append((artifact["name"], source)) for index, (name, source) in enumerate(sources): for other_name, other_source in sources[index + 1:]: diff --git a/tests/test_contract.py b/tests/test_contract.py index 3452d9d..a8c239f 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -219,6 +219,17 @@ class ConfigContractTests(unittest.TestCase): with self.assertRaisesRegex(ConfigError, "build inputs overlap after resolution"): validate_artifact_inputs(root, cfg) + def test_descendant_symlink_cannot_cross_artifact_boundary(self): + cfg = normalize_site_config(self.raw, "baseline.fritzlab.net") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "dist").mkdir() + (root / "portal" / "build").mkdir(parents=True) + (root / "portal" / "build" / "private.txt").write_text("private") + (root / "dist" / "portal-link").symlink_to(root / "portal" / "build") + with self.assertRaisesRegex(ConfigError, "build input contains symlink"): + validate_artifact_inputs(root, cfg) + class GenerationTests(unittest.TestCase): def render(self, raw):