feat(site-publish): add split-surface publishing

Authored-By: OpenAI (GPT-5) <noreply@openai.com>

scripts/deploy.py

@@ -1,17 +1,18 @@
 """Deploy phase — S3 sync, manifest rendering, alias reconcile."""
 
+import fnmatch
+import hashlib
 import json
+import mimetypes
 import os
-import shlex
+import re
 import shutil
-import tempfile
+import subprocess
 from pathlib import Path
 from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
 
 from utils import (
-    DEFAULT_S3_ENDPOINT,
-    GITEA_HOST,
     NAMESPACE,
     clone_apps,
     commit_and_push,
@@ -21,6 +22,7 @@ from utils import (
     parse_site_yaml,
     render_templates,
     run,
+    validate_artifact_inputs,
 )
 
 GARAGE_ADMIN_ENDPOINT = os.environ.get(
@@ -28,44 +30,194 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
 )
 
 
-CACHE_CONTROL = "public, max-age=0, must-revalidate"
+def validate_artifact_output(site_dir, artifact):
+    """Prove every artifact and declared cache prefix exists before publishing any."""
+    html_dir = site_dir / artifact["build_dir"]
+    if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")):
+        die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}")
+    for rule in artifact["cache_rules"]:
+        if not rule["path"]:
+            continue
+        cache_root = html_dir / rule["path"]
+        if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")):
+            die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files")
 
 
-def s3_sync(site_name, site_dir, excludes=None):
-    endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
-    html_dir = site_dir / "build" / "html"
-    if not html_dir.exists():
-        die(f"build/html not found — did the build step run? ({html_dir})")
-    env("AWS_ACCESS_KEY_ID")
-    env("AWS_SECRET_ACCESS_KEY")
-    os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
+def validate_publication_environment(cfg):
+    """Resolve every declared credential before the first bucket is changed."""
+    for artifact in cfg["artifacts"]:
+        env(artifact["credentials"]["access_key_env"])
+        env(artifact["credentials"]["secret_key_env"])
+    if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
+        die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
+
+
+def _is_immutable(rule):
+    return "immutable" in {
+        part.strip().lower().split("=", 1)[0]
+        for part in rule["cache_control"].split(",")
+    }
+
+
+def _aws_capture(args, aws_env):
+    """Run a non-streaming AWS request without exposing environment credentials."""
+    print(f"  $ {' '.join(str(part) for part in args)}")
+    return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
+
+
+def _immutable_head(endpoint, bucket, key, aws_env):
+    args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
+            "--bucket", bucket, "--key", key, "--output", "json"]
+    result = _aws_capture(args, aws_env)
+    if result.returncode == 0:
+        return json.loads(result.stdout)
+    error = f"{result.stdout}\n{result.stderr}"
+    if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
+        return None
+    raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
+
+
+def _immutable_digests(source, cache_control, content_type):
+    with source.open("rb") as stream:
+        content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
+    publication = hashlib.sha256()
+    publication.update(cache_control.encode())
+    publication.update(b"\0")
+    publication.update(content_type.encode())
+    publication.update(b"\0")
+    with source.open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            publication.update(block)
+    return content_digest, publication.hexdigest()
+
+
+def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
+    metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
+    return (
+        metadata.get("sha256") == content_digest
+        and metadata.get("publication-sha256") == publication_digest
+        and info.get("CacheControl") == cache_control
+        and info.get("ContentType") == content_type
+    )
+
+
+def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
+    """Publish a content-addressed key; identical retries converge."""
+    content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
+    content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
+    address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
+    if address_digests != [publication_digest]:
+        raise RuntimeError(
+            f"immutable key must contain its one publication SHA-256 {publication_digest}: "
+            f"s3://{bucket}/{key}"
+        )
+    existing = _immutable_head(endpoint, bucket, key, aws_env)
+    if existing is not None:
+        if _same_immutable_object(
+            existing, content_digest, publication_digest, cache_control, content_type,
+        ):
+            print(f"  Immutable object already matches: s3://{bucket}/{key}")
+            return False
+        raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
+    args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
+            "--bucket", bucket, "--key", key, "--body", str(source),
+            "--content-type", content_type, "--cache-control", cache_control,
+            "--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
+    result = _aws_capture(args, aws_env)
+    if result.returncode == 0:
+        return True
+    error = f"{result.stdout}\n{result.stderr}"
+    raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
+
+
+def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
+    """Publish one immutable cache partition without overwrite or deletion."""
+    rule_root = html_dir / rule["path"]
+    child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
+                   if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
+    object_prefix = route["path"].strip("/")
+    for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
+        relative = source.relative_to(html_dir).as_posix()
+        if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
+            continue
+        if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
+            continue
+        key = "/".join(part for part in (object_prefix, relative) if part)
+        publish_immutable_file(
+            artifact["s3_endpoint"], artifact["bucket"], key, source,
+            rule["cache_control"], aws_env,
+        )
+
+
+def s3_sync(artifact, route, site_dir, credential_env_names=None):
+    endpoint = artifact["s3_endpoint"]
+    html_dir = site_dir / artifact["build_dir"]
+    access_key = env(artifact["credentials"]["access_key_env"])
+    secret_key = env(artifact["credentials"]["secret_key_env"])
+    aws_env = os.environ.copy()
+    for name in credential_env_names or artifact["credentials"].values():
+        aws_env.pop(name, None)
+    for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE",
+                 "AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"):
+        aws_env.pop(name, None)
+    aws_env.update({
+        "AWS_ACCESS_KEY_ID": access_key,
+        "AWS_SECRET_ACCESS_KEY": secret_key,
+        "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
+    })
+    bucket = artifact["bucket"]
+    object_prefix = route["path"].strip("/")
+    destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}"
+    default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
+    immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
     # `excludes` are patterns (site.yaml `excludes:` list) that should never
     # be uploaded *and* should never be deleted from the bucket — escape hatch
     # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
-    exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
-    if excludes:
-        print(f"Excluding patterns: {excludes}")
-    print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
-    # `sync --delete` handles new/changed/orphaned files. `cp --recursive`
-    # then re-uploads everything to refresh metadata (cache-control,
-    # content-type) on objects sync skipped because nothing changed.
-    # Cost: a no-op deploy still re-uploads every byte. Sites here are
-    # small enough that that's free; correctness wins over throughput.
+    exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)]
+    if artifact["excludes"]:
+        print(f"Excluding patterns: {artifact['excludes']}")
+    # Validate and publish every append-only target before a mutable channel can
+    # point at it. Partial immutable success is safe; partial mutable success is
+    # not.
+    for rule in artifact["cache_rules"]:
+        if _is_immutable(rule):
+            publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
+    print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
+    # Upload with the final cache policy before cleanup. Sync and deletion are
+    # scoped to the same current route prefix and cache partition. A route move
+    # leaves its old bucket partition intact but unreachable after the old
+    # Ingress disappears, while stale mutable keys on the serving prefix are
+    # deleted. Immutable subtrees are structurally excluded. `cp --recursive`
+    # refreshes metadata atomically per object before `sync --delete` removes
+    # stale keys without ever exposing new bytes under a provisional policy.
+    # A no-op deploy therefore transfers the artifact bytes once.
     # AWS CLI guesses Content-Type from file extension on local→S3 uploads,
     # so a fresh upload always carries the right MIME type.
-    run(
-        f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
-        f"--delete --only-show-errors "
-        f"--cache-control '{CACHE_CONTROL}' "
-        f"{exclude_flags}".rstrip()
-    )
-    print("Re-stamping metadata on all objects...")
-    run(
-        f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
-        f"--recursive --only-show-errors "
-        f"--cache-control '{CACHE_CONTROL}' "
-        f"{exclude_flags}".rstrip()
-    )
+    specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
+    default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
+    run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
+         "--recursive", "--only-show-errors", "--cache-control", default_cache,
+         *default_filters, *exclude_args], env=aws_env)
+    run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
+         "--delete", "--only-show-errors", "--cache-control", default_cache,
+         *default_filters, *exclude_args], env=aws_env)
+    for rule in artifact["cache_rules"]:
+        if not rule["path"]:
+            continue
+        if _is_immutable(rule):
+            continue
+        include = f"{rule['path'].rstrip('/')}/*"
+        child_filters = [arg for path in specific_paths
+                         if path.startswith(f"{rule['path'].rstrip('/')}/")
+                         for arg in ("--exclude", f"{path}/*")]
+        # Apply rules from the artifact root so artifact-level exclusions keep
+        # their original meaning under every cache override.
+        run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
+             "--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
+             "--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
+        run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
+             "--delete", "--only-show-errors", "--cache-control", rule["cache_control"],
+             "--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
 
 
 def garage_admin(method, path, token, body=None):
@@ -89,15 +241,13 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
     if not aliases:
         return
     if not admin_token:
-        print("  (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
-        return
+        die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
 
     try:
         info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
                             admin_token)
     except (HTTPError, URLError) as e:
-        print(f"  WARNING: bucket lookup failed: {e}")
-        return
+        raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
 
     bucket_id = info.get("id")
     existing = set(info.get("globalAliases") or [])
@@ -120,20 +270,36 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
     """Always re-render manifests from current site.yaml. Templates own
     domain + aliases, so changes propagate without manual edits."""
     manifests_dir.mkdir(parents=True, exist_ok=True)
+    artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
+    routes = []
+    for route in cfg["routes"]:
+        artifact = artifact_by_name[route["artifact"]]
+        resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
+        routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
     template_vars = {
         "site": site_name,
         "site_k8s": k8s_name(site_name),
         "domain": cfg["domain"],
         "aliases": cfg["aliases"],
         "namespace": NAMESPACE,
-        "middlewares": cfg["middlewares"],
+        "compatibility": cfg["compatibility"],
+        "routes": routes,
     }
     render_templates(action_dir, template_vars, app_dir, manifests_dir)
 
 
 def deploy_static(site_name, site_dir, action_dir, token, cfg):
-    s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
-    ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
+    artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
+    credential_env_names = {
+        name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
+    }
+    validate_publication_environment(cfg)
+    for artifact in cfg["artifacts"]:
+        validate_artifact_output(site_dir, artifact)
+    for route in cfg["routes"]:
+        s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
+    if cfg["compatibility"]:
+        ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
 
     apps_dir = clone_apps(token)
     app_dir = apps_dir / "sjc001" / "websites" / site_name
@@ -141,25 +307,21 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
 
     render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
 
-    commit_and_push(apps_dir, f"Deploy {site_name}")
+    commit_and_push(apps_dir, f"Deploy {site_name}", token)
 
 
-def decommission(site_name, token):
+def decommission(site_name, token, buckets=None):
     """Remove manifests from apps repo."""
-    user = env("CI_BOT_USER", "ci-bot")
-    with tempfile.TemporaryDirectory() as tmp:
-        apps_dir = Path(tmp)
-        run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
-        site_path = apps_dir / "sjc001" / "websites" / site_name
-        if not site_path.exists():
-            print(f"No manifests for {site_name} — nothing to remove")
-            return
-        shutil.rmtree(site_path)
-        run(f"git -C {apps_dir} config user.name {user}")
-        run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
-        commit_and_push(apps_dir, f"Decommission {site_name}")
-    print(f"Bucket {site_name} and its objects are NOT purged automatically.")
-    print(f"  garage bucket delete {site_name} --yes")
+    apps_dir = clone_apps(token)
+    site_path = apps_dir / "sjc001" / "websites" / site_name
+    if not site_path.exists():
+        print(f"No manifests for {site_name} — nothing to remove")
+        return
+    shutil.rmtree(site_path)
+    commit_and_push(apps_dir, f"Decommission {site_name}", token)
+    for bucket in buckets or [site_name]:
+        print(f"Bucket {bucket} and its objects are NOT purged automatically.")
+        print(f"  garage bucket delete {bucket} --yes")
 
 
 def cmd_deploy():
@@ -173,7 +335,8 @@ def cmd_deploy():
 
     if not cfg["enabled"]:
         print("Site disabled — running decommission...")
-        decommission(site_name, token)
+        decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
         return
 
+    validate_artifact_inputs(site_dir, cfg)
     deploy_static(site_name, site_dir, action_dir, token, cfg)