feat: add split-surface site publishing

scripts/deploy.py

@@ -2,7 +2,6 @@
 
 import json
 import os
-import shlex
 import shutil
 import tempfile
 from pathlib import Path
@@ -10,8 +9,6 @@ from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
 
 from utils import (
-    DEFAULT_S3_ENDPOINT,
-    GITEA_HOST,
     NAMESPACE,
     clone_apps,
     commit_and_push,
@@ -28,44 +25,100 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
 )
 
 
-CACHE_CONTROL = "public, max-age=0, must-revalidate"
+def validate_artifact_output(site_dir, artifact):
+    """Prove every artifact and declared cache prefix exists before publishing any."""
+    html_dir = site_dir / artifact["build_dir"]
+    if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")):
+        die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}")
+    for rule in artifact["cache_rules"]:
+        if not rule["path"]:
+            continue
+        cache_root = html_dir / rule["path"]
+        if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")):
+            die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files")
 
 
-def s3_sync(site_name, site_dir, excludes=None):
-    endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
-    html_dir = site_dir / "build" / "html"
-    if not html_dir.exists():
-        die(f"build/html not found — did the build step run? ({html_dir})")
-    env("AWS_ACCESS_KEY_ID")
-    env("AWS_SECRET_ACCESS_KEY")
-    os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
+def validate_publication_environment(cfg):
+    """Resolve every declared credential before the first bucket is changed."""
+    for artifact in cfg["artifacts"]:
+        env(artifact["credentials"]["access_key_env"])
+        env(artifact["credentials"]["secret_key_env"])
+    if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
+        die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
+
+
+def s3_sync(artifact, route, site_dir, credential_env_names=None):
+    endpoint = artifact["s3_endpoint"]
+    html_dir = site_dir / artifact["build_dir"]
+    access_key = env(artifact["credentials"]["access_key_env"])
+    secret_key = env(artifact["credentials"]["secret_key_env"])
+    aws_env = os.environ.copy()
+    for name in credential_env_names or artifact["credentials"].values():
+        aws_env.pop(name, None)
+    for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE",
+                 "AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"):
+        aws_env.pop(name, None)
+    aws_env.update({
+        "AWS_ACCESS_KEY_ID": access_key,
+        "AWS_SECRET_ACCESS_KEY": secret_key,
+        "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
+    })
+    bucket = artifact["bucket"]
+    object_prefix = route["path"].strip("/")
+    bucket_destination = f"s3://{bucket}/"
+    destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}"
+    default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
     # `excludes` are patterns (site.yaml `excludes:` list) that should never
     # be uploaded *and* should never be deleted from the bucket — escape hatch
     # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
-    exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
-    if excludes:
-        print(f"Excluding patterns: {excludes}")
-    print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
-    # `sync --delete` handles new/changed/orphaned files. `cp --recursive`
-    # then re-uploads everything to refresh metadata (cache-control,
-    # content-type) on objects sync skipped because nothing changed.
-    # Cost: a no-op deploy still re-uploads every byte. Sites here are
-    # small enough that that's free; correctness wins over throughput.
+    exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)]
+    if artifact["excludes"]:
+        print(f"Excluding patterns: {artifact['excludes']}")
+    print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
+    # `sync --delete` handles new/changed/orphaned files. Partitioned
+    # `cp --recursive` calls then re-upload each file once to refresh metadata
+    # (cache-control, content-type) on objects sync skipped as unchanged.
+    # A no-op deploy therefore transfers the artifact bytes once.
     # AWS CLI guesses Content-Type from file extension on local→S3 uploads,
     # so a fresh upload always carries the right MIME type.
-    run(
-        f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
-        f"--delete --only-show-errors "
-        f"--cache-control '{CACHE_CONTROL}' "
-        f"{exclude_flags}".rstrip()
-    )
+    stage = None
+    sync_source = html_dir
+    sync_excludes = exclude_args
+    if object_prefix:
+        stage = tempfile.TemporaryDirectory()
+        sync_source = Path(stage.name)
+        staged_artifact = sync_source / object_prefix
+        staged_artifact.parent.mkdir(parents=True, exist_ok=True)
+        staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True)
+        sync_excludes = [arg for pattern in artifact["excludes"]
+                         for arg in ("--exclude", f"{object_prefix}/{pattern}")]
+    try:
+        # Sync the complete bucket authority so moving a route prefix also
+        # deletes objects under its old prefix instead of leaving them public.
+        run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{sync_source}/", bucket_destination,
+             "--delete", "--only-show-errors", "--cache-control", default_cache,
+             *sync_excludes], env=aws_env)
+    finally:
+        if stage:
+            stage.cleanup()
     print("Re-stamping metadata on all objects...")
-    run(
-        f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
-        f"--recursive --only-show-errors "
-        f"--cache-control '{CACHE_CONTROL}' "
-        f"{exclude_flags}".rstrip()
-    )
+    specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
+    default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
+    run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
+         "--recursive", "--only-show-errors", "--cache-control", default_cache,
+         *default_filters, *exclude_args], env=aws_env)
+    for rule in artifact["cache_rules"]:
+        if not rule["path"]:
+            continue
+        include = f"{rule['path'].rstrip('/')}/*"
+        child_filters = [arg for path in specific_paths
+                         if path.startswith(f"{rule['path'].rstrip('/')}/")
+                         for arg in ("--exclude", f"{path}/*")]
+        # Apply rules from the artifact root so artifact-level exclusions keep
+        # their original meaning under every cache override.
+        run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
+             "--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
+             "--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
 
 
 def garage_admin(method, path, token, body=None):
@@ -89,15 +142,13 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
     if not aliases:
         return
     if not admin_token:
-        print("  (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
-        return
+        die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
 
     try:
         info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
                             admin_token)
     except (HTTPError, URLError) as e:
-        print(f"  WARNING: bucket lookup failed: {e}")
-        return
+        raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
 
     bucket_id = info.get("id")
     existing = set(info.get("globalAliases") or [])
@@ -120,20 +171,36 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
     """Always re-render manifests from current site.yaml. Templates own
     domain + aliases, so changes propagate without manual edits."""
     manifests_dir.mkdir(parents=True, exist_ok=True)
+    artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
+    routes = []
+    for route in cfg["routes"]:
+        artifact = artifact_by_name[route["artifact"]]
+        resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
+        routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
     template_vars = {
         "site": site_name,
         "site_k8s": k8s_name(site_name),
         "domain": cfg["domain"],
         "aliases": cfg["aliases"],
         "namespace": NAMESPACE,
-        "middlewares": cfg["middlewares"],
+        "compatibility": cfg["compatibility"],
+        "routes": routes,
     }
     render_templates(action_dir, template_vars, app_dir, manifests_dir)
 
 
 def deploy_static(site_name, site_dir, action_dir, token, cfg):
-    s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
-    ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
+    artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
+    credential_env_names = {
+        name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
+    }
+    validate_publication_environment(cfg)
+    for artifact in cfg["artifacts"]:
+        validate_artifact_output(site_dir, artifact)
+    for route in cfg["routes"]:
+        s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
+    if cfg["compatibility"]:
+        ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
 
     apps_dir = clone_apps(token)
     app_dir = apps_dir / "sjc001" / "websites" / site_name
@@ -144,22 +211,18 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
     commit_and_push(apps_dir, f"Deploy {site_name}")
 
 
-def decommission(site_name, token):
+def decommission(site_name, token, buckets=None):
     """Remove manifests from apps repo."""
-    user = env("CI_BOT_USER", "ci-bot")
-    with tempfile.TemporaryDirectory() as tmp:
-        apps_dir = Path(tmp)
-        run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
-        site_path = apps_dir / "sjc001" / "websites" / site_name
-        if not site_path.exists():
-            print(f"No manifests for {site_name} — nothing to remove")
-            return
-        shutil.rmtree(site_path)
-        run(f"git -C {apps_dir} config user.name {user}")
-        run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
-        commit_and_push(apps_dir, f"Decommission {site_name}")
-    print(f"Bucket {site_name} and its objects are NOT purged automatically.")
-    print(f"  garage bucket delete {site_name} --yes")
+    apps_dir = clone_apps(token)
+    site_path = apps_dir / "sjc001" / "websites" / site_name
+    if not site_path.exists():
+        print(f"No manifests for {site_name} — nothing to remove")
+        return
+    shutil.rmtree(site_path)
+    commit_and_push(apps_dir, f"Decommission {site_name}")
+    for bucket in buckets or [site_name]:
+        print(f"Bucket {bucket} and its objects are NOT purged automatically.")
+        print(f"  garage bucket delete {bucket} --yes")
 
 
 def cmd_deploy():
@@ -173,7 +236,7 @@ def cmd_deploy():
 
     if not cfg["enabled"]:
         print("Site disabled — running decommission...")
-        decommission(site_name, token)
+        decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
         return
 
     deploy_static(site_name, site_dir, action_dir, token, cfg)