feat: publish isolated site artifacts

scripts/deploy.py

@@ -2,16 +2,15 @@
 
 import json
 import os
-import shlex
 import shutil
 import tempfile
 from pathlib import Path
 from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit
 from urllib.request import Request, urlopen
 
 from utils import (
     DEFAULT_S3_ENDPOINT,
-    GITEA_HOST,
     NAMESPACE,
     clone_apps,
     commit_and_push,
@@ -20,7 +19,7 @@ from utils import (
     k8s_name,
     parse_site_yaml,
     render_templates,
-    run,
+    run_args,
 )
 
 GARAGE_ADMIN_ENDPOINT = os.environ.get(
@@ -31,21 +30,66 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
 CACHE_CONTROL = "public, max-age=0, must-revalidate"
 
 
-def s3_sync(site_name, site_dir, excludes=None):
-    endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
-    html_dir = site_dir / "build" / "html"
-    if not html_dir.exists():
-        die(f"build/html not found — did the build step run? ({html_dir})")
-    env("AWS_ACCESS_KEY_ID")
-    env("AWS_SECRET_ACCESS_KEY")
-    os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
+def credential_environment(profile):
+    if profile == "default":
+        access_key = env("AWS_ACCESS_KEY_ID")
+        secret_key = env("AWS_SECRET_ACCESS_KEY")
+    else:
+        prefix = f"SITE_PUBLISH_{profile.upper().replace('-', '_')}"
+        access_key = env(f"{prefix}_S3_ACCESS_KEY_ID")
+        secret_key = env(f"{prefix}_S3_SECRET_ACCESS_KEY")
+    child_env = os.environ.copy()
+    child_env["AWS_ACCESS_KEY_ID"] = access_key
+    child_env["AWS_SECRET_ACCESS_KEY"] = secret_key
+    child_env.setdefault("AWS_DEFAULT_REGION", "sjc001")
+    return child_env
+
+
+def configure_cors(bucket, origins, endpoint, child_env):
+    if origins is None:
+        return
+    if not origins:
+        run_args(
+            ["aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
+             "--bucket", bucket],
+            env=child_env,
+        )
+        return
+    config = {
+        "CORSRules": [{
+            "AllowedOrigins": origins,
+            "AllowedMethods": ["GET", "HEAD"],
+            "AllowedHeaders": ["*"],
+            "ExposeHeaders": ["ETag"],
+            "MaxAgeSeconds": 3600,
+        }]
+    }
+    with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
+        json.dump(config, handle)
+        handle.flush()
+        run_args(
+            ["aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
+             "--bucket", bucket, "--cors-configuration", f"file://{handle.name}"],
+            env=child_env,
+        )
+
+
+def s3_sync(bucket, source, *, credential="default", endpoint=None,
+            cache=None, cors_origins=None, excludes=None):
+    endpoint = endpoint or os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
+    if not source.exists():
+        die(f"staged artifact not found — did the build step run? ({source})")
+    child_env = credential_environment(credential)
+    cache = cache or {"default": CACHE_CONTROL, "rules": []}
     # `excludes` are patterns (site.yaml `excludes:` list) that should never
     # be uploaded *and* should never be deleted from the bucket — escape hatch
     # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
-    exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
+    exclude_args = []
+    for pattern in excludes or []:
+        exclude_args.extend(["--exclude", pattern])
     if excludes:
         print(f"Excluding patterns: {excludes}")
-    print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
+    print(f"Syncing {source} → s3://{bucket} via {endpoint}")
     # `sync --delete` handles new/changed/orphaned files. `cp --recursive`
     # then re-uploads everything to refresh metadata (cache-control,
     # content-type) on objects sync skipped because nothing changed.
@@ -53,22 +97,63 @@ def s3_sync(site_name, site_dir, excludes=None):
     # small enough that that's free; correctness wins over throughput.
     # AWS CLI guesses Content-Type from file extension on local→S3 uploads,
     # so a fresh upload always carries the right MIME type.
-    run(
-        f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
-        f"--delete --only-show-errors "
-        f"--cache-control '{CACHE_CONTROL}' "
-        f"{exclude_flags}".rstrip()
+    run_args(
+        ["aws", "--endpoint-url", endpoint, "s3", "sync", f"{source}/", f"s3://{bucket}/",
+         "--delete", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
+        env=child_env,
     )
     print("Re-stamping metadata on all objects...")
-    run(
-        f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
-        f"--recursive --only-show-errors "
-        f"--cache-control '{CACHE_CONTROL}' "
-        f"{exclude_flags}".rstrip()
+    run_args(
+        ["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
+         "--recursive", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
+        env=child_env,
     )
+    for rule in cache["rules"]:
+        print(f"Applying cache metadata for {rule['match']}")
+        run_args(
+            ["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
+             "--recursive", "--only-show-errors", "--exclude", "*",
+             "--include", rule["match"], "--cache-control", rule["value"], *exclude_args],
+            env=child_env,
+        )
+    configure_cors(bucket, cors_origins, endpoint, child_env)
+
+
+def preflight_artifacts(site_dir, cfg):
+    """Validate every source, credential, and bucket before the first write."""
+    for artifact in cfg["artifacts"].values():
+        source = site_dir / ".site-publish" / artifact["name"]
+        if not source.is_dir() or not any(entry.is_file() for entry in source.rglob("*")):
+            die(f"staged artifact is absent or empty: {source}")
+        child_env = credential_environment(artifact["credential"])
+        endpoint = artifact["endpoint"] or os.environ.get(
+            "GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT
+        )
+        run_args(
+            ["aws", "--endpoint-url", endpoint, "s3api", "head-bucket",
+             "--bucket", artifact["bucket"]],
+            env=child_env,
+        )
+
+
+def routed_cache(cache, routes):
+    """Translate source-relative cache patterns to their public object keys."""
+    rules = []
+    for route in routes:
+        prefix = route["path"].lstrip("/")
+        for rule in cache["rules"]:
+            pattern = f"{prefix}/{rule['match']}" if prefix else rule["match"]
+            rules.append({"match": pattern, "value": rule["value"]})
+    return {"default": cache["default"], "rules": rules}
 
 
 def garage_admin(method, path, token, body=None):
+    parsed = urlsplit(GARAGE_ADMIN_ENDPOINT)
+    if (
+        parsed.scheme not in {"http", "https"} or not parsed.netloc
+        or parsed.username or parsed.password or parsed.query or parsed.fragment
+    ):
+        die("GARAGE_ADMIN_ENDPOINT must be an HTTP URL without credentials")
     url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
     data = json.dumps(body).encode() if body is not None else None
     headers = {"Authorization": f"Bearer {token}"}
@@ -89,17 +174,17 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
     if not aliases:
         return
     if not admin_token:
-        print("  (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
-        return
+        die("GARAGE_ADMIN_TOKEN is required when aliases are configured")
 
     try:
         info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
                             admin_token)
     except (HTTPError, URLError) as e:
-        print(f"  WARNING: bucket lookup failed: {e}")
-        return
+        raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
 
     bucket_id = info.get("id")
+    if not isinstance(bucket_id, str) or not bucket_id:
+        die(f"bucket lookup for {site_name} returned no bucket id")
     existing = set(info.get("globalAliases") or [])
     print(f"  Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
 
@@ -119,21 +204,92 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
 def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
     """Always re-render manifests from current site.yaml. Templates own
     domain + aliases, so changes propagate without manual edits."""
-    manifests_dir.mkdir(parents=True, exist_ok=True)
+    if manifests_dir.exists():
+        shutil.rmtree(manifests_dir)
+    manifests_dir.mkdir(parents=True)
+    site_k8s = k8s_name(site_name)
+
+    if cfg["mode"] == "legacy":
+        artifacts = [{
+            "name": "site",
+            "bucket": site_name,
+            "service_name": site_k8s,
+            "external_name": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
+            "virtual_host": False,
+        }]
+        routes = [{
+            "name": "site",
+            "path": "/",
+            "artifact": "site",
+            "service_name": site_k8s,
+            "ingress_name": site_k8s,
+            "middlewares": cfg["middlewares"],
+            "middleware_refs": [f"{name}@file" for name in cfg["middlewares"]],
+        }]
+    else:
+        artifacts = []
+        artifact_by_name = {}
+        for name, artifact in cfg["artifacts"].items():
+            service_name = k8s_name(f"{site_name}-{name}")
+            view = {
+                **artifact,
+                "service_name": service_name,
+                "external_name": f"{artifact['bucket']}.web.sjc001.fritzlab.net",
+                "virtual_host": True,
+            }
+            artifacts.append(view)
+            artifact_by_name[name] = view
+        routes = []
+        for route in cfg["routes"]:
+            artifact = artifact_by_name[route["artifact"]]
+            ingress_name = k8s_name(f"{site_name}-{route['name']}")
+            middleware_refs = [f"{name}@file" for name in route["middlewares"]]
+            routes.append({
+                **route,
+                "service_name": artifact["service_name"],
+                "ingress_name": ingress_name,
+                "middleware_refs": middleware_refs,
+            })
+
     template_vars = {
         "site": site_name,
-        "site_k8s": k8s_name(site_name),
+        "site_k8s": site_k8s,
         "domain": cfg["domain"],
         "aliases": cfg["aliases"],
         "namespace": NAMESPACE,
-        "middlewares": cfg["middlewares"],
+        "artifacts": artifacts,
+        "routes": routes,
+        "modern": cfg["mode"] == "multi",
     }
     render_templates(action_dir, template_vars, app_dir, manifests_dir)
 
 
 def deploy_static(site_name, site_dir, action_dir, token, cfg):
-    s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
-    ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
+    if cfg["mode"] == "legacy":
+        s3_sync(
+            site_name,
+            site_dir / "build" / "html",
+            excludes=cfg.get("excludes"),
+        )
+        ensure_bucket_aliases(
+            site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")
+        )
+    else:
+        preflight_artifacts(site_dir, cfg)
+        for artifact in cfg["artifacts"].values():
+            routes = [
+                route for route in cfg["routes"]
+                if route["artifact"] == artifact["name"]
+            ]
+            s3_sync(
+                artifact["bucket"],
+                site_dir / ".site-publish" / artifact["name"],
+                credential=artifact["credential"],
+                endpoint=artifact["endpoint"],
+                cache=routed_cache(artifact["cache"], routes),
+                cors_origins=artifact["cors_origins"],
+                excludes=artifact["excludes"],
+            )
 
     apps_dir = clone_apps(token)
     app_dir = apps_dir / "sjc001" / "websites" / site_name
@@ -141,23 +297,24 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
 
     render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
 
-    commit_and_push(apps_dir, f"Deploy {site_name}")
+    try:
+        commit_and_push(apps_dir, f"Deploy {site_name}", token)
+    finally:
+        shutil.rmtree(apps_dir.parent, ignore_errors=True)
 
 
 def decommission(site_name, token):
     """Remove manifests from apps repo."""
-    user = env("CI_BOT_USER", "ci-bot")
-    with tempfile.TemporaryDirectory() as tmp:
-        apps_dir = Path(tmp)
-        run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
+    apps_dir = clone_apps(token)
+    try:
         site_path = apps_dir / "sjc001" / "websites" / site_name
         if not site_path.exists():
             print(f"No manifests for {site_name} — nothing to remove")
             return
         shutil.rmtree(site_path)
-        run(f"git -C {apps_dir} config user.name {user}")
-        run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
-        commit_and_push(apps_dir, f"Decommission {site_name}")
+        commit_and_push(apps_dir, f"Decommission {site_name}", token)
+    finally:
+        shutil.rmtree(apps_dir.parent, ignore_errors=True)
     print(f"Bucket {site_name} and its objects are NOT purged automatically.")
     print(f"  garage bucket delete {site_name} --yes")