site-publish/scripts/deploy.py

"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""

import json
import os
import shutil
import tempfile
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import Request, urlopen

from utils import (
    DEFAULT_S3_ENDPOINT,
    NAMESPACE,
    clone_apps,
    commit_and_push,
    die,
    env,
    k8s_name,
    parse_site_yaml,
    render_templates,
    run_args,
)

GARAGE_ADMIN_ENDPOINT = os.environ.get(
    "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
)


CACHE_CONTROL = "public, max-age=0, must-revalidate"


def credential_environment(profile):
    if profile == "default":
        access_key = env("AWS_ACCESS_KEY_ID")
        secret_key = env("AWS_SECRET_ACCESS_KEY")
    else:
        prefix = f"SITE_PUBLISH_{profile.upper().replace('-', '_')}"
        access_key = env(f"{prefix}_S3_ACCESS_KEY_ID")
        secret_key = env(f"{prefix}_S3_SECRET_ACCESS_KEY")
    child_env = os.environ.copy()
    child_env["AWS_ACCESS_KEY_ID"] = access_key
    child_env["AWS_SECRET_ACCESS_KEY"] = secret_key
    child_env.setdefault("AWS_DEFAULT_REGION", "sjc001")
    return child_env


def configure_cors(bucket, origins, endpoint, child_env):
    if origins is None:
        return
    if not origins:
        run_args(
            ["aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
             "--bucket", bucket],
            env=child_env,
        )
        return
    config = {
        "CORSRules": [{
            "AllowedOrigins": origins,
            "AllowedMethods": ["GET", "HEAD"],
            "AllowedHeaders": ["*"],
            "ExposeHeaders": ["ETag"],
            "MaxAgeSeconds": 3600,
        }]
    }
    with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
        json.dump(config, handle)
        handle.flush()
        run_args(
            ["aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
             "--bucket", bucket, "--cors-configuration", f"file://{handle.name}"],
            env=child_env,
        )


def s3_sync(bucket, source, *, credential="default", endpoint=None,
            cache=None, cors_origins=None, excludes=None):
    endpoint = endpoint or os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
    if not source.exists():
        die(f"staged artifact not found — did the build step run? ({source})")
    child_env = credential_environment(credential)
    cache = cache or {"default": CACHE_CONTROL, "rules": []}
    # `excludes` are patterns (site.yaml `excludes:` list) that should never
    # be uploaded *and* should never be deleted from the bucket — escape hatch
    # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
    exclude_args = []
    for pattern in excludes or []:
        exclude_args.extend(["--exclude", pattern])
    if excludes:
        print(f"Excluding patterns: {excludes}")
    print(f"Syncing {source} → s3://{bucket} via {endpoint}")
    # `sync --delete` handles new/changed/orphaned files. `cp --recursive`
    # then re-uploads everything to refresh metadata (cache-control,
    # content-type) on objects sync skipped because nothing changed.
    # Cost: a no-op deploy still re-uploads every byte. Sites here are
    # small enough that that's free; correctness wins over throughput.
    # AWS CLI guesses Content-Type from file extension on local→S3 uploads,
    # so a fresh upload always carries the right MIME type.
    run_args(
        ["aws", "--endpoint-url", endpoint, "s3", "sync", f"{source}/", f"s3://{bucket}/",
         "--delete", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
        env=child_env,
    )
    print("Re-stamping metadata on all objects...")
    run_args(
        ["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
         "--recursive", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
        env=child_env,
    )
    for rule in cache["rules"]:
        print(f"Applying cache metadata for {rule['match']}")
        run_args(
            ["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
             "--recursive", "--only-show-errors", "--exclude", "*",
             "--include", rule["match"], "--cache-control", rule["value"], *exclude_args],
            env=child_env,
        )
    configure_cors(bucket, cors_origins, endpoint, child_env)


def preflight_artifacts(site_dir, cfg):
    """Validate every source, credential, and bucket before the first write."""
    for artifact in cfg["artifacts"].values():
        source = site_dir / ".site-publish" / artifact["name"]
        if not source.is_dir() or not any(entry.is_file() for entry in source.rglob("*")):
            die(f"staged artifact is absent or empty: {source}")
        child_env = credential_environment(artifact["credential"])
        endpoint = artifact["endpoint"] or os.environ.get(
            "GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT
        )
        run_args(
            ["aws", "--endpoint-url", endpoint, "s3api", "head-bucket",
             "--bucket", artifact["bucket"]],
            env=child_env,
        )


def routed_cache(cache, routes):
    """Translate source-relative cache patterns to their public object keys."""
    rules = []
    for route in routes:
        prefix = route["path"].lstrip("/")
        for rule in cache["rules"]:
            pattern = f"{prefix}/{rule['match']}" if prefix else rule["match"]
            rules.append({"match": pattern, "value": rule["value"]})
    return {"default": cache["default"], "rules": rules}


def garage_admin(method, path, token, body=None):
    parsed = urlsplit(GARAGE_ADMIN_ENDPOINT)
    if (
        parsed.scheme not in {"http", "https"} or not parsed.netloc
        or parsed.username or parsed.password or parsed.query or parsed.fragment
    ):
        die("GARAGE_ADMIN_ENDPOINT must be an HTTP URL without credentials")
    url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
    data = json.dumps(body).encode() if body is not None else None
    headers = {"Authorization": f"Bearer {token}"}
    if data is not None:
        headers["Content-Type"] = "application/json"
    req = Request(url, data=data, method=method, headers=headers)
    with urlopen(req) as resp:
        raw = resp.read()
        return json.loads(raw) if raw else {}


def ensure_bucket_aliases(site_name, aliases, admin_token):
    """Add cfg['aliases'] as Garage globalAliases on the site bucket.

    Idempotent: skips aliases already present. Never removes aliases not in
    the desired set (safety — orphan removal is manual).
    """
    if not aliases:
        return
    if not admin_token:
        die("GARAGE_ADMIN_TOKEN is required when aliases are configured")

    try:
        info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
                            admin_token)
    except (HTTPError, URLError) as e:
        raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e

    bucket_id = info.get("id")
    if not isinstance(bucket_id, str) or not bucket_id:
        die(f"bucket lookup for {site_name} returned no bucket id")
    existing = set(info.get("globalAliases") or [])
    print(f"  Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")

    for alias in aliases:
        if alias in existing:
            continue
        print(f"  Adding globalAlias: {alias}")
        try:
            garage_admin("POST", "/v2/AddBucketAlias", admin_token,
                         {"bucketId": bucket_id, "globalAlias": alias})
        except HTTPError as e:
            body = e.read().decode(errors="replace") if hasattr(e, "read") else ""
            print(f"  ERROR adding alias {alias}: {e} {body}")
            raise


def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
    """Always re-render manifests from current site.yaml. Templates own
    domain + aliases, so changes propagate without manual edits."""
    if manifests_dir.exists():
        shutil.rmtree(manifests_dir)
    manifests_dir.mkdir(parents=True)
    site_k8s = k8s_name(site_name)

    if cfg["mode"] == "legacy":
        artifacts = [{
            "name": "site",
            "bucket": site_name,
            "service_name": site_k8s,
            "external_name": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
            "virtual_host": False,
        }]
        routes = [{
            "name": "site",
            "path": "/",
            "artifact": "site",
            "service_name": site_k8s,
            "ingress_name": site_k8s,
            "middlewares": cfg["middlewares"],
            "middleware_refs": [f"{name}@file" for name in cfg["middlewares"]],
        }]
    else:
        artifacts = []
        artifact_by_name = {}
        for name, artifact in cfg["artifacts"].items():
            service_name = k8s_name(f"{site_name}-{name}")
            view = {
                **artifact,
                "service_name": service_name,
                "external_name": f"{artifact['bucket']}.web.sjc001.fritzlab.net",
                "virtual_host": True,
            }
            artifacts.append(view)
            artifact_by_name[name] = view
        routes = []
        for route in cfg["routes"]:
            artifact = artifact_by_name[route["artifact"]]
            ingress_name = k8s_name(f"{site_name}-{route['name']}")
            middleware_refs = [f"{name}@file" for name in route["middlewares"]]
            routes.append({
                **route,
                "service_name": artifact["service_name"],
                "ingress_name": ingress_name,
                "middleware_refs": middleware_refs,
            })

    template_vars = {
        "site": site_name,
        "site_k8s": site_k8s,
        "domain": cfg["domain"],
        "aliases": cfg["aliases"],
        "namespace": NAMESPACE,
        "artifacts": artifacts,
        "routes": routes,
        "modern": cfg["mode"] == "multi",
    }
    render_templates(action_dir, template_vars, app_dir, manifests_dir)


def deploy_static(site_name, site_dir, action_dir, token, cfg):
    if cfg["mode"] == "legacy":
        s3_sync(
            site_name,
            site_dir / "build" / "html",
            excludes=cfg.get("excludes"),
        )
        ensure_bucket_aliases(
            site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")
        )
    else:
        preflight_artifacts(site_dir, cfg)
        for artifact in cfg["artifacts"].values():
            routes = [
                route for route in cfg["routes"]
                if route["artifact"] == artifact["name"]
            ]
            s3_sync(
                artifact["bucket"],
                site_dir / ".site-publish" / artifact["name"],
                credential=artifact["credential"],
                endpoint=artifact["endpoint"],
                cache=routed_cache(artifact["cache"], routes),
                cors_origins=artifact["cors_origins"],
                excludes=artifact["excludes"],
            )

    apps_dir = clone_apps(token)
    app_dir = apps_dir / "sjc001" / "websites" / site_name
    manifests_dir = app_dir / "manifests"

    render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)

    try:
        commit_and_push(apps_dir, f"Deploy {site_name}", token)
    finally:
        shutil.rmtree(apps_dir.parent, ignore_errors=True)


def decommission(site_name, token):
    """Remove manifests from apps repo."""
    apps_dir = clone_apps(token)
    try:
        site_path = apps_dir / "sjc001" / "websites" / site_name
        if not site_path.exists():
            print(f"No manifests for {site_name} — nothing to remove")
            return
        shutil.rmtree(site_path)
        commit_and_push(apps_dir, f"Decommission {site_name}", token)
    finally:
        shutil.rmtree(apps_dir.parent, ignore_errors=True)
    print(f"Bucket {site_name} and its objects are NOT purged automatically.")
    print(f"  garage bucket delete {site_name} --yes")


def cmd_deploy():
    site_repo = env("SITE_REPO")
    site_dir = Path(env("SITE_DIR"))
    action_dir = Path(env("ACTION_DIR"))
    token = env("CI_BOT_TOKEN")
    site_name = site_repo.split("/", 1)[1]

    cfg = parse_site_yaml(site_dir)

    if not cfg["enabled"]:
        print("Site disabled — running decommission...")
        decommission(site_name, token)
        return

    deploy_static(site_name, site_dir, action_dir, token, cfg)