site-publish/scripts/deploy.py

"""Deploy phase — artifact publication and split-route manifest rendering."""

import fnmatch
import hashlib
import json
import mimetypes
import os
import shutil
import subprocess
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import quote
from urllib.request import Request, urlopen

try:
    from botocore.auth import S3SigV4Auth
    from botocore.awsrequest import AWSRequest
    from botocore.credentials import Credentials
except ModuleNotFoundError:  # aws-cli v1's bundled botocore layout
    from awscli.botocore.auth import S3SigV4Auth
    from awscli.botocore.awsrequest import AWSRequest
    from awscli.botocore.credentials import Credentials

from utils import (
    DEFAULT_S3_ENDPOINT, GARAGE_WEBSITE_HOST, NAMESPACE, clone_apps,
    commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run,
)

GARAGE_ADMIN_ENDPOINT = os.environ.get(
    "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903")


def _aws_env(artifact):
    credentials = artifact["credentials"]
    access = env(credentials["access_key_env"])
    secret = env(credentials["secret_key_env"])
    child = os.environ.copy()
    child.update({"AWS_ACCESS_KEY_ID": access, "AWS_SECRET_ACCESS_KEY": secret,
                  "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001")})
    return child


def _aws(endpoint, operation):
    return ["aws", "--endpoint-url", endpoint, *operation]


def _excluded(relative, patterns):
    return any(fnmatch.fnmatch(relative, pattern) for pattern in patterns)


def _conditional_put(endpoint, artifact, key, path, content_type, digest):
    """Atomically create one S3 object without exposing signing credentials."""
    selectors = artifact["credentials"]
    credentials = Credentials(env(selectors["access_key_env"]),
                              env(selectors["secret_key_env"]))
    region = os.environ.get("AWS_DEFAULT_REGION", "sjc001")
    url = f"{endpoint.rstrip('/')}/{quote(artifact['bucket'], safe='')}/{quote(key)}"
    data = path.read_bytes()
    request = AWSRequest(method="PUT", url=url, data=data, headers={
        "Content-Type": content_type,
        "Cache-Control": artifact["cache_control"],
        "x-amz-meta-sha256": digest,
        "If-None-Match": "*",
    })
    S3SigV4Auth(credentials, "s3", region).add_auth(request)
    signed = Request(url, data=data, method="PUT", headers=dict(request.headers.items()))
    try:
        with urlopen(signed) as response:
            response.read()
    except HTTPError as error:
        if error.code == 412:
            die(f"immutable artifact {artifact['name']}: concurrent object creation: {key}; "
                "rerun to verify identical content")
        die(f"immutable artifact {artifact['name']}: upload failed for {key}: {error}")


def _immutable_sync(source, artifact, endpoint, excludes):
    """Publish write-once objects; identical retries are no-ops."""
    aws_env = _aws_env(artifact)
    bucket = artifact["bucket"]
    for path in sorted(item for item in source.rglob("*") if item.is_file()):
        relative = path.relative_to(source).as_posix()
        if _excluded(relative, excludes):
            continue
        key = "/".join(part for part in (artifact.get("key_prefix"), relative) if part)
        digest = hashlib.sha256(path.read_bytes()).hexdigest()
        head = subprocess.run(
            _aws(endpoint, ["s3api", "head-object", "--bucket", bucket, "--key", key]),
            env=aws_env, check=False, text=True, capture_output=True)
        if head.returncode == 0:
            metadata = json.loads(head.stdout)
            if (metadata.get("Metadata") or {}).get("sha256") != digest:
                die(f"immutable artifact {artifact['name']}: object changed: {key}")
            if metadata.get("CacheControl") != artifact["cache_control"]:
                die(f"immutable artifact {artifact['name']}: cache metadata changed: {key}")
            print(f"  Immutable object unchanged: s3://{bucket}/{key}")
            continue
        missing = head.stderr.lower()
        if not any(marker in missing for marker in ("404", "not found", "nosuchkey")):
            die(f"immutable artifact {artifact['name']}: head-object failed for {key}: "
                f"{head.stderr.strip()}")
        content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
        _conditional_put(endpoint, artifact, key, path, content_type, digest)


def _replaceable_sync(source, artifact, endpoint, excludes):
    aws_env = _aws_env(artifact)
    prefix = artifact.get("key_prefix")
    destination = f"s3://{artifact['bucket']}/{prefix + '/' if prefix else ''}"
    exclude_args = [part for pattern in excludes for part in ("--exclude", pattern)]
    common = ["--only-show-errors", "--cache-control", artifact["cache_control"],
              *exclude_args]
    run(_aws(endpoint, ["s3", "sync", f"{source}/", destination,
                        "--delete", *common]), env=aws_env)
    run(_aws(endpoint, ["s3", "cp", f"{source}/", destination,
                        "--recursive", *common]), env=aws_env)


def publish_artifact(build_root, artifact, excludes=None):
    endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
    source = (build_root / artifact["source"]).resolve()
    if not source.is_relative_to(build_root.resolve()) or not source.is_dir():
        die(f"artifact {artifact['name']}: source directory not found: {source}")
    excludes = excludes or []
    print(f"Publishing {artifact['name']} from {source} to s3://{artifact['bucket']}")
    if artifact["immutable"]:
        _immutable_sync(source, artifact, endpoint, excludes)
    else:
        _replaceable_sync(source, artifact, endpoint, excludes)


def garage_admin(method, path, token, body=None):
    url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
    data = json.dumps(body).encode() if body is not None else None
    headers = {"Authorization": f"Bearer {token}"}
    if data is not None:
        headers["Content-Type"] = "application/json"
    request = Request(url, data=data, method=method, headers=headers)
    with urlopen(request) as response:
        raw = response.read()
        return json.loads(raw) if raw else {}


def ensure_bucket_aliases(site_name, aliases, admin_token):
    """Fail closed while reconciling legacy Garage global aliases."""
    if not aliases:
        return
    if not admin_token:
        die("GARAGE_ADMIN_TOKEN is required when aliases are configured")
    try:
        info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={quote(site_name)}",
                            admin_token)
    except (HTTPError, URLError) as error:
        die(f"bucket lookup failed: {error}")
    bucket_id = info.get("id")
    if not bucket_id:
        die(f"bucket lookup returned no id for {site_name}")
    existing = set(info.get("globalAliases") or [])
    for alias in aliases:
        if alias not in existing:
            garage_admin("POST", "/v2/AddBucketAlias", admin_token,
                         {"bucketId": bucket_id, "globalAlias": alias})


def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
    if manifests_dir.exists():
        shutil.rmtree(manifests_dir)
    routes = []
    for route in cfg["routes"]:
        artifact = cfg["artifacts"][route["artifact"]]
        routes.append({
            **route,
            "artifact_config": artifact,
            "resource_name": k8s_name(site_name, route["artifact"]),
            "backend_host": (GARAGE_WEBSITE_HOST if cfg["compatibility"] else
                             f"{artifact['bucket']}.web.sjc001.fritzlab.net"),
            "pass_host_header": cfg["compatibility"],
        })
    render_templates(action_dir, {
        "site": site_name,
        "site_k8s": k8s_name(site_name),
        "domain": cfg["domain"],
        "aliases": cfg["aliases"],
        "namespace": NAMESPACE,
        "website_host": GARAGE_WEBSITE_HOST,
        "routes": routes,
    }, app_dir, manifests_dir)


def deploy_static(site_name, site_dir, action_dir, token, cfg):
    build_root = (site_dir / "build" / "html").resolve()
    if not build_root.is_dir():
        die(f"build/html not found — did the build step run? ({build_root})")
    for artifact in cfg["artifacts"].values():
        publish_artifact(build_root, artifact, cfg.get("excludes"))
    if cfg["compatibility"]:
        ensure_bucket_aliases(site_name, cfg["aliases"],
                              os.environ.get("GARAGE_ADMIN_TOKEN"))

    apps_dir = clone_apps(token)
    app_dir = apps_dir / "sjc001" / "websites" / site_name
    render_site_manifests(site_name, action_dir, app_dir, app_dir / "manifests", cfg)
    commit_and_push(apps_dir, f"Deploy {site_name}", token)


def decommission(site_name, token):
    apps_dir = clone_apps(token)
    site_path = apps_dir / "sjc001" / "websites" / site_name
    if not site_path.exists():
        print(f"No manifests for {site_name} — nothing to remove")
        return
    shutil.rmtree(site_path)
    commit_and_push(apps_dir, f"Decommission {site_name}", token)
    print(f"Bucket {site_name} and its objects are NOT purged automatically.")


def cmd_deploy():
    site_repo = env("SITE_REPO")
    site_dir = Path(env("SITE_DIR"))
    action_dir = Path(env("ACTION_DIR"))
    token = env("CI_BOT_TOKEN")
    site_name = site_repo.split("/", 1)[1]
    cfg = parse_site_yaml(site_dir, site_name)
    if not cfg["enabled"]:
        print("Site disabled — running decommission...")
        decommission(site_name, token)
        return
    deploy_static(site_name, site_dir, action_dir, token, cfg)