"""Deploy phase — S3 sync, manifest rendering, alias reconcile.""" import json import os import shutil import tempfile from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urlsplit from urllib.request import Request, urlopen from utils import ( DEFAULT_S3_ENDPOINT, NAMESPACE, clone_apps, commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run_args, ) GARAGE_ADMIN_ENDPOINT = os.environ.get( "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903" ) CACHE_CONTROL = "public, max-age=0, must-revalidate" def credential_environment(profile): if profile == "default": access_key = env("AWS_ACCESS_KEY_ID") secret_key = env("AWS_SECRET_ACCESS_KEY") else: prefix = f"SITE_PUBLISH_{profile.upper().replace('-', '_')}" access_key = env(f"{prefix}_S3_ACCESS_KEY_ID") secret_key = env(f"{prefix}_S3_SECRET_ACCESS_KEY") child_env = os.environ.copy() child_env["AWS_ACCESS_KEY_ID"] = access_key child_env["AWS_SECRET_ACCESS_KEY"] = secret_key child_env.setdefault("AWS_DEFAULT_REGION", "sjc001") return child_env def configure_cors(bucket, origins, endpoint, child_env): if origins is None: return if not origins: run_args( ["aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors", "--bucket", bucket], env=child_env, ) return config = { "CORSRules": [{ "AllowedOrigins": origins, "AllowedMethods": ["GET", "HEAD"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag"], "MaxAgeSeconds": 3600, }] } with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle: json.dump(config, handle) handle.flush() run_args( ["aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors", "--bucket", bucket, "--cors-configuration", f"file://{handle.name}"], env=child_env, ) def s3_sync(bucket, source, *, credential="default", endpoint=None, cache=None, cors_origins=None, excludes=None): endpoint = endpoint or os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT) if not source.exists(): die(f"staged artifact not found — did the build step run? ({source})") child_env = credential_environment(credential) cache = cache or {"default": CACHE_CONTROL, "rules": []} # `excludes` are patterns (site.yaml `excludes:` list) that should never # be uploaded *and* should never be deleted from the bucket — escape hatch # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli). exclude_args = [] for pattern in excludes or []: exclude_args.extend(["--exclude", pattern]) if excludes: print(f"Excluding patterns: {excludes}") print(f"Syncing {source} → s3://{bucket} via {endpoint}") # `sync --delete` handles new/changed/orphaned files. `cp --recursive` # then re-uploads everything to refresh metadata (cache-control, # content-type) on objects sync skipped because nothing changed. # Cost: a no-op deploy still re-uploads every byte. Sites here are # small enough that that's free; correctness wins over throughput. # AWS CLI guesses Content-Type from file extension on local→S3 uploads, # so a fresh upload always carries the right MIME type. run_args( ["aws", "--endpoint-url", endpoint, "s3", "sync", f"{source}/", f"s3://{bucket}/", "--delete", "--only-show-errors", "--cache-control", cache["default"], *exclude_args], env=child_env, ) print("Re-stamping metadata on all objects...") run_args( ["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/", "--recursive", "--only-show-errors", "--cache-control", cache["default"], *exclude_args], env=child_env, ) for rule in cache["rules"]: print(f"Applying cache metadata for {rule['match']}") run_args( ["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/", "--recursive", "--only-show-errors", "--exclude", "*", "--include", rule["match"], "--cache-control", rule["value"], *exclude_args], env=child_env, ) configure_cors(bucket, cors_origins, endpoint, child_env) def preflight_artifacts(site_dir, cfg): """Validate every source, credential, and bucket before the first write.""" for artifact in cfg["artifacts"].values(): source = site_dir / ".site-publish" / artifact["name"] if not source.is_dir() or not any(entry.is_file() for entry in source.rglob("*")): die(f"staged artifact is absent or empty: {source}") child_env = credential_environment(artifact["credential"]) endpoint = artifact["endpoint"] or os.environ.get( "GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT ) run_args( ["aws", "--endpoint-url", endpoint, "s3api", "head-bucket", "--bucket", artifact["bucket"]], env=child_env, ) def routed_cache(cache, routes): """Translate source-relative cache patterns to their public object keys.""" rules = [] for route in routes: prefix = route["path"].lstrip("/") for rule in cache["rules"]: pattern = f"{prefix}/{rule['match']}" if prefix else rule["match"] rules.append({"match": pattern, "value": rule["value"]}) return {"default": cache["default"], "rules": rules} def garage_admin(method, path, token, body=None): parsed = urlsplit(GARAGE_ADMIN_ENDPOINT) if ( parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment ): die("GARAGE_ADMIN_ENDPOINT must be an HTTP URL without credentials") url = f"{GARAGE_ADMIN_ENDPOINT}{path}" data = json.dumps(body).encode() if body is not None else None headers = {"Authorization": f"Bearer {token}"} if data is not None: headers["Content-Type"] = "application/json" req = Request(url, data=data, method=method, headers=headers) with urlopen(req) as resp: raw = resp.read() return json.loads(raw) if raw else {} def ensure_bucket_aliases(site_name, aliases, admin_token): """Add cfg['aliases'] as Garage globalAliases on the site bucket. Idempotent: skips aliases already present. Never removes aliases not in the desired set (safety — orphan removal is manual). """ if not aliases: return if not admin_token: die("GARAGE_ADMIN_TOKEN is required when aliases are configured") try: info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}", admin_token) except (HTTPError, URLError) as e: raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e bucket_id = info.get("id") if not isinstance(bucket_id, str) or not bucket_id: die(f"bucket lookup for {site_name} returned no bucket id") existing = set(info.get("globalAliases") or []) print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}") for alias in aliases: if alias in existing: continue print(f" Adding globalAlias: {alias}") try: garage_admin("POST", "/v2/AddBucketAlias", admin_token, {"bucketId": bucket_id, "globalAlias": alias}) except HTTPError as e: body = e.read().decode(errors="replace") if hasattr(e, "read") else "" print(f" ERROR adding alias {alias}: {e} {body}") raise def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg): """Always re-render manifests from current site.yaml. Templates own domain + aliases, so changes propagate without manual edits.""" if manifests_dir.exists(): shutil.rmtree(manifests_dir) manifests_dir.mkdir(parents=True) site_k8s = k8s_name(site_name) if cfg["mode"] == "legacy": artifacts = [{ "name": "site", "bucket": site_name, "service_name": site_k8s, "external_name": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net", "virtual_host": False, }] routes = [{ "name": "site", "path": "/", "artifact": "site", "service_name": site_k8s, "ingress_name": site_k8s, "middlewares": cfg["middlewares"], "middleware_refs": [f"{name}@file" for name in cfg["middlewares"]], }] else: artifacts = [] artifact_by_name = {} for name, artifact in cfg["artifacts"].items(): service_name = k8s_name(f"{site_name}-{name}") view = { **artifact, "service_name": service_name, "external_name": f"{artifact['bucket']}.web.sjc001.fritzlab.net", "virtual_host": True, } artifacts.append(view) artifact_by_name[name] = view routes = [] for route in cfg["routes"]: artifact = artifact_by_name[route["artifact"]] ingress_name = k8s_name(f"{site_name}-{route['name']}") middleware_refs = [f"{name}@file" for name in route["middlewares"]] routes.append({ **route, "service_name": artifact["service_name"], "ingress_name": ingress_name, "middleware_refs": middleware_refs, }) template_vars = { "site": site_name, "site_k8s": site_k8s, "domain": cfg["domain"], "aliases": cfg["aliases"], "namespace": NAMESPACE, "artifacts": artifacts, "routes": routes, "modern": cfg["mode"] == "multi", } render_templates(action_dir, template_vars, app_dir, manifests_dir) def deploy_static(site_name, site_dir, action_dir, token, cfg): if cfg["mode"] == "legacy": s3_sync( site_name, site_dir / "build" / "html", excludes=cfg.get("excludes"), ) ensure_bucket_aliases( site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN") ) else: preflight_artifacts(site_dir, cfg) for artifact in cfg["artifacts"].values(): routes = [ route for route in cfg["routes"] if route["artifact"] == artifact["name"] ] s3_sync( artifact["bucket"], site_dir / ".site-publish" / artifact["name"], credential=artifact["credential"], endpoint=artifact["endpoint"], cache=routed_cache(artifact["cache"], routes), cors_origins=artifact["cors_origins"], excludes=artifact["excludes"], ) apps_dir = clone_apps(token) app_dir = apps_dir / "sjc001" / "websites" / site_name manifests_dir = app_dir / "manifests" render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg) try: commit_and_push(apps_dir, f"Deploy {site_name}", token) finally: shutil.rmtree(apps_dir.parent, ignore_errors=True) def decommission(site_name, token): """Remove manifests from apps repo.""" apps_dir = clone_apps(token) try: site_path = apps_dir / "sjc001" / "websites" / site_name if not site_path.exists(): print(f"No manifests for {site_name} — nothing to remove") return shutil.rmtree(site_path) commit_and_push(apps_dir, f"Decommission {site_name}", token) finally: shutil.rmtree(apps_dir.parent, ignore_errors=True) print(f"Bucket {site_name} and its objects are NOT purged automatically.") print(f" garage bucket delete {site_name} --yes") def cmd_deploy(): site_repo = env("SITE_REPO") site_dir = Path(env("SITE_DIR")) action_dir = Path(env("ACTION_DIR")) token = env("CI_BOT_TOKEN") site_name = site_repo.split("/", 1)[1] cfg = parse_site_yaml(site_dir) if not cfg["enabled"]: print("Site disabled — running decommission...") decommission(site_name, token) return deploy_static(site_name, site_dir, action_dir, token, cfg)