"""Deploy phase — S3 sync, manifest rendering, alias reconcile.""" import json import os import shutil import tempfile from pathlib import Path from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen from utils import ( NAMESPACE, clone_apps, commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run, ) GARAGE_ADMIN_ENDPOINT = os.environ.get( "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903" ) def validate_artifact_output(site_dir, artifact): """Prove every artifact and declared cache prefix exists before publishing any.""" html_dir = site_dir / artifact["build_dir"] if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")): die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}") for rule in artifact["cache_rules"]: if not rule["path"]: continue cache_root = html_dir / rule["path"] if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")): die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files") def validate_publication_environment(cfg): """Resolve every declared credential before the first bucket is changed.""" for artifact in cfg["artifacts"]: env(artifact["credentials"]["access_key_env"]) env(artifact["credentials"]["secret_key_env"]) if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"): die("GARAGE_ADMIN_TOKEN is required when aliases are declared") def s3_sync(artifact, route, site_dir, credential_env_names=None): endpoint = artifact["s3_endpoint"] html_dir = site_dir / artifact["build_dir"] access_key = env(artifact["credentials"]["access_key_env"]) secret_key = env(artifact["credentials"]["secret_key_env"]) aws_env = os.environ.copy() for name in credential_env_names or artifact["credentials"].values(): aws_env.pop(name, None) for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE", "AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"): aws_env.pop(name, None) aws_env.update({ "AWS_ACCESS_KEY_ID": access_key, "AWS_SECRET_ACCESS_KEY": secret_key, "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"), }) bucket = artifact["bucket"] object_prefix = route["path"].strip("/") bucket_destination = f"s3://{bucket}/" destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}" default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"]) # `excludes` are patterns (site.yaml `excludes:` list) that should never # be uploaded *and* should never be deleted from the bucket — escape hatch # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli). exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)] if artifact["excludes"]: print(f"Excluding patterns: {artifact['excludes']}") print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}") # `sync --delete` handles new/changed/orphaned files. Partitioned # `cp --recursive` calls then re-upload each file once to refresh metadata # (cache-control, content-type) on objects sync skipped as unchanged. # A no-op deploy therefore transfers the artifact bytes once. # AWS CLI guesses Content-Type from file extension on local→S3 uploads, # so a fresh upload always carries the right MIME type. stage = None sync_source = html_dir sync_excludes = exclude_args if object_prefix: stage = tempfile.TemporaryDirectory() sync_source = Path(stage.name) staged_artifact = sync_source / object_prefix staged_artifact.parent.mkdir(parents=True, exist_ok=True) staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True) sync_excludes = [arg for pattern in artifact["excludes"] for arg in ("--exclude", f"{object_prefix}/{pattern}")] try: # Sync the complete bucket authority so moving a route prefix also # deletes objects under its old prefix instead of leaving them public. run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{sync_source}/", bucket_destination, "--delete", "--only-show-errors", "--cache-control", default_cache, *sync_excludes], env=aws_env) finally: if stage: stage.cleanup() print("Re-stamping metadata on all objects...") specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]] default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")] run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination, "--recursive", "--only-show-errors", "--cache-control", default_cache, *default_filters, *exclude_args], env=aws_env) for rule in artifact["cache_rules"]: if not rule["path"]: continue include = f"{rule['path'].rstrip('/')}/*" child_filters = [arg for path in specific_paths if path.startswith(f"{rule['path'].rstrip('/')}/") for arg in ("--exclude", f"{path}/*")] # Apply rules from the artifact root so artifact-level exclusions keep # their original meaning under every cache override. run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination, "--recursive", "--only-show-errors", "--cache-control", rule["cache_control"], "--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env) def garage_admin(method, path, token, body=None): url = f"{GARAGE_ADMIN_ENDPOINT}{path}" data = json.dumps(body).encode() if body is not None else None headers = {"Authorization": f"Bearer {token}"} if data is not None: headers["Content-Type"] = "application/json" req = Request(url, data=data, method=method, headers=headers) with urlopen(req) as resp: raw = resp.read() return json.loads(raw) if raw else {} def ensure_bucket_aliases(site_name, aliases, admin_token): """Add cfg['aliases'] as Garage globalAliases on the site bucket. Idempotent: skips aliases already present. Never removes aliases not in the desired set (safety — orphan removal is manual). """ if not aliases: return if not admin_token: die("GARAGE_ADMIN_TOKEN is required when aliases are declared") try: info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}", admin_token) except (HTTPError, URLError) as e: raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e bucket_id = info.get("id") existing = set(info.get("globalAliases") or []) print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}") for alias in aliases: if alias in existing: continue print(f" Adding globalAlias: {alias}") try: garage_admin("POST", "/v2/AddBucketAlias", admin_token, {"bucketId": bucket_id, "globalAlias": alias}) except HTTPError as e: body = e.read().decode(errors="replace") if hasattr(e, "read") else "" print(f" ERROR adding alias {alias}: {e} {body}") raise def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg): """Always re-render manifests from current site.yaml. Templates own domain + aliases, so changes propagate without manual edits.""" manifests_dir.mkdir(parents=True, exist_ok=True) artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} routes = [] for route in cfg["routes"]: artifact = artifact_by_name[route["artifact"]] resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}" routes.append({**route, "resource_name": resource_name, "artifact_config": artifact}) template_vars = { "site": site_name, "site_k8s": k8s_name(site_name), "domain": cfg["domain"], "aliases": cfg["aliases"], "namespace": NAMESPACE, "compatibility": cfg["compatibility"], "routes": routes, } render_templates(action_dir, template_vars, app_dir, manifests_dir) def deploy_static(site_name, site_dir, action_dir, token, cfg): artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} credential_env_names = { name for artifact in cfg["artifacts"] for name in artifact["credentials"].values() } validate_publication_environment(cfg) for artifact in cfg["artifacts"]: validate_artifact_output(site_dir, artifact) for route in cfg["routes"]: s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names) if cfg["compatibility"]: ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")) apps_dir = clone_apps(token) app_dir = apps_dir / "sjc001" / "websites" / site_name manifests_dir = app_dir / "manifests" render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg) commit_and_push(apps_dir, f"Deploy {site_name}") def decommission(site_name, token, buckets=None): """Remove manifests from apps repo.""" apps_dir = clone_apps(token) site_path = apps_dir / "sjc001" / "websites" / site_name if not site_path.exists(): print(f"No manifests for {site_name} — nothing to remove") return shutil.rmtree(site_path) commit_and_push(apps_dir, f"Decommission {site_name}") for bucket in buckets or [site_name]: print(f"Bucket {bucket} and its objects are NOT purged automatically.") print(f" garage bucket delete {bucket} --yes") def cmd_deploy(): site_repo = env("SITE_REPO") site_dir = Path(env("SITE_DIR")) action_dir = Path(env("ACTION_DIR")) token = env("CI_BOT_TOKEN") site_name = site_repo.split("/", 1)[1] cfg = parse_site_yaml(site_dir) if not cfg["enabled"]: print("Site disabled — running decommission...") decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]]) return deploy_static(site_name, site_dir, action_dir, token, cfg)