"""Deploy phase — S3 sync, manifest rendering, alias reconcile.""" import fnmatch import hashlib import json import mimetypes import os import re import shutil import subprocess import tempfile from pathlib import Path from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen import yaml from utils import ( NAMESPACE, clone_apps, commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run, selected_artifacts, selected_routes, validate_artifact_inputs, ) GARAGE_ADMIN_ENDPOINT = os.environ.get( "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903" ) def validate_artifact_output(site_dir, artifact): """Prove every artifact and declared cache prefix exists before publishing any.""" html_dir = site_dir / artifact["build_dir"] if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")): die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}") for rule in artifact["cache_rules"]: if not rule["path"]: continue cache_root = html_dir / rule["path"] if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")): die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files") def validate_publication_environment(cfg): """Resolve every credential this run needs before the first bucket is changed.""" for artifact in selected_artifacts(cfg): env(artifact["credentials"]["access_key_env"]) env(artifact["credentials"]["secret_key_env"]) if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"): die("GARAGE_ADMIN_TOKEN is required when aliases are declared") def _is_immutable(rule): return "immutable" in { part.strip().lower().split("=", 1)[0] for part in rule["cache_control"].split(",") } def _aws_capture(args, aws_env): """Run a non-streaming AWS request without exposing environment credentials.""" print(f" $ {' '.join(str(part) for part in args)}") return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False) def _immutable_head(endpoint, bucket, key, aws_env): args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object", "--bucket", bucket, "--key", key, "--output", "json"] result = _aws_capture(args, aws_env) if result.returncode == 0: return json.loads(result.stdout) error = f"{result.stdout}\n{result.stderr}" if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")): return None raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}") def _immutable_digests(source, cache_control, content_type): with source.open("rb") as stream: content_digest = hashlib.file_digest(stream, "sha256").hexdigest() publication = hashlib.sha256() publication.update(cache_control.encode()) publication.update(b"\0") publication.update(content_type.encode()) publication.update(b"\0") with source.open("rb") as stream: for block in iter(lambda: stream.read(1024 * 1024), b""): publication.update(block) return content_digest, publication.hexdigest() def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type): metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()} return ( metadata.get("sha256") == content_digest and metadata.get("publication-sha256") == publication_digest and info.get("CacheControl") == cache_control and info.get("ContentType") == content_type ) def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env): """Publish a content-addressed key; identical retries converge.""" content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream" content_digest, publication_digest = _immutable_digests(source, cache_control, content_type) address_digests = re.findall(r"(? {current}); publish it in the same run" ) def deploy_static(site_name, site_dir, action_dir, token, cfg): artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} credential_env_names = { name for artifact in cfg["artifacts"] for name in artifact["credentials"].values() } # Publication is scoped to the selected artifacts; the rendered route # contract is not. Manifests and immutable-path history stay whole, so a # partial publish can never retire another artifact's route or bucket. publishing = selected_routes(cfg) validate_publication_environment(cfg) for artifact in selected_artifacts(cfg): validate_artifact_output(site_dir, artifact) apps_dir = clone_apps(token) app_dir = apps_dir / "sjc001" / "websites" / site_name manifests_dir = app_dir / "manifests" previous_contracts = previous_route_contracts(app_dir) validate_route_migrations(cfg, previous_contracts) validate_scoped_history(cfg, previous_contracts) # Complete immutable work across the whole publication before any route's # mutable pointers can change. Partial immutable success is safe; mixing a # new route with an old route after a later immutable failure is not. for route in publishing: publish_route_immutables( artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, ) # Reconcile every browser-read policy before publishing mutable content. # A CORS failure therefore cannot leave a new channel pointing at a release # whose cross-origin assets browsers cannot consume. reconcile_artifact_cors(selected_artifacts(cfg), credential_env_names) for route in publishing: s3_sync( artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]), ) if cfg["compatibility"]: ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")) render_site_manifests( site_name, action_dir, app_dir, manifests_dir, cfg, previous_contracts, ) commit_and_push(apps_dir, f"Deploy {site_name}", token) def decommission(site_name, token, buckets=None): """Remove manifests from apps repo.""" apps_dir = clone_apps(token) site_path = apps_dir / "sjc001" / "websites" / site_name if not site_path.exists(): print(f"No manifests for {site_name} — nothing to remove") return shutil.rmtree(site_path) commit_and_push(apps_dir, f"Decommission {site_name}", token) for bucket in buckets or [site_name]: print(f"Bucket {bucket} and its objects are NOT purged automatically.") print(f" garage bucket delete {bucket} --yes") def cmd_deploy(): site_repo = env("SITE_REPO") site_dir = Path(env("SITE_DIR")) action_dir = Path(env("ACTION_DIR")) token = env("CI_BOT_TOKEN") site_name = site_repo.split("/", 1)[1] cfg = parse_site_yaml(site_dir) if not cfg["enabled"]: if len(cfg["selected"]) != len(cfg["artifacts"]): die("a disabled site decommissions whole; drop the artifacts selection") print("Site disabled — running decommission...") decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]]) return validate_artifact_inputs(site_dir, cfg) deploy_static(site_name, site_dir, action_dir, token, cfg)