"""Deploy phase — artifact publication and split-route manifest rendering.""" import fnmatch import hashlib import json import mimetypes import os import shutil import subprocess from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import quote from urllib.request import Request, urlopen try: from botocore.auth import S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials except ModuleNotFoundError: # aws-cli v1's bundled botocore layout from awscli.botocore.auth import S3SigV4Auth from awscli.botocore.awsrequest import AWSRequest from awscli.botocore.credentials import Credentials from utils import ( DEFAULT_S3_ENDPOINT, GARAGE_WEBSITE_HOST, NAMESPACE, clone_apps, commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run, ) GARAGE_ADMIN_ENDPOINT = os.environ.get( "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903") def _aws_env(artifact): credentials = artifact["credentials"] access = env(credentials["access_key_env"]) secret = env(credentials["secret_key_env"]) child = os.environ.copy() child.update({"AWS_ACCESS_KEY_ID": access, "AWS_SECRET_ACCESS_KEY": secret, "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001")}) return child def _aws(endpoint, operation): return ["aws", "--endpoint-url", endpoint, *operation] def _excluded(relative, patterns): return any(fnmatch.fnmatch(relative, pattern) for pattern in patterns) def _conditional_put(endpoint, artifact, key, path, content_type, digest): """Atomically create one S3 object without exposing signing credentials.""" selectors = artifact["credentials"] credentials = Credentials(env(selectors["access_key_env"]), env(selectors["secret_key_env"])) region = os.environ.get("AWS_DEFAULT_REGION", "sjc001") url = f"{endpoint.rstrip('/')}/{quote(artifact['bucket'], safe='')}/{quote(key)}" data = path.read_bytes() request = AWSRequest(method="PUT", url=url, data=data, headers={ "Content-Type": content_type, "Cache-Control": artifact["cache_control"], "x-amz-meta-sha256": digest, "If-None-Match": "*", }) S3SigV4Auth(credentials, "s3", region).add_auth(request) signed = Request(url, data=data, method="PUT", headers=dict(request.headers.items())) try: with urlopen(signed) as response: response.read() except HTTPError as error: if error.code == 412: die(f"immutable artifact {artifact['name']}: concurrent object creation: {key}; " "rerun to verify identical content") die(f"immutable artifact {artifact['name']}: upload failed for {key}: {error}") def _immutable_sync(source, artifact, endpoint, excludes): """Publish write-once objects; identical retries are no-ops.""" aws_env = _aws_env(artifact) bucket = artifact["bucket"] for path in sorted(item for item in source.rglob("*") if item.is_file()): relative = path.relative_to(source).as_posix() if _excluded(relative, excludes): continue key = "/".join(part for part in (artifact.get("key_prefix"), relative) if part) digest = hashlib.sha256(path.read_bytes()).hexdigest() head = subprocess.run( _aws(endpoint, ["s3api", "head-object", "--bucket", bucket, "--key", key]), env=aws_env, check=False, text=True, capture_output=True) if head.returncode == 0: metadata = json.loads(head.stdout) if (metadata.get("Metadata") or {}).get("sha256") != digest: die(f"immutable artifact {artifact['name']}: object changed: {key}") if metadata.get("CacheControl") != artifact["cache_control"]: die(f"immutable artifact {artifact['name']}: cache metadata changed: {key}") print(f" Immutable object unchanged: s3://{bucket}/{key}") continue missing = head.stderr.lower() if not any(marker in missing for marker in ("404", "not found", "nosuchkey")): die(f"immutable artifact {artifact['name']}: head-object failed for {key}: " f"{head.stderr.strip()}") content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" _conditional_put(endpoint, artifact, key, path, content_type, digest) def _replaceable_sync(source, artifact, endpoint, excludes): aws_env = _aws_env(artifact) prefix = artifact.get("key_prefix") destination = f"s3://{artifact['bucket']}/{prefix + '/' if prefix else ''}" exclude_args = [part for pattern in excludes for part in ("--exclude", pattern)] common = ["--only-show-errors", "--cache-control", artifact["cache_control"], *exclude_args] run(_aws(endpoint, ["s3", "sync", f"{source}/", destination, "--delete", *common]), env=aws_env) run(_aws(endpoint, ["s3", "cp", f"{source}/", destination, "--recursive", *common]), env=aws_env) def publish_artifact(build_root, artifact, excludes=None): endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT) source = (build_root / artifact["source"]).resolve() if not source.is_relative_to(build_root.resolve()) or not source.is_dir(): die(f"artifact {artifact['name']}: source directory not found: {source}") excludes = excludes or [] print(f"Publishing {artifact['name']} from {source} to s3://{artifact['bucket']}") if artifact["immutable"]: _immutable_sync(source, artifact, endpoint, excludes) else: _replaceable_sync(source, artifact, endpoint, excludes) def garage_admin(method, path, token, body=None): url = f"{GARAGE_ADMIN_ENDPOINT}{path}" data = json.dumps(body).encode() if body is not None else None headers = {"Authorization": f"Bearer {token}"} if data is not None: headers["Content-Type"] = "application/json" request = Request(url, data=data, method=method, headers=headers) with urlopen(request) as response: raw = response.read() return json.loads(raw) if raw else {} def ensure_bucket_aliases(site_name, aliases, admin_token): """Fail closed while reconciling legacy Garage global aliases.""" if not aliases: return if not admin_token: die("GARAGE_ADMIN_TOKEN is required when aliases are configured") try: info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={quote(site_name)}", admin_token) except (HTTPError, URLError) as error: die(f"bucket lookup failed: {error}") bucket_id = info.get("id") if not bucket_id: die(f"bucket lookup returned no id for {site_name}") existing = set(info.get("globalAliases") or []) for alias in aliases: if alias not in existing: garage_admin("POST", "/v2/AddBucketAlias", admin_token, {"bucketId": bucket_id, "globalAlias": alias}) def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg): if manifests_dir.exists(): shutil.rmtree(manifests_dir) routes = [] for route in cfg["routes"]: artifact = cfg["artifacts"][route["artifact"]] routes.append({ **route, "artifact_config": artifact, "resource_name": k8s_name(site_name, route["artifact"]), "backend_host": (GARAGE_WEBSITE_HOST if cfg["compatibility"] else f"{artifact['bucket']}.web.sjc001.fritzlab.net"), "pass_host_header": cfg["compatibility"], }) render_templates(action_dir, { "site": site_name, "site_k8s": k8s_name(site_name), "domain": cfg["domain"], "aliases": cfg["aliases"], "namespace": NAMESPACE, "website_host": GARAGE_WEBSITE_HOST, "routes": routes, }, app_dir, manifests_dir) def deploy_static(site_name, site_dir, action_dir, token, cfg): build_root = (site_dir / "build" / "html").resolve() if not build_root.is_dir(): die(f"build/html not found — did the build step run? ({build_root})") for artifact in cfg["artifacts"].values(): publish_artifact(build_root, artifact, cfg.get("excludes")) if cfg["compatibility"]: ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")) apps_dir = clone_apps(token) app_dir = apps_dir / "sjc001" / "websites" / site_name render_site_manifests(site_name, action_dir, app_dir, app_dir / "manifests", cfg) commit_and_push(apps_dir, f"Deploy {site_name}", token) def decommission(site_name, token): apps_dir = clone_apps(token) site_path = apps_dir / "sjc001" / "websites" / site_name if not site_path.exists(): print(f"No manifests for {site_name} — nothing to remove") return shutil.rmtree(site_path) commit_and_push(apps_dir, f"Decommission {site_name}", token) print(f"Bucket {site_name} and its objects are NOT purged automatically.") def cmd_deploy(): site_repo = env("SITE_REPO") site_dir = Path(env("SITE_DIR")) action_dir = Path(env("ACTION_DIR")) token = env("CI_BOT_TOKEN") site_name = site_repo.split("/", 1)[1] cfg = parse_site_yaml(site_dir, site_name) if not cfg["enabled"]: print("Site disabled — running decommission...") decommission(site_name, token) return deploy_static(site_name, site_dir, action_dir, token, cfg)