Evelyn Chen
59541fdf53
Test / contract (pull_request) Successful in 6s
Authored-By: OpenAI (GPT-5) <noreply@openai.com>
243 lines
10 KiB
Python
243 lines
10 KiB
Python
"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request, urlopen
|
|
|
|
from utils import (
|
|
NAMESPACE,
|
|
clone_apps,
|
|
commit_and_push,
|
|
die,
|
|
env,
|
|
k8s_name,
|
|
parse_site_yaml,
|
|
render_templates,
|
|
run,
|
|
)
|
|
|
|
GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
|
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
|
|
)
|
|
|
|
|
|
def validate_artifact_output(site_dir, artifact):
|
|
"""Prove every artifact and declared cache prefix exists before publishing any."""
|
|
html_dir = site_dir / artifact["build_dir"]
|
|
if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")):
|
|
die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}")
|
|
for rule in artifact["cache_rules"]:
|
|
if not rule["path"]:
|
|
continue
|
|
cache_root = html_dir / rule["path"]
|
|
if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")):
|
|
die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files")
|
|
|
|
|
|
def validate_publication_environment(cfg):
|
|
"""Resolve every declared credential before the first bucket is changed."""
|
|
for artifact in cfg["artifacts"]:
|
|
env(artifact["credentials"]["access_key_env"])
|
|
env(artifact["credentials"]["secret_key_env"])
|
|
if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
|
|
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
|
|
|
|
|
|
def s3_sync(artifact, route, site_dir, credential_env_names=None):
|
|
endpoint = artifact["s3_endpoint"]
|
|
html_dir = site_dir / artifact["build_dir"]
|
|
access_key = env(artifact["credentials"]["access_key_env"])
|
|
secret_key = env(artifact["credentials"]["secret_key_env"])
|
|
aws_env = os.environ.copy()
|
|
for name in credential_env_names or artifact["credentials"].values():
|
|
aws_env.pop(name, None)
|
|
for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE",
|
|
"AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"):
|
|
aws_env.pop(name, None)
|
|
aws_env.update({
|
|
"AWS_ACCESS_KEY_ID": access_key,
|
|
"AWS_SECRET_ACCESS_KEY": secret_key,
|
|
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
|
|
})
|
|
bucket = artifact["bucket"]
|
|
object_prefix = route["path"].strip("/")
|
|
bucket_destination = f"s3://{bucket}/"
|
|
destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}"
|
|
default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
|
|
# `excludes` are patterns (site.yaml `excludes:` list) that should never
|
|
# be uploaded *and* should never be deleted from the bucket — escape hatch
|
|
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
|
|
exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)]
|
|
if artifact["excludes"]:
|
|
print(f"Excluding patterns: {artifact['excludes']}")
|
|
print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
|
|
# `sync --delete` handles new/changed/orphaned files. Partitioned
|
|
# `cp --recursive` calls then re-upload each file once to refresh metadata
|
|
# (cache-control, content-type) on objects sync skipped as unchanged.
|
|
# A no-op deploy therefore transfers the artifact bytes once.
|
|
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
|
|
# so a fresh upload always carries the right MIME type.
|
|
stage = None
|
|
sync_source = html_dir
|
|
sync_excludes = exclude_args
|
|
if object_prefix:
|
|
stage = tempfile.TemporaryDirectory()
|
|
sync_source = Path(stage.name)
|
|
staged_artifact = sync_source / object_prefix
|
|
staged_artifact.parent.mkdir(parents=True, exist_ok=True)
|
|
staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True)
|
|
sync_excludes = [arg for pattern in artifact["excludes"]
|
|
for arg in ("--exclude", f"{object_prefix}/{pattern}")]
|
|
try:
|
|
# Sync the complete bucket authority so moving a route prefix also
|
|
# deletes objects under its old prefix instead of leaving them public.
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{sync_source}/", bucket_destination,
|
|
"--delete", "--only-show-errors", "--cache-control", default_cache,
|
|
*sync_excludes], env=aws_env)
|
|
finally:
|
|
if stage:
|
|
stage.cleanup()
|
|
print("Re-stamping metadata on all objects...")
|
|
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
|
|
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
|
|
"--recursive", "--only-show-errors", "--cache-control", default_cache,
|
|
*default_filters, *exclude_args], env=aws_env)
|
|
for rule in artifact["cache_rules"]:
|
|
if not rule["path"]:
|
|
continue
|
|
include = f"{rule['path'].rstrip('/')}/*"
|
|
child_filters = [arg for path in specific_paths
|
|
if path.startswith(f"{rule['path'].rstrip('/')}/")
|
|
for arg in ("--exclude", f"{path}/*")]
|
|
# Apply rules from the artifact root so artifact-level exclusions keep
|
|
# their original meaning under every cache override.
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
|
|
"--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
|
|
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
|
|
|
|
|
|
def garage_admin(method, path, token, body=None):
|
|
url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
headers = {"Authorization": f"Bearer {token}"}
|
|
if data is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
req = Request(url, data=data, method=method, headers=headers)
|
|
with urlopen(req) as resp:
|
|
raw = resp.read()
|
|
return json.loads(raw) if raw else {}
|
|
|
|
|
|
def ensure_bucket_aliases(site_name, aliases, admin_token):
|
|
"""Add cfg['aliases'] as Garage globalAliases on the site bucket.
|
|
|
|
Idempotent: skips aliases already present. Never removes aliases not in
|
|
the desired set (safety — orphan removal is manual).
|
|
"""
|
|
if not aliases:
|
|
return
|
|
if not admin_token:
|
|
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
|
|
|
|
try:
|
|
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
|
|
admin_token)
|
|
except (HTTPError, URLError) as e:
|
|
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
|
|
|
|
bucket_id = info.get("id")
|
|
existing = set(info.get("globalAliases") or [])
|
|
print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
|
|
|
|
for alias in aliases:
|
|
if alias in existing:
|
|
continue
|
|
print(f" Adding globalAlias: {alias}")
|
|
try:
|
|
garage_admin("POST", "/v2/AddBucketAlias", admin_token,
|
|
{"bucketId": bucket_id, "globalAlias": alias})
|
|
except HTTPError as e:
|
|
body = e.read().decode(errors="replace") if hasattr(e, "read") else ""
|
|
print(f" ERROR adding alias {alias}: {e} {body}")
|
|
raise
|
|
|
|
|
|
def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
|
|
"""Always re-render manifests from current site.yaml. Templates own
|
|
domain + aliases, so changes propagate without manual edits."""
|
|
manifests_dir.mkdir(parents=True, exist_ok=True)
|
|
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
|
routes = []
|
|
for route in cfg["routes"]:
|
|
artifact = artifact_by_name[route["artifact"]]
|
|
resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
|
|
routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
|
|
template_vars = {
|
|
"site": site_name,
|
|
"site_k8s": k8s_name(site_name),
|
|
"domain": cfg["domain"],
|
|
"aliases": cfg["aliases"],
|
|
"namespace": NAMESPACE,
|
|
"compatibility": cfg["compatibility"],
|
|
"routes": routes,
|
|
}
|
|
render_templates(action_dir, template_vars, app_dir, manifests_dir)
|
|
|
|
|
|
def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
|
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
|
credential_env_names = {
|
|
name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
|
|
}
|
|
validate_publication_environment(cfg)
|
|
for artifact in cfg["artifacts"]:
|
|
validate_artifact_output(site_dir, artifact)
|
|
for route in cfg["routes"]:
|
|
s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
|
|
if cfg["compatibility"]:
|
|
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
|
|
|
|
apps_dir = clone_apps(token)
|
|
app_dir = apps_dir / "sjc001" / "websites" / site_name
|
|
manifests_dir = app_dir / "manifests"
|
|
|
|
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
|
|
|
|
commit_and_push(apps_dir, f"Deploy {site_name}")
|
|
|
|
|
|
def decommission(site_name, token, buckets=None):
|
|
"""Remove manifests from apps repo."""
|
|
apps_dir = clone_apps(token)
|
|
site_path = apps_dir / "sjc001" / "websites" / site_name
|
|
if not site_path.exists():
|
|
print(f"No manifests for {site_name} — nothing to remove")
|
|
return
|
|
shutil.rmtree(site_path)
|
|
commit_and_push(apps_dir, f"Decommission {site_name}")
|
|
for bucket in buckets or [site_name]:
|
|
print(f"Bucket {bucket} and its objects are NOT purged automatically.")
|
|
print(f" garage bucket delete {bucket} --yes")
|
|
|
|
|
|
def cmd_deploy():
|
|
site_repo = env("SITE_REPO")
|
|
site_dir = Path(env("SITE_DIR"))
|
|
action_dir = Path(env("ACTION_DIR"))
|
|
token = env("CI_BOT_TOKEN")
|
|
site_name = site_repo.split("/", 1)[1]
|
|
|
|
cfg = parse_site_yaml(site_dir)
|
|
|
|
if not cfg["enabled"]:
|
|
print("Site disabled — running decommission...")
|
|
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
|
|
return
|
|
|
|
deploy_static(site_name, site_dir, action_dir, token, cfg)
|