337 lines
12 KiB
Python
337 lines
12 KiB
Python
"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
from utils import (
|
|
DEFAULT_S3_ENDPOINT,
|
|
NAMESPACE,
|
|
clone_apps,
|
|
commit_and_push,
|
|
die,
|
|
env,
|
|
k8s_name,
|
|
parse_site_yaml,
|
|
render_templates,
|
|
run_args,
|
|
)
|
|
|
|
GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
|
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
|
|
)
|
|
|
|
|
|
CACHE_CONTROL = "public, max-age=0, must-revalidate"
|
|
|
|
|
|
def credential_environment(profile):
|
|
if profile == "default":
|
|
access_key = env("AWS_ACCESS_KEY_ID")
|
|
secret_key = env("AWS_SECRET_ACCESS_KEY")
|
|
else:
|
|
prefix = f"SITE_PUBLISH_{profile.upper().replace('-', '_')}"
|
|
access_key = env(f"{prefix}_S3_ACCESS_KEY_ID")
|
|
secret_key = env(f"{prefix}_S3_SECRET_ACCESS_KEY")
|
|
child_env = os.environ.copy()
|
|
child_env["AWS_ACCESS_KEY_ID"] = access_key
|
|
child_env["AWS_SECRET_ACCESS_KEY"] = secret_key
|
|
child_env.setdefault("AWS_DEFAULT_REGION", "sjc001")
|
|
return child_env
|
|
|
|
|
|
def configure_cors(bucket, origins, endpoint, child_env):
|
|
if origins is None:
|
|
return
|
|
if not origins:
|
|
run_args(
|
|
["aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
|
|
"--bucket", bucket],
|
|
env=child_env,
|
|
)
|
|
return
|
|
config = {
|
|
"CORSRules": [{
|
|
"AllowedOrigins": origins,
|
|
"AllowedMethods": ["GET", "HEAD"],
|
|
"AllowedHeaders": ["*"],
|
|
"ExposeHeaders": ["ETag"],
|
|
"MaxAgeSeconds": 3600,
|
|
}]
|
|
}
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
|
|
json.dump(config, handle)
|
|
handle.flush()
|
|
run_args(
|
|
["aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
|
|
"--bucket", bucket, "--cors-configuration", f"file://{handle.name}"],
|
|
env=child_env,
|
|
)
|
|
|
|
|
|
def s3_sync(bucket, source, *, credential="default", endpoint=None,
|
|
cache=None, cors_origins=None, excludes=None):
|
|
endpoint = endpoint or os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
|
|
if not source.exists():
|
|
die(f"staged artifact not found — did the build step run? ({source})")
|
|
child_env = credential_environment(credential)
|
|
cache = cache or {"default": CACHE_CONTROL, "rules": []}
|
|
# `excludes` are patterns (site.yaml `excludes:` list) that should never
|
|
# be uploaded *and* should never be deleted from the bucket — escape hatch
|
|
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
|
|
exclude_args = []
|
|
for pattern in excludes or []:
|
|
exclude_args.extend(["--exclude", pattern])
|
|
if excludes:
|
|
print(f"Excluding patterns: {excludes}")
|
|
print(f"Syncing {source} → s3://{bucket} via {endpoint}")
|
|
# `sync --delete` handles new/changed/orphaned files. `cp --recursive`
|
|
# then re-uploads everything to refresh metadata (cache-control,
|
|
# content-type) on objects sync skipped because nothing changed.
|
|
# Cost: a no-op deploy still re-uploads every byte. Sites here are
|
|
# small enough that that's free; correctness wins over throughput.
|
|
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
|
|
# so a fresh upload always carries the right MIME type.
|
|
run_args(
|
|
["aws", "--endpoint-url", endpoint, "s3", "sync", f"{source}/", f"s3://{bucket}/",
|
|
"--delete", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
|
|
env=child_env,
|
|
)
|
|
print("Re-stamping metadata on all objects...")
|
|
run_args(
|
|
["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
|
|
"--recursive", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
|
|
env=child_env,
|
|
)
|
|
for rule in cache["rules"]:
|
|
print(f"Applying cache metadata for {rule['match']}")
|
|
run_args(
|
|
["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
|
|
"--recursive", "--only-show-errors", "--exclude", "*",
|
|
"--include", rule["match"], "--cache-control", rule["value"], *exclude_args],
|
|
env=child_env,
|
|
)
|
|
configure_cors(bucket, cors_origins, endpoint, child_env)
|
|
|
|
|
|
def preflight_artifacts(site_dir, cfg):
|
|
"""Validate every source, credential, and bucket before the first write."""
|
|
for artifact in cfg["artifacts"].values():
|
|
source = site_dir / ".site-publish" / artifact["name"]
|
|
if not source.is_dir() or not any(entry.is_file() for entry in source.rglob("*")):
|
|
die(f"staged artifact is absent or empty: {source}")
|
|
child_env = credential_environment(artifact["credential"])
|
|
endpoint = artifact["endpoint"] or os.environ.get(
|
|
"GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT
|
|
)
|
|
run_args(
|
|
["aws", "--endpoint-url", endpoint, "s3api", "head-bucket",
|
|
"--bucket", artifact["bucket"]],
|
|
env=child_env,
|
|
)
|
|
|
|
|
|
def routed_cache(cache, routes):
|
|
"""Translate source-relative cache patterns to their public object keys."""
|
|
rules = []
|
|
for route in routes:
|
|
prefix = route["path"].lstrip("/")
|
|
for rule in cache["rules"]:
|
|
pattern = f"{prefix}/{rule['match']}" if prefix else rule["match"]
|
|
rules.append({"match": pattern, "value": rule["value"]})
|
|
return {"default": cache["default"], "rules": rules}
|
|
|
|
|
|
def garage_admin(method, path, token, body=None):
|
|
parsed = urlsplit(GARAGE_ADMIN_ENDPOINT)
|
|
if (
|
|
parsed.scheme not in {"http", "https"} or not parsed.netloc
|
|
or parsed.username or parsed.password or parsed.query or parsed.fragment
|
|
):
|
|
die("GARAGE_ADMIN_ENDPOINT must be an HTTP URL without credentials")
|
|
url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
headers = {"Authorization": f"Bearer {token}"}
|
|
if data is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
req = Request(url, data=data, method=method, headers=headers)
|
|
with urlopen(req) as resp:
|
|
raw = resp.read()
|
|
return json.loads(raw) if raw else {}
|
|
|
|
|
|
def ensure_bucket_aliases(site_name, aliases, admin_token):
|
|
"""Add cfg['aliases'] as Garage globalAliases on the site bucket.
|
|
|
|
Idempotent: skips aliases already present. Never removes aliases not in
|
|
the desired set (safety — orphan removal is manual).
|
|
"""
|
|
if not aliases:
|
|
return
|
|
if not admin_token:
|
|
die("GARAGE_ADMIN_TOKEN is required when aliases are configured")
|
|
|
|
try:
|
|
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
|
|
admin_token)
|
|
except (HTTPError, URLError) as e:
|
|
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
|
|
|
|
bucket_id = info.get("id")
|
|
if not isinstance(bucket_id, str) or not bucket_id:
|
|
die(f"bucket lookup for {site_name} returned no bucket id")
|
|
existing = set(info.get("globalAliases") or [])
|
|
print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
|
|
|
|
for alias in aliases:
|
|
if alias in existing:
|
|
continue
|
|
print(f" Adding globalAlias: {alias}")
|
|
try:
|
|
garage_admin("POST", "/v2/AddBucketAlias", admin_token,
|
|
{"bucketId": bucket_id, "globalAlias": alias})
|
|
except HTTPError as e:
|
|
body = e.read().decode(errors="replace") if hasattr(e, "read") else ""
|
|
print(f" ERROR adding alias {alias}: {e} {body}")
|
|
raise
|
|
|
|
|
|
def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
|
|
"""Always re-render manifests from current site.yaml. Templates own
|
|
domain + aliases, so changes propagate without manual edits."""
|
|
if manifests_dir.exists():
|
|
shutil.rmtree(manifests_dir)
|
|
manifests_dir.mkdir(parents=True)
|
|
site_k8s = k8s_name(site_name)
|
|
|
|
if cfg["mode"] == "legacy":
|
|
artifacts = [{
|
|
"name": "site",
|
|
"bucket": site_name,
|
|
"service_name": site_k8s,
|
|
"external_name": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
|
|
"virtual_host": False,
|
|
}]
|
|
routes = [{
|
|
"name": "site",
|
|
"path": "/",
|
|
"artifact": "site",
|
|
"service_name": site_k8s,
|
|
"ingress_name": site_k8s,
|
|
"middlewares": cfg["middlewares"],
|
|
"middleware_refs": [f"{name}@file" for name in cfg["middlewares"]],
|
|
}]
|
|
else:
|
|
artifacts = []
|
|
artifact_by_name = {}
|
|
for name, artifact in cfg["artifacts"].items():
|
|
service_name = k8s_name(f"{site_name}-{name}")
|
|
view = {
|
|
**artifact,
|
|
"service_name": service_name,
|
|
"external_name": f"{artifact['bucket']}.web.sjc001.fritzlab.net",
|
|
"virtual_host": True,
|
|
}
|
|
artifacts.append(view)
|
|
artifact_by_name[name] = view
|
|
routes = []
|
|
for route in cfg["routes"]:
|
|
artifact = artifact_by_name[route["artifact"]]
|
|
ingress_name = k8s_name(f"{site_name}-{route['name']}")
|
|
middleware_refs = [f"{name}@file" for name in route["middlewares"]]
|
|
routes.append({
|
|
**route,
|
|
"service_name": artifact["service_name"],
|
|
"ingress_name": ingress_name,
|
|
"middleware_refs": middleware_refs,
|
|
})
|
|
|
|
template_vars = {
|
|
"site": site_name,
|
|
"site_k8s": site_k8s,
|
|
"domain": cfg["domain"],
|
|
"aliases": cfg["aliases"],
|
|
"namespace": NAMESPACE,
|
|
"artifacts": artifacts,
|
|
"routes": routes,
|
|
"modern": cfg["mode"] == "multi",
|
|
}
|
|
render_templates(action_dir, template_vars, app_dir, manifests_dir)
|
|
|
|
|
|
def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
|
if cfg["mode"] == "legacy":
|
|
s3_sync(
|
|
site_name,
|
|
site_dir / "build" / "html",
|
|
excludes=cfg.get("excludes"),
|
|
)
|
|
ensure_bucket_aliases(
|
|
site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")
|
|
)
|
|
else:
|
|
preflight_artifacts(site_dir, cfg)
|
|
for artifact in cfg["artifacts"].values():
|
|
routes = [
|
|
route for route in cfg["routes"]
|
|
if route["artifact"] == artifact["name"]
|
|
]
|
|
s3_sync(
|
|
artifact["bucket"],
|
|
site_dir / ".site-publish" / artifact["name"],
|
|
credential=artifact["credential"],
|
|
endpoint=artifact["endpoint"],
|
|
cache=routed_cache(artifact["cache"], routes),
|
|
cors_origins=artifact["cors_origins"],
|
|
excludes=artifact["excludes"],
|
|
)
|
|
|
|
apps_dir = clone_apps(token)
|
|
app_dir = apps_dir / "sjc001" / "websites" / site_name
|
|
manifests_dir = app_dir / "manifests"
|
|
|
|
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
|
|
|
|
try:
|
|
commit_and_push(apps_dir, f"Deploy {site_name}", token)
|
|
finally:
|
|
shutil.rmtree(apps_dir.parent, ignore_errors=True)
|
|
|
|
|
|
def decommission(site_name, token):
|
|
"""Remove manifests from apps repo."""
|
|
apps_dir = clone_apps(token)
|
|
try:
|
|
site_path = apps_dir / "sjc001" / "websites" / site_name
|
|
if not site_path.exists():
|
|
print(f"No manifests for {site_name} — nothing to remove")
|
|
return
|
|
shutil.rmtree(site_path)
|
|
commit_and_push(apps_dir, f"Decommission {site_name}", token)
|
|
finally:
|
|
shutil.rmtree(apps_dir.parent, ignore_errors=True)
|
|
print(f"Bucket {site_name} and its objects are NOT purged automatically.")
|
|
print(f" garage bucket delete {site_name} --yes")
|
|
|
|
|
|
def cmd_deploy():
|
|
site_repo = env("SITE_REPO")
|
|
site_dir = Path(env("SITE_DIR"))
|
|
action_dir = Path(env("ACTION_DIR"))
|
|
token = env("CI_BOT_TOKEN")
|
|
site_name = site_repo.split("/", 1)[1]
|
|
|
|
cfg = parse_site_yaml(site_dir)
|
|
|
|
if not cfg["enabled"]:
|
|
print("Site disabled — running decommission...")
|
|
decommission(site_name, token)
|
|
return
|
|
|
|
deploy_static(site_name, site_dir, action_dir, token, cfg)
|