Evelyn ChenandClaude Fable 5.1
b14f6a856b
Test / contract (pull_request) Successful in 7s
Review found the hole in the first commit's claim. render_site_manifests advances the stored route contract for every route in site.yaml, and `access` there is a replacement, not a union the way immutable_paths is. So a catalogue-only publish could write `protected` for the distributions bucket that nothing published — and validate_route_migrations then refuses to put that bucket back public. Unpublished intent became an irreversible fact. Reproduced from the repo's own fixture: after a whole publish the record reads public; after a catalogue-only publish with the route flipped it reads protected, with nothing written to baseline-dist, and reverting fails with "artifact distributions cannot become public while reusing protected bucket baseline-dist". A scoped run now refuses before the first bucket is touched when an unselected artifact's path, access or artifact name differs from what is recorded, naming both contracts. It also refuses an unselected artifact with no published history, which is the same defect at time zero. Publish the artifact in the same run. Three tests: the reviewer's flip scenario (and the same change published in the same run, which proceeds), the no-history case, and one proving deploy_static reaches the guard before publish_route_immutables, reconcile_artifact_cors or s3_sync. Disabling the call site alone turns the last one red. The README sentence is narrowed to what the code actually guarantees, and gains the CORS consequence: a scoped run holds no credential for the other bucket, so a cors_origins change lands with that artifact's next publish rather than on the merge that edits site.yaml. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UjQqc4qFmdpAWaYfy2Aypb
651 lines
29 KiB
Python
651 lines
29 KiB
Python
"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""
|
|
|
|
import fnmatch
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request, urlopen
|
|
|
|
import yaml
|
|
|
|
from utils import (
|
|
NAMESPACE,
|
|
clone_apps,
|
|
commit_and_push,
|
|
die,
|
|
env,
|
|
k8s_name,
|
|
parse_site_yaml,
|
|
render_templates,
|
|
run,
|
|
selected_artifacts,
|
|
selected_routes,
|
|
validate_artifact_inputs,
|
|
)
|
|
|
|
GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
|
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
|
|
)
|
|
|
|
|
|
def validate_artifact_output(site_dir, artifact):
|
|
"""Prove every artifact and declared cache prefix exists before publishing any."""
|
|
html_dir = site_dir / artifact["build_dir"]
|
|
if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")):
|
|
die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}")
|
|
for rule in artifact["cache_rules"]:
|
|
if not rule["path"]:
|
|
continue
|
|
cache_root = html_dir / rule["path"]
|
|
if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")):
|
|
die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files")
|
|
|
|
|
|
def validate_publication_environment(cfg):
|
|
"""Resolve every credential this run needs before the first bucket is changed."""
|
|
for artifact in selected_artifacts(cfg):
|
|
env(artifact["credentials"]["access_key_env"])
|
|
env(artifact["credentials"]["secret_key_env"])
|
|
if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
|
|
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
|
|
|
|
|
|
def _is_immutable(rule):
|
|
return "immutable" in {
|
|
part.strip().lower().split("=", 1)[0]
|
|
for part in rule["cache_control"].split(",")
|
|
}
|
|
|
|
|
|
def _aws_capture(args, aws_env):
|
|
"""Run a non-streaming AWS request without exposing environment credentials."""
|
|
print(f" $ {' '.join(str(part) for part in args)}")
|
|
return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
|
|
|
|
|
|
def _immutable_head(endpoint, bucket, key, aws_env):
|
|
args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
|
|
"--bucket", bucket, "--key", key, "--output", "json"]
|
|
result = _aws_capture(args, aws_env)
|
|
if result.returncode == 0:
|
|
return json.loads(result.stdout)
|
|
error = f"{result.stdout}\n{result.stderr}"
|
|
if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
|
|
return None
|
|
raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
|
|
|
|
|
|
def _immutable_digests(source, cache_control, content_type):
|
|
with source.open("rb") as stream:
|
|
content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
|
|
publication = hashlib.sha256()
|
|
publication.update(cache_control.encode())
|
|
publication.update(b"\0")
|
|
publication.update(content_type.encode())
|
|
publication.update(b"\0")
|
|
with source.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
publication.update(block)
|
|
return content_digest, publication.hexdigest()
|
|
|
|
|
|
def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
|
|
metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
|
|
return (
|
|
metadata.get("sha256") == content_digest
|
|
and metadata.get("publication-sha256") == publication_digest
|
|
and info.get("CacheControl") == cache_control
|
|
and info.get("ContentType") == content_type
|
|
)
|
|
|
|
|
|
def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
|
|
"""Publish a content-addressed key; identical retries converge."""
|
|
content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
|
|
content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
|
|
address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
|
|
if address_digests != [publication_digest]:
|
|
raise RuntimeError(
|
|
f"immutable key must contain its one publication SHA-256 {publication_digest}: "
|
|
f"s3://{bucket}/{key}"
|
|
)
|
|
existing = _immutable_head(endpoint, bucket, key, aws_env)
|
|
if existing is not None:
|
|
if _same_immutable_object(
|
|
existing, content_digest, publication_digest, cache_control, content_type,
|
|
):
|
|
print(f" Immutable object already matches: s3://{bucket}/{key}")
|
|
return False
|
|
raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
|
|
args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
|
|
"--bucket", bucket, "--key", key, "--body", str(source),
|
|
"--content-type", content_type, "--cache-control", cache_control,
|
|
"--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
|
|
result = _aws_capture(args, aws_env)
|
|
if result.returncode == 0:
|
|
return True
|
|
error = f"{result.stdout}\n{result.stderr}"
|
|
raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
|
|
|
|
|
|
def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
|
|
"""Publish one immutable cache partition without overwrite or deletion."""
|
|
rule_root = html_dir / rule["path"]
|
|
child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
|
|
if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
|
|
object_prefix = route["path"].strip("/")
|
|
for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
|
|
relative = source.relative_to(html_dir).as_posix()
|
|
if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
|
|
continue
|
|
if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
|
|
continue
|
|
key = "/".join(part for part in (object_prefix, relative) if part)
|
|
publish_immutable_file(
|
|
artifact["s3_endpoint"], artifact["bucket"], key, source,
|
|
rule["cache_control"], aws_env,
|
|
)
|
|
|
|
|
|
def publication_aws_env(artifact, credential_env_names=None):
|
|
"""Build the route-scoped AWS environment without leaking other credentials."""
|
|
access_key = env(artifact["credentials"]["access_key_env"])
|
|
secret_key = env(artifact["credentials"]["secret_key_env"])
|
|
aws_env = os.environ.copy()
|
|
for name in credential_env_names or artifact["credentials"].values():
|
|
aws_env.pop(name, None)
|
|
for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE",
|
|
"AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"):
|
|
aws_env.pop(name, None)
|
|
aws_env.update({
|
|
"AWS_ACCESS_KEY_ID": access_key,
|
|
"AWS_SECRET_ACCESS_KEY": secret_key,
|
|
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
|
|
})
|
|
return aws_env
|
|
|
|
|
|
def configure_cors(bucket, origins, endpoint, aws_env):
|
|
"""Reconcile read-only browser access without exposing publication credentials."""
|
|
if origins is None:
|
|
return
|
|
config = None
|
|
if origins:
|
|
config = {
|
|
"CORSRules": [{
|
|
"AllowedOrigins": origins,
|
|
"AllowedMethods": ["GET", "HEAD"],
|
|
"AllowedHeaders": ["*"],
|
|
"ExposeHeaders": ["ETag"],
|
|
"MaxAgeSeconds": 3600,
|
|
}],
|
|
}
|
|
set_cors_configuration(bucket, config, endpoint, aws_env)
|
|
|
|
|
|
def set_cors_configuration(bucket, config, endpoint, aws_env):
|
|
"""Apply an exact bucket CORS configuration, or remove it when absent."""
|
|
if config is None:
|
|
run([
|
|
"aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
|
|
"--bucket", bucket,
|
|
], env=aws_env)
|
|
return
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
|
|
json.dump(config, handle)
|
|
handle.flush()
|
|
run([
|
|
"aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
|
|
"--bucket", bucket, "--cors-configuration", f"file://{handle.name}",
|
|
], env=aws_env)
|
|
|
|
|
|
def get_cors_configuration(bucket, endpoint, aws_env):
|
|
"""Read the exact bucket CORS configuration for rollback."""
|
|
result = _aws_capture([
|
|
"aws", "--endpoint-url", endpoint, "s3api", "get-bucket-cors",
|
|
"--bucket", bucket, "--output", "json",
|
|
], aws_env)
|
|
if result.returncode == 0:
|
|
try:
|
|
config = json.loads(result.stdout)
|
|
except json.JSONDecodeError as error:
|
|
raise RuntimeError(f"get-bucket-cors returned invalid JSON for {bucket}") from error
|
|
if not isinstance(config, dict) or not isinstance(config.get("CORSRules"), list):
|
|
raise RuntimeError(f"get-bucket-cors returned an invalid policy for {bucket}")
|
|
return config
|
|
error = f"{result.stdout}\n{result.stderr}"
|
|
if "NoSuchCORSConfiguration" in error:
|
|
return None
|
|
raise RuntimeError(f"get-bucket-cors failed for {bucket}: {error.strip()}")
|
|
|
|
|
|
def reconcile_artifact_cors(artifacts, credential_env_names=None):
|
|
"""Reconcile all policies, restoring the prior set if any write fails."""
|
|
snapshots = []
|
|
for artifact in artifacts:
|
|
if artifact["cors_origins"] is None:
|
|
continue
|
|
aws_env = publication_aws_env(artifact, credential_env_names)
|
|
snapshots.append((
|
|
artifact,
|
|
aws_env,
|
|
get_cors_configuration(artifact["bucket"], artifact["s3_endpoint"], aws_env),
|
|
))
|
|
|
|
attempted = []
|
|
try:
|
|
for artifact, aws_env, previous in snapshots:
|
|
attempted.append((artifact, aws_env, previous))
|
|
configure_cors(
|
|
artifact["bucket"], artifact["cors_origins"], artifact["s3_endpoint"], aws_env,
|
|
)
|
|
except Exception as error:
|
|
rollback_errors = []
|
|
for artifact, aws_env, previous in reversed(attempted):
|
|
try:
|
|
set_cors_configuration(
|
|
artifact["bucket"], previous, artifact["s3_endpoint"], aws_env,
|
|
)
|
|
except Exception as rollback_error:
|
|
rollback_errors.append(f"{artifact['bucket']}: {rollback_error}")
|
|
if rollback_errors:
|
|
raise RuntimeError(
|
|
f"CORS reconciliation failed ({error}); rollback also failed for "
|
|
f"{'; '.join(rollback_errors)}"
|
|
) from error
|
|
raise
|
|
|
|
|
|
def publish_route_immutables(artifact, route, site_dir, credential_env_names=None):
|
|
"""Publish one route's immutable partitions during the global preflight."""
|
|
html_dir = site_dir / artifact["build_dir"]
|
|
aws_env = publication_aws_env(artifact, credential_env_names)
|
|
for rule in artifact["cache_rules"]:
|
|
if _is_immutable(rule):
|
|
publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
|
|
|
|
|
|
def immutable_key_prefixes(artifact, route):
|
|
"""Return immutable partitions as bucket-relative key prefixes."""
|
|
route_prefix = route["path"].strip("/")
|
|
return [
|
|
"/".join(part for part in (route_prefix, rule["path"]) if part)
|
|
for rule in artifact["cache_rules"] if _is_immutable(rule)
|
|
]
|
|
|
|
|
|
def retained_immutable_paths(artifact, route, previous_contract):
|
|
"""Carry all bucket history forward so later route moves cannot delete it."""
|
|
previous_paths = previous_contract["immutable_paths"] if previous_contract else []
|
|
return sorted(set(previous_paths) | set(immutable_key_prefixes(artifact, route)))
|
|
|
|
|
|
def retired_immutable_filters(artifact, route, html_dir, previous_contract):
|
|
"""Protect historical immutable keys that fall inside the current sync scope."""
|
|
if not previous_contract:
|
|
return []
|
|
current_prefix = route["path"].strip("/")
|
|
filters = []
|
|
current_immutable = set(immutable_key_prefixes(artifact, route))
|
|
for immutable_path in previous_contract["immutable_paths"]:
|
|
if immutable_path in current_immutable:
|
|
continue
|
|
if current_prefix:
|
|
marker = f"{current_prefix}/"
|
|
if not immutable_path.startswith(marker):
|
|
continue
|
|
retired_path = immutable_path[len(marker):]
|
|
else:
|
|
retired_path = immutable_path
|
|
collision = html_dir / retired_path
|
|
if collision.exists() and any(path.is_file() for path in collision.rglob("*")):
|
|
raise RuntimeError(
|
|
f"current artifact collides with retired immutable partition: {retired_path}"
|
|
)
|
|
filters.extend(("--exclude", f"{retired_path}/*"))
|
|
return filters
|
|
|
|
|
|
def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contract=None):
|
|
endpoint = artifact["s3_endpoint"]
|
|
html_dir = site_dir / artifact["build_dir"]
|
|
aws_env = publication_aws_env(artifact, credential_env_names)
|
|
bucket = artifact["bucket"]
|
|
object_prefix = route["path"].strip("/")
|
|
destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}"
|
|
default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
|
|
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
|
|
# `excludes` are patterns (site.yaml `excludes:` list) that should never
|
|
# be uploaded *and* should never be deleted from the bucket — escape hatch
|
|
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
|
|
exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)]
|
|
if artifact["excludes"]:
|
|
print(f"Excluding patterns: {artifact['excludes']}")
|
|
print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
|
|
# Upload with the final cache policy before cleanup. Sync and deletion are
|
|
# scoped to the same current route prefix and cache partition. A route move
|
|
# leaves its old bucket partition intact but unreachable after the old
|
|
# Ingress disappears, while stale mutable keys on the serving prefix are
|
|
# deleted. Immutable subtrees are structurally excluded. `cp --recursive`
|
|
# refreshes metadata atomically per object before `sync --delete` removes
|
|
# stale keys without ever exposing new bytes under a provisional policy.
|
|
# A no-op deploy therefore transfers the artifact bytes once.
|
|
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
|
|
# so a fresh upload always carries the right MIME type.
|
|
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
|
|
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
|
|
retired_filters = retired_immutable_filters(artifact, route, html_dir, previous_contract)
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
|
|
"--recursive", "--only-show-errors", "--cache-control", default_cache,
|
|
*default_filters, *retired_filters, *exclude_args], env=aws_env)
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
|
|
"--delete", "--only-show-errors", "--cache-control", default_cache,
|
|
*default_filters, *retired_filters, *exclude_args], env=aws_env)
|
|
for rule in artifact["cache_rules"]:
|
|
if not rule["path"]:
|
|
continue
|
|
if _is_immutable(rule):
|
|
continue
|
|
include = f"{rule['path'].rstrip('/')}/*"
|
|
child_filters = [arg for path in specific_paths
|
|
if path.startswith(f"{rule['path'].rstrip('/')}/")
|
|
for arg in ("--exclude", f"{path}/*")]
|
|
# Apply rules from the artifact root so artifact-level exclusions keep
|
|
# their original meaning under every cache override.
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
|
|
"--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
|
|
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
|
|
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
|
|
"--delete", "--only-show-errors", "--cache-control", rule["cache_control"],
|
|
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
|
|
|
|
|
|
def garage_admin(method, path, token, body=None):
|
|
url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
headers = {"Authorization": f"Bearer {token}"}
|
|
if data is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
req = Request(url, data=data, method=method, headers=headers)
|
|
with urlopen(req) as resp:
|
|
raw = resp.read()
|
|
return json.loads(raw) if raw else {}
|
|
|
|
|
|
def ensure_bucket_aliases(site_name, aliases, admin_token):
|
|
"""Add cfg['aliases'] as Garage globalAliases on the site bucket.
|
|
|
|
Idempotent: skips aliases already present. Never removes aliases not in
|
|
the desired set (safety — orphan removal is manual).
|
|
"""
|
|
if not aliases:
|
|
return
|
|
if not admin_token:
|
|
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
|
|
|
|
try:
|
|
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
|
|
admin_token)
|
|
except (HTTPError, URLError) as e:
|
|
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
|
|
|
|
bucket_id = info.get("id")
|
|
existing = set(info.get("globalAliases") or [])
|
|
print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
|
|
|
|
for alias in aliases:
|
|
if alias in existing:
|
|
continue
|
|
print(f" Adding globalAlias: {alias}")
|
|
try:
|
|
garage_admin("POST", "/v2/AddBucketAlias", admin_token,
|
|
{"bucketId": bucket_id, "globalAlias": alias})
|
|
except HTTPError as e:
|
|
body = e.read().decode(errors="replace") if hasattr(e, "read") else ""
|
|
print(f" ERROR adding alias {alias}: {e} {body}")
|
|
raise
|
|
|
|
|
|
def render_site_manifests(
|
|
site_name, action_dir, app_dir, manifests_dir, cfg, previous_contracts=None,
|
|
):
|
|
"""Always re-render manifests from current site.yaml. Templates own
|
|
domain + aliases, so changes propagate without manual edits."""
|
|
manifests_dir.mkdir(parents=True, exist_ok=True)
|
|
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
|
routes = []
|
|
previous_contracts = previous_contracts or {}
|
|
next_contracts = {bucket: dict(contract)
|
|
for bucket, contract in previous_contracts.items()}
|
|
for route in cfg["routes"]:
|
|
artifact = artifact_by_name[route["artifact"]]
|
|
resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
|
|
previous = previous_contracts.get(artifact["bucket"])
|
|
immutable_paths = retained_immutable_paths(artifact, route, previous)
|
|
next_contracts[artifact["bucket"]] = {
|
|
"path": route["path"],
|
|
"access": "public" if route["access"] == "legacy" else route["access"],
|
|
"artifact": route["artifact"],
|
|
"immutable_paths": immutable_paths,
|
|
}
|
|
routes.append({
|
|
**route, "resource_name": resource_name, "artifact_config": artifact,
|
|
"immutable_paths_json": json.dumps(immutable_paths, separators=(",", ":")),
|
|
})
|
|
template_vars = {
|
|
"site": site_name,
|
|
"site_k8s": k8s_name(site_name),
|
|
"domain": cfg["domain"],
|
|
"aliases": cfg["aliases"],
|
|
"namespace": NAMESPACE,
|
|
"compatibility": cfg["compatibility"],
|
|
"routes": routes,
|
|
}
|
|
render_templates(action_dir, template_vars, app_dir, manifests_dir)
|
|
if not cfg["compatibility"] or previous_contracts:
|
|
(app_dir / "site-publish-history.yaml").write_text(yaml.safe_dump(
|
|
{"version": 1, "buckets": next_contracts}, sort_keys=True,
|
|
))
|
|
|
|
|
|
def _validate_route_contract(bucket, contract, path):
|
|
expected = {"path", "access", "artifact", "immutable_paths"}
|
|
if (not isinstance(bucket, str) or not bucket or not isinstance(contract, dict)
|
|
or set(contract) != expected
|
|
or contract.get("access") not in {"public", "protected"}
|
|
or not isinstance(contract.get("path"), str)
|
|
or not contract["path"].startswith("/")
|
|
or not isinstance(contract.get("artifact"), str) or not contract["artifact"]
|
|
or not isinstance(contract.get("immutable_paths"), list)
|
|
or any(not isinstance(item, str) or not item or item.startswith("/")
|
|
or any(part in {"", ".", ".."} for part in item.split("/"))
|
|
for item in contract["immutable_paths"])):
|
|
raise RuntimeError(f"invalid site-publish route history in {path}")
|
|
return {
|
|
"path": contract["path"], "access": contract["access"],
|
|
"artifact": contract["artifact"],
|
|
"immutable_paths": sorted(set(contract["immutable_paths"])),
|
|
}
|
|
|
|
|
|
def previous_route_contracts(app_dir):
|
|
"""Read bucket-keyed route history from generated Ingresses."""
|
|
history_path = app_dir / "site-publish-history.yaml"
|
|
if history_path.exists():
|
|
document = yaml.safe_load(history_path.read_text())
|
|
if (not isinstance(document, dict) or set(document) != {"version", "buckets"}
|
|
or document["version"] != 1 or not isinstance(document["buckets"], dict)):
|
|
raise RuntimeError(f"invalid site-publish route history in {history_path}")
|
|
return {
|
|
bucket: _validate_route_contract(bucket, contract, history_path)
|
|
for bucket, contract in document["buckets"].items()
|
|
}
|
|
contracts = {}
|
|
manifests = app_dir / "manifests"
|
|
if not manifests.exists():
|
|
return contracts
|
|
for path in sorted(manifests.glob("ingress*.yaml")):
|
|
document = yaml.safe_load(path.read_text()) or {}
|
|
annotations = document.get("metadata", {}).get("annotations", {})
|
|
artifact = annotations.get("site-publish.fritzlab.net/artifact")
|
|
access = annotations.get("site-publish.fritzlab.net/access")
|
|
bucket = annotations.get("site-publish.fritzlab.net/bucket")
|
|
immutable_paths_json = annotations.get("site-publish.fritzlab.net/immutable-paths")
|
|
route_path = annotations.get("site-publish.fritzlab.net/route-path")
|
|
values = (artifact, access, bucket, immutable_paths_json, route_path)
|
|
if all(value is None for value in values):
|
|
continue
|
|
if (not all(isinstance(value, str) for value in values)
|
|
or access not in {"public", "protected"} or not route_path.startswith("/")):
|
|
raise RuntimeError(f"invalid site-publish route history in {path}")
|
|
try:
|
|
immutable_paths = json.loads(immutable_paths_json)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(f"invalid site-publish route history in {path}") from exc
|
|
if not isinstance(immutable_paths, list) or any(not isinstance(item, str) for item in immutable_paths):
|
|
raise RuntimeError(f"invalid site-publish route history in {path}")
|
|
if bucket in contracts:
|
|
raise RuntimeError(f"duplicate site-publish route history for bucket {bucket}")
|
|
contracts[bucket] = _validate_route_contract(bucket, {
|
|
"path": route_path, "access": access, "artifact": artifact,
|
|
"immutable_paths": immutable_paths,
|
|
}, path)
|
|
return contracts
|
|
|
|
|
|
def validate_route_migrations(cfg, previous_contracts):
|
|
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
|
for route in cfg["routes"]:
|
|
artifact = artifacts[route["artifact"]]
|
|
previous = previous_contracts.get(artifact["bucket"])
|
|
if (previous and previous["access"] == "protected"
|
|
and route["access"] in {"public", "legacy"}
|
|
):
|
|
raise RuntimeError(
|
|
f"artifact {route['artifact']} cannot become public while reusing protected "
|
|
f"bucket {artifact['bucket']}"
|
|
)
|
|
|
|
|
|
def validate_scoped_history(cfg, previous_contracts):
|
|
"""A scoped run may not record a route contract it did not publish.
|
|
|
|
render_site_manifests advances the stored contract for every route in
|
|
site.yaml, and `access` is overwritten rather than unioned the way
|
|
immutable_paths is. Without this, a catalogue-only publish could write a
|
|
protected access for the distributions bucket that nothing published, and
|
|
validate_route_migrations would then refuse to put that bucket back —
|
|
unpublished intent turned into an irreversible fact.
|
|
"""
|
|
chosen = set(cfg["selected"])
|
|
if len(chosen) == len(cfg["artifacts"]):
|
|
return
|
|
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
|
for route in cfg["routes"]:
|
|
if route["artifact"] in chosen:
|
|
continue
|
|
artifact = artifact_by_name[route["artifact"]]
|
|
previous = previous_contracts.get(artifact["bucket"])
|
|
if previous is None:
|
|
raise RuntimeError(
|
|
f"a scoped run cannot introduce artifact {route['artifact']}; "
|
|
f"publish it in the same run"
|
|
)
|
|
current = {
|
|
"path": route["path"],
|
|
"access": "public" if route["access"] == "legacy" else route["access"],
|
|
"artifact": route["artifact"],
|
|
}
|
|
recorded = {key: previous[key] for key in current}
|
|
if recorded != current:
|
|
raise RuntimeError(
|
|
f"a scoped run may not change unselected artifact {route['artifact']}'s route "
|
|
f"contract ({recorded} -> {current}); publish it in the same run"
|
|
)
|
|
|
|
|
|
def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
|
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
|
credential_env_names = {
|
|
name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
|
|
}
|
|
# Publication is scoped to the selected artifacts; the rendered route
|
|
# contract is not. Manifests and immutable-path history stay whole, so a
|
|
# partial publish can never retire another artifact's route or bucket.
|
|
publishing = selected_routes(cfg)
|
|
validate_publication_environment(cfg)
|
|
for artifact in selected_artifacts(cfg):
|
|
validate_artifact_output(site_dir, artifact)
|
|
apps_dir = clone_apps(token)
|
|
app_dir = apps_dir / "sjc001" / "websites" / site_name
|
|
manifests_dir = app_dir / "manifests"
|
|
previous_contracts = previous_route_contracts(app_dir)
|
|
validate_route_migrations(cfg, previous_contracts)
|
|
validate_scoped_history(cfg, previous_contracts)
|
|
# Complete immutable work across the whole publication before any route's
|
|
# mutable pointers can change. Partial immutable success is safe; mixing a
|
|
# new route with an old route after a later immutable failure is not.
|
|
for route in publishing:
|
|
publish_route_immutables(
|
|
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
|
|
)
|
|
# Reconcile every browser-read policy before publishing mutable content.
|
|
# A CORS failure therefore cannot leave a new channel pointing at a release
|
|
# whose cross-origin assets browsers cannot consume.
|
|
reconcile_artifact_cors(selected_artifacts(cfg), credential_env_names)
|
|
for route in publishing:
|
|
s3_sync(
|
|
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
|
|
previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]),
|
|
)
|
|
if cfg["compatibility"]:
|
|
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
|
|
|
|
render_site_manifests(
|
|
site_name, action_dir, app_dir, manifests_dir, cfg, previous_contracts,
|
|
)
|
|
|
|
commit_and_push(apps_dir, f"Deploy {site_name}", token)
|
|
|
|
|
|
def decommission(site_name, token, buckets=None):
|
|
"""Remove manifests from apps repo."""
|
|
apps_dir = clone_apps(token)
|
|
site_path = apps_dir / "sjc001" / "websites" / site_name
|
|
if not site_path.exists():
|
|
print(f"No manifests for {site_name} — nothing to remove")
|
|
return
|
|
shutil.rmtree(site_path)
|
|
commit_and_push(apps_dir, f"Decommission {site_name}", token)
|
|
for bucket in buckets or [site_name]:
|
|
print(f"Bucket {bucket} and its objects are NOT purged automatically.")
|
|
print(f" garage bucket delete {bucket} --yes")
|
|
|
|
|
|
def cmd_deploy():
|
|
site_repo = env("SITE_REPO")
|
|
site_dir = Path(env("SITE_DIR"))
|
|
action_dir = Path(env("ACTION_DIR"))
|
|
token = env("CI_BOT_TOKEN")
|
|
site_name = site_repo.split("/", 1)[1]
|
|
|
|
cfg = parse_site_yaml(site_dir)
|
|
|
|
if not cfg["enabled"]:
|
|
if len(cfg["selected"]) != len(cfg["artifacts"]):
|
|
die("a disabled site decommissions whole; drop the artifacts selection")
|
|
print("Site disabled — running decommission...")
|
|
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
|
|
return
|
|
|
|
validate_artifact_inputs(site_dir, cfg)
|
|
deploy_static(site_name, site_dir, action_dir, token, cfg)
|