feat(site-publish): add split-surface publishing
Test / contract (pull_request) Successful in 6s

Authored-By: OpenAI (GPT-5) <noreply@openai.com>
This commit is contained in:
Evelyn Chen
2026-08-29 22:04:50 +00:00
parent f1f780f5a3
commit 5c2630b972
15 changed files with 1450 additions and 193 deletions
+26 -21
View File
@@ -1,25 +1,25 @@
"""Build phase — content prep for static-content sites."""
"""Build each declared static-content artifact independently."""
import shutil
import subprocess
import tempfile
from pathlib import Path
from utils import EXCLUDE_FILES, env, parse_site_yaml, run
from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs
def build_static(site_dir, cfg):
build_dir = site_dir / "build"
html_dir = build_dir / "html"
def build_artifact(site_dir, artifact):
html_dir = site_dir / artifact["build_dir"]
if html_dir.parent.exists():
shutil.rmtree(html_dir.parent)
if build_dir.exists():
shutil.rmtree(build_dir)
content_dir = cfg["content_dir"]
content_dir = artifact["content_dir"]
src = site_dir / content_dir if content_dir else site_dir
if not src.exists():
raise FileNotFoundError(f"artifact {artifact['name']} content_dir not found: {src}")
if cfg["type"] == "static":
print(f"Copying static content from {src}")
if artifact["type"] == "static":
print(f"Copying artifact {artifact['name']} from {src}")
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp) / "html"
shutil.copytree(src, tmp_path, dirs_exist_ok=True)
@@ -29,18 +29,18 @@ def build_static(site_dir, cfg):
shutil.rmtree(p)
elif p.exists():
p.unlink()
build_dir.mkdir(parents=True)
html_dir.parent.mkdir(parents=True)
shutil.move(str(tmp_path), str(html_dir))
elif cfg["type"] == "hugo":
print(f"Building Hugo site from {src}")
run(f"hugo --source {src} --destination {html_dir}")
elif artifact["type"] == "hugo":
print(f"Building Hugo artifact {artifact['name']} from {src}")
run(["hugo", "--source", str(src), "--destination", str(html_dir)])
elif cfg["type"] == "mkdocs":
print(f"Building MkDocs site from {src}")
run(f"cd {src} && mkdocs build -d {html_dir}")
elif artifact["type"] == "mkdocs":
print(f"Building MkDocs artifact {artifact['name']} from {src}")
run(["mkdocs", "build", "-d", str(html_dir)], cwd=src)
if cfg.get("tidy", True):
if artifact["tidy"]:
print("Running tidy on HTML files...")
for html_file in html_dir.rglob("*.html"):
subprocess.run(
@@ -51,7 +51,9 @@ def build_static(site_dir, cfg):
check=False,
)
print(f"Build complete — content at {html_dir}")
if not any(path.is_file() for path in html_dir.rglob("*")):
raise FileNotFoundError(f"artifact {artifact['name']} produced no files in {html_dir}")
print(f"Artifact {artifact['name']} complete — content at {html_dir}")
def cmd_build():
@@ -62,4 +64,7 @@ def cmd_build():
print("Site disabled — skipping build")
return
build_static(site_dir, cfg)
validate_artifact_inputs(site_dir, cfg)
for artifact in cfg["artifacts"]:
build_artifact(site_dir, artifact)
+222 -59
View File
@@ -1,17 +1,18 @@
"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""
import fnmatch
import hashlib
import json
import mimetypes
import os
import shlex
import re
import shutil
import tempfile
import subprocess
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from utils import (
DEFAULT_S3_ENDPOINT,
GITEA_HOST,
NAMESPACE,
clone_apps,
commit_and_push,
@@ -21,6 +22,7 @@ from utils import (
parse_site_yaml,
render_templates,
run,
validate_artifact_inputs,
)
GARAGE_ADMIN_ENDPOINT = os.environ.get(
@@ -28,44 +30,194 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
)
CACHE_CONTROL = "public, max-age=0, must-revalidate"
def validate_artifact_output(site_dir, artifact):
"""Prove every artifact and declared cache prefix exists before publishing any."""
html_dir = site_dir / artifact["build_dir"]
if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")):
die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}")
for rule in artifact["cache_rules"]:
if not rule["path"]:
continue
cache_root = html_dir / rule["path"]
if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")):
die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files")
def s3_sync(site_name, site_dir, excludes=None):
endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
html_dir = site_dir / "build" / "html"
if not html_dir.exists():
die(f"build/html not found — did the build step run? ({html_dir})")
env("AWS_ACCESS_KEY_ID")
env("AWS_SECRET_ACCESS_KEY")
os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
def validate_publication_environment(cfg):
"""Resolve every declared credential before the first bucket is changed."""
for artifact in cfg["artifacts"]:
env(artifact["credentials"]["access_key_env"])
env(artifact["credentials"]["secret_key_env"])
if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
def _is_immutable(rule):
return "immutable" in {
part.strip().lower().split("=", 1)[0]
for part in rule["cache_control"].split(",")
}
def _aws_capture(args, aws_env):
"""Run a non-streaming AWS request without exposing environment credentials."""
print(f" $ {' '.join(str(part) for part in args)}")
return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
def _immutable_head(endpoint, bucket, key, aws_env):
args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
"--bucket", bucket, "--key", key, "--output", "json"]
result = _aws_capture(args, aws_env)
if result.returncode == 0:
return json.loads(result.stdout)
error = f"{result.stdout}\n{result.stderr}"
if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
return None
raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
def _immutable_digests(source, cache_control, content_type):
with source.open("rb") as stream:
content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
publication = hashlib.sha256()
publication.update(cache_control.encode())
publication.update(b"\0")
publication.update(content_type.encode())
publication.update(b"\0")
with source.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
publication.update(block)
return content_digest, publication.hexdigest()
def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
return (
metadata.get("sha256") == content_digest
and metadata.get("publication-sha256") == publication_digest
and info.get("CacheControl") == cache_control
and info.get("ContentType") == content_type
)
def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
"""Publish a content-addressed key; identical retries converge."""
content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
if address_digests != [publication_digest]:
raise RuntimeError(
f"immutable key must contain its one publication SHA-256 {publication_digest}: "
f"s3://{bucket}/{key}"
)
existing = _immutable_head(endpoint, bucket, key, aws_env)
if existing is not None:
if _same_immutable_object(
existing, content_digest, publication_digest, cache_control, content_type,
):
print(f" Immutable object already matches: s3://{bucket}/{key}")
return False
raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
"--bucket", bucket, "--key", key, "--body", str(source),
"--content-type", content_type, "--cache-control", cache_control,
"--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
result = _aws_capture(args, aws_env)
if result.returncode == 0:
return True
error = f"{result.stdout}\n{result.stderr}"
raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
"""Publish one immutable cache partition without overwrite or deletion."""
rule_root = html_dir / rule["path"]
child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
object_prefix = route["path"].strip("/")
for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
relative = source.relative_to(html_dir).as_posix()
if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
continue
if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
continue
key = "/".join(part for part in (object_prefix, relative) if part)
publish_immutable_file(
artifact["s3_endpoint"], artifact["bucket"], key, source,
rule["cache_control"], aws_env,
)
def s3_sync(artifact, route, site_dir, credential_env_names=None):
endpoint = artifact["s3_endpoint"]
html_dir = site_dir / artifact["build_dir"]
access_key = env(artifact["credentials"]["access_key_env"])
secret_key = env(artifact["credentials"]["secret_key_env"])
aws_env = os.environ.copy()
for name in credential_env_names or artifact["credentials"].values():
aws_env.pop(name, None)
for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE",
"AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"):
aws_env.pop(name, None)
aws_env.update({
"AWS_ACCESS_KEY_ID": access_key,
"AWS_SECRET_ACCESS_KEY": secret_key,
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
})
bucket = artifact["bucket"]
object_prefix = route["path"].strip("/")
destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}"
default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
# `excludes` are patterns (site.yaml `excludes:` list) that should never
# be uploaded *and* should never be deleted from the bucket — escape hatch
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
if excludes:
print(f"Excluding patterns: {excludes}")
print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
# `sync --delete` handles new/changed/orphaned files. `cp --recursive`
# then re-uploads everything to refresh metadata (cache-control,
# content-type) on objects sync skipped because nothing changed.
# Cost: a no-op deploy still re-uploads every byte. Sites here are
# small enough that that's free; correctness wins over throughput.
exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)]
if artifact["excludes"]:
print(f"Excluding patterns: {artifact['excludes']}")
# Validate and publish every append-only target before a mutable channel can
# point at it. Partial immutable success is safe; partial mutable success is
# not.
for rule in artifact["cache_rules"]:
if _is_immutable(rule):
publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
print(f"Syncing artifact {artifact['name']}{destination} via {endpoint}")
# Upload with the final cache policy before cleanup. Sync and deletion are
# scoped to the same current route prefix and cache partition. A route move
# leaves its old bucket partition intact but unreachable after the old
# Ingress disappears, while stale mutable keys on the serving prefix are
# deleted. Immutable subtrees are structurally excluded. `cp --recursive`
# refreshes metadata atomically per object before `sync --delete` removes
# stale keys without ever exposing new bytes under a provisional policy.
# A no-op deploy therefore transfers the artifact bytes once.
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
# so a fresh upload always carries the right MIME type.
run(
f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
f"--delete --only-show-errors "
f"--cache-control '{CACHE_CONTROL}' "
f"{exclude_flags}".rstrip()
)
print("Re-stamping metadata on all objects...")
run(
f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
f"--recursive --only-show-errors "
f"--cache-control '{CACHE_CONTROL}' "
f"{exclude_flags}".rstrip()
)
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *exclude_args], env=aws_env)
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--delete", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *exclude_args], env=aws_env)
for rule in artifact["cache_rules"]:
if not rule["path"]:
continue
if _is_immutable(rule):
continue
include = f"{rule['path'].rstrip('/')}/*"
child_filters = [arg for path in specific_paths
if path.startswith(f"{rule['path'].rstrip('/')}/")
for arg in ("--exclude", f"{path}/*")]
# Apply rules from the artifact root so artifact-level exclusions keep
# their original meaning under every cache override.
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--delete", "--only-show-errors", "--cache-control", rule["cache_control"],
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
def garage_admin(method, path, token, body=None):
@@ -89,15 +241,13 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
if not aliases:
return
if not admin_token:
print(" (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
return
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
try:
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
admin_token)
except (HTTPError, URLError) as e:
print(f" WARNING: bucket lookup failed: {e}")
return
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
bucket_id = info.get("id")
existing = set(info.get("globalAliases") or [])
@@ -120,20 +270,36 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
"""Always re-render manifests from current site.yaml. Templates own
domain + aliases, so changes propagate without manual edits."""
manifests_dir.mkdir(parents=True, exist_ok=True)
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
routes = []
for route in cfg["routes"]:
artifact = artifact_by_name[route["artifact"]]
resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
template_vars = {
"site": site_name,
"site_k8s": k8s_name(site_name),
"domain": cfg["domain"],
"aliases": cfg["aliases"],
"namespace": NAMESPACE,
"middlewares": cfg["middlewares"],
"compatibility": cfg["compatibility"],
"routes": routes,
}
render_templates(action_dir, template_vars, app_dir, manifests_dir)
def deploy_static(site_name, site_dir, action_dir, token, cfg):
s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
credential_env_names = {
name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
}
validate_publication_environment(cfg)
for artifact in cfg["artifacts"]:
validate_artifact_output(site_dir, artifact)
for route in cfg["routes"]:
s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
if cfg["compatibility"]:
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
@@ -141,25 +307,21 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
commit_and_push(apps_dir, f"Deploy {site_name}")
commit_and_push(apps_dir, f"Deploy {site_name}", token)
def decommission(site_name, token):
def decommission(site_name, token, buckets=None):
"""Remove manifests from apps repo."""
user = env("CI_BOT_USER", "ci-bot")
with tempfile.TemporaryDirectory() as tmp:
apps_dir = Path(tmp)
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
site_path = apps_dir / "sjc001" / "websites" / site_name
if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove")
return
shutil.rmtree(site_path)
run(f"git -C {apps_dir} config user.name {user}")
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
commit_and_push(apps_dir, f"Decommission {site_name}")
print(f"Bucket {site_name} and its objects are NOT purged automatically.")
print(f" garage bucket delete {site_name} --yes")
apps_dir = clone_apps(token)
site_path = apps_dir / "sjc001" / "websites" / site_name
if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove")
return
shutil.rmtree(site_path)
commit_and_push(apps_dir, f"Decommission {site_name}", token)
for bucket in buckets or [site_name]:
print(f"Bucket {bucket} and its objects are NOT purged automatically.")
print(f" garage bucket delete {bucket} --yes")
def cmd_deploy():
@@ -173,7 +335,8 @@ def cmd_deploy():
if not cfg["enabled"]:
print("Site disabled — running decommission...")
decommission(site_name, token)
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
return
validate_artifact_inputs(site_dir, cfg)
deploy_static(site_name, site_dir, action_dir, token, cfg)
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
case "$1" in
*Username*) printf '%s\n' "$CI_BOT_USER" ;;
*) printf '%s\n' "$CI_BOT_TOKEN" ;;
esac
+3 -2
View File
@@ -24,9 +24,10 @@ def ensure_aws():
subprocess.run(["aws", "--version"], check=True)
def ensure_jinja2():
def ensure_python_dependencies():
try:
import jinja2
import yaml
except ImportError:
print("Installing jinja2 + pyyaml...")
subprocess.run(
@@ -36,6 +37,6 @@ def ensure_jinja2():
if __name__ == "__main__":
ensure_jinja2()
ensure_python_dependencies()
ensure_aws()
print("Setup complete")
+445 -69
View File
@@ -1,26 +1,34 @@
"""Shared utilities for the site-publish action."""
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
from pathlib import Path, PurePosixPath
from urllib.parse import urlparse
import yaml
from jinja2 import Environment, FileSystemLoader
from jinja2 import Environment, FileSystemLoader, StrictUndefined
APPS_REPO = "fritzlab/apps"
GITEA_HOST = "code.fritzlab.net"
NAMESPACE = "websites"
DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
DEFAULT_WEBSITE_SUFFIX = "web.sjc001.fritzlab.net"
DEFAULT_CACHE_CONTROL = "public, max-age=0, must-revalidate"
EXCLUDE_FILES = {
".git", ".gitea", ".gitignore", "site.yaml",
"build", "Makefile", "README.md", "CLAUDE.md",
"Dockerfile", ".dockerignore", "go.mod", "go.sum",
".git", ".gitea", ".gitignore", "site.yaml", "build", ".site-publish",
"Makefile", "README.md", "CLAUDE.md", "Dockerfile", ".dockerignore",
"go.mod", "go.sum",
}
VALID_TYPES = {"static", "hugo", "mkdocs"}
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
ENV_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$")
ACCESS_KEY_ENV_RE = re.compile(r"^([A-Z][A-Z0-9_]*)_S3_ACCESS_KEY$")
BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?$")
MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9_-]{0,61}[A-Za-z0-9])?$")
DOCKER_DEPRECATION_MSG = """\
type: docker is no longer supported by action/site-publish.
@@ -42,113 +50,481 @@ example.\
"""
class ConfigError(ValueError):
"""A site.yaml contract violation."""
def k8s_name(name):
"""Sanitize for DNS-1035 label (dots → dashes)."""
return name.replace(".", "-")
def env(key, default=None):
val = os.environ.get(key, default)
if val is None:
if val is None or val == "":
die(f"Missing required env var: {key}")
return val
def die(msg):
print(f"ERROR: {msg}", file=sys.stderr)
sys.exit(1)
raise SystemExit(1)
def run(cmd, **kwargs):
print(f" $ {cmd}")
return subprocess.run(cmd, shell=True, check=True, **kwargs)
def run(cmd, *, display=None, **kwargs):
"""Run an argv command, printing only a safe display form."""
if not isinstance(cmd, (list, tuple)):
raise TypeError("run() requires an argv list")
shown = display if display is not None else " ".join(str(part) for part in cmd)
print(f" $ {shown}")
return subprocess.run(cmd, check=True, **kwargs)
def parse_site_yaml(site_dir):
path = Path(site_dir) / "site.yaml"
if not path.exists():
die("site.yaml not found in repo root")
def _mapping(value, label):
if not isinstance(value, dict):
raise ConfigError(f"{label} must be a mapping")
return value
with open(path) as f:
cfg = yaml.safe_load(f)
if not cfg.get("domain"):
die("domain is required in site.yaml")
def _list(value, label):
if not isinstance(value, list):
raise ConfigError(f"{label} must be a list")
return value
site_type = cfg.get("type", "static")
def _known_keys(value, allowed, label):
unknown = set(value) - set(allowed)
if unknown:
raise ConfigError(f"{label} has unknown fields: {', '.join(sorted(unknown))}")
def _strings(value, label):
values = _list(value or [], label)
if any(not isinstance(item, str) or not item for item in values):
raise ConfigError(f"{label} must be a list of non-empty strings")
return values
def _hostname(value, label):
if not isinstance(value, str) or len(value) > 253 or value.endswith("."):
raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot")
labels = value.split(".")
if len(labels) < 2 or any(not NAME_RE.fullmatch(part) for part in labels):
raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot")
return value
def _aliases(value, domain):
aliases = _strings(value or [], "aliases")
aliases = [_hostname(alias, "aliases entry") for alias in aliases]
if len(aliases) != len(set(aliases)) or domain in aliases:
raise ConfigError("aliases must be unique and cannot repeat domain")
return aliases
def _relative_path(value, label, *, allow_root=False):
value = "" if value is None else value
if not isinstance(value, str):
raise ConfigError(f"{label} must be a string")
if value == "/" and allow_root:
return ""
path = PurePosixPath(value)
if path.is_absolute() or ".." in path.parts:
raise ConfigError(f"{label} must be a relative path without '..'")
normalized = str(path).strip("/")
return "" if normalized == "." else normalized
def _route_path(value, label):
if not isinstance(value, str) or not value.startswith("/"):
raise ConfigError(f"{label} must start with '/'")
if "//" in value or "?" in value or "#" in value or ".." in value.split("/"):
raise ConfigError(f"{label} is not a canonical URL path")
normalized = value.rstrip("/") or "/"
if not re.fullmatch(r"/[A-Za-z0-9._~/-]*", normalized):
raise ConfigError(f"{label} contains unsupported URL path characters")
return normalized
def _middlewares(value, label):
names = _strings(value, label)
if any(not MIDDLEWARE_RE.fullmatch(name) for name in names):
raise ConfigError(f"{label} contains an invalid middleware name")
if len(names) != len(set(names)):
raise ConfigError(f"{label} contains duplicate middleware names")
return names
def _cache_control(value, label):
if not isinstance(value, str) or not value.strip():
raise ConfigError(f"{label} must be a non-empty Cache-Control value")
directives = [part.strip().lower() for part in value.split(",")]
names = [part.split("=", 1)[0] for part in directives]
if len(names) != len(set(names)):
raise ConfigError(f"{label} repeats a Cache-Control directive")
present = set(names)
if {"public", "private"} <= present:
raise ConfigError(f"{label} cannot be both public and private")
ages = {}
for part in directives:
name = part.split("=", 1)[0]
if name in {"max-age", "s-maxage"}:
raw = part.split("=", 1)[1] if "=" in part else ""
if not raw.isdigit():
raise ConfigError(f"{label} {name} must be a non-negative integer")
ages[name] = int(raw)
max_age = ages.get("max-age")
if "immutable" in present and (not max_age or present & {"no-store", "no-cache", "must-revalidate"}):
raise ConfigError(f"{label} immutable requires positive max-age without revalidation")
if "no-store" in present and any(ages.values()):
raise ConfigError(f"{label} no-store contradicts positive caching")
if "private" in present and ages.get("s-maxage"):
raise ConfigError(f"{label} private contradicts shared-cache max-age")
return ", ".join(part.strip() for part in value.split(","))
def _site_type(value, label):
site_type = value or "static"
if site_type == "docker":
die(DOCKER_DEPRECATION_MSG)
raise ConfigError(DOCKER_DEPRECATION_MSG)
if site_type not in VALID_TYPES:
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
raise ConfigError(f"Unknown {label}: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
return site_type
excludes = cfg.get("excludes") or []
if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes):
die("excludes must be a list of string patterns")
middlewares = cfg.get("middlewares") or []
if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares):
die("middlewares must be a list of Traefik file-provider middleware names")
def _endpoint(value, label):
parsed = urlparse(value)
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.path not in {"", "/"}:
raise ConfigError(f"{label} must be an http(s) origin without a path")
return value.rstrip("/")
site = {
"domain": cfg["domain"],
"type": site_type,
"enabled": cfg.get("enabled", True),
"aliases": cfg.get("aliases") or [],
"content_dir": cfg.get("content_dir", ""),
"tidy": cfg.get("tidy", True),
"excludes": excludes,
def _legacy_config(raw, site_name):
if not isinstance(raw.get("tidy", True), bool):
raise ConfigError("tidy must be a boolean")
if not isinstance(raw.get("enabled", True), bool):
raise ConfigError("enabled must be a boolean")
artifact = {
"name": "site",
"type": _site_type(raw.get("type", "static"), "site type"),
"content_dir": _relative_path(raw.get("content_dir", ""), "content_dir"),
"tidy": raw.get("tidy", True),
"excludes": _strings(raw.get("excludes") or [], "excludes"),
"build_dir": "build/html",
"bucket": site_name,
"s3_endpoint": os.environ.get("GARAGE_S3_ENDPOINT") or DEFAULT_S3_ENDPOINT,
"website_authority": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
"credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", "secret_key_env": "AWS_SECRET_ACCESS_KEY"},
"cache_rules": [{"path": "", "cache_control": DEFAULT_CACHE_CONTROL}],
}
return {
"version": 1,
"compatibility": "single-surface-v1",
"domain": raw["domain"],
"aliases": _aliases(raw.get("aliases"), raw["domain"]),
"enabled": raw.get("enabled", True),
"artifacts": [artifact],
"routes": [{
"name": "site", "path": "/", "artifact": "site", "access": "legacy",
"access_middleware": None,
"middlewares": _middlewares(raw.get("middlewares") or [], "middlewares"),
}],
}
def _artifact(item, index):
label = f"artifacts[{index}]"
item = _mapping(item, label)
_known_keys(item, {"name", "type", "content_dir", "tidy", "excludes", "publish", "cache"}, label)
name = item.get("name")
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
raise ConfigError(f"{label}.name must be a DNS label")
publish = _mapping(item.get("publish"), f"{label}.publish")
_known_keys(publish, {"bucket", "credentials"}, f"{label}.publish")
bucket = publish.get("bucket")
if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket):
raise ConfigError(f"{label}.publish.bucket is not a valid bucket name")
credentials = _mapping(publish.get("credentials"), f"{label}.publish.credentials")
_known_keys(credentials, {"access_key_env", "secret_key_env"}, f"{label}.publish.credentials")
normalized_credentials = {}
for key in ("access_key_env", "secret_key_env"):
value = credentials.get(key)
if not isinstance(value, str) or not ENV_RE.fullmatch(value):
raise ConfigError(f"{label}.publish.credentials.{key} must name an environment variable")
normalized_credentials[key] = value
access_match = ACCESS_KEY_ENV_RE.fullmatch(normalized_credentials["access_key_env"])
expected_secret = (
f"{access_match.group(1)}_S3_SECRET_KEY" if access_match else None
)
if normalized_credentials["secret_key_env"] != expected_secret:
raise ConfigError(
f"{label}.publish.credentials must be a matched "
"<NAME>_S3_ACCESS_KEY and <NAME>_S3_SECRET_KEY pair"
)
cache = _mapping(item.get("cache"), f"{label}.cache")
_known_keys(cache, {"rules"}, f"{label}.cache")
rules = _list(cache.get("rules"), f"{label}.cache.rules")
if not rules:
raise ConfigError(f"{label}.cache.rules must declare a '/' default")
cache_rules, paths = [], set()
for rule_index, rule in enumerate(rules):
rule_label = f"{label}.cache.rules[{rule_index}]"
rule = _mapping(rule, rule_label)
_known_keys(rule, {"path", "cache_control"}, rule_label)
path = _relative_path(rule.get("path"), f"{rule_label}.path", allow_root=True)
if path in paths:
raise ConfigError(f"{label}.cache.rules has duplicate path /{path}")
paths.add(path)
cache_rules.append({"path": path, "cache_control": _cache_control(
rule.get("cache_control"), f"{rule_label}.cache_control"
)})
if "" not in paths:
raise ConfigError(f"{label}.cache.rules must declare a '/' default")
cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"]))
immutable_paths = [rule["path"] for rule in cache_rules
if "immutable" in {part.strip().lower().split("=", 1)[0]
for part in rule["cache_control"].split(",")}]
if "" in immutable_paths:
raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'")
for path in immutable_paths:
if any(other.startswith(f"{path}/") for other in paths):
raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}")
authority = f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}"
if not isinstance(item.get("tidy", True), bool):
raise ConfigError(f"{label}.tidy must be a boolean")
return {
"name": name,
"type": _site_type(item.get("type", "static"), f"{label}.type"),
"content_dir": _relative_path(item.get("content_dir", ""), f"{label}.content_dir"),
"tidy": item.get("tidy", True),
"excludes": _strings(item.get("excludes") or [], f"{label}.excludes"),
"build_dir": f".site-publish/{name}/html",
"bucket": bucket,
"s3_endpoint": DEFAULT_S3_ENDPOINT,
"website_authority": authority,
"credentials": normalized_credentials,
"cache_rules": cache_rules,
}
def _route(item, index):
label = f"routes[{index}]"
item = _mapping(item, label)
_known_keys(item, {"name", "path", "artifact", "access", "middlewares"}, label)
name = item.get("name")
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
raise ConfigError(f"{label}.name must be a DNS label")
access = _mapping(item.get("access"), f"{label}.access")
_known_keys(access, {"mode", "middleware"}, f"{label}.access")
mode, middleware = access.get("mode"), access.get("middleware")
if mode not in {"public", "protected"}:
raise ConfigError(f"{label}.access.mode must be public or protected")
if mode == "protected" and (not isinstance(middleware, str) or not MIDDLEWARE_RE.fullmatch(middleware)):
raise ConfigError(f"{label}.access.middleware is required for protected access")
if mode == "public" and middleware is not None:
raise ConfigError(f"{label}.access.middleware is forbidden for public access")
middlewares = _middlewares(item.get("middlewares") or [], f"{label}.middlewares")
if middleware in middlewares:
raise ConfigError(f"{label} repeats its access middleware")
artifact = item.get("artifact")
if not isinstance(artifact, str):
raise ConfigError(f"{label}.artifact must name an artifact")
return {
"name": name, "path": _route_path(item.get("path"), f"{label}.path"),
"artifact": artifact, "access": mode, "access_middleware": middleware,
"middlewares": middlewares,
}
def _validate_multi(cfg):
artifacts, routes = cfg["artifacts"], cfg["routes"]
artifact_names = [item["name"] for item in artifacts]
route_names = [item["name"] for item in routes]
route_paths = [item["path"] for item in routes]
if len(artifact_names) != len(set(artifact_names)):
raise ConfigError("artifact names must be unique")
if len(route_names) != len(set(route_names)):
raise ConfigError("route names must be unique")
if len(route_paths) != len(set(route_paths)):
raise ConfigError("route paths are ambiguous after normalization")
if "/" not in route_paths:
raise ConfigError("routes must declare a '/' catch-all")
artifact_by_name = {item["name"]: item for item in artifacts}
references = {name: [] for name in artifact_by_name}
for route in routes:
if route["artifact"] not in artifact_by_name:
raise ConfigError(f"route {route['name']} references unknown artifact {route['artifact']}")
references[route["artifact"]].append(route)
for name, used_by in references.items():
if len(used_by) != 1:
raise ConfigError(f"artifact {name} must be referenced by exactly one route")
buckets, authorities, credential_owners = {}, {}, {}
for route in routes:
artifact = artifact_by_name[route["artifact"]]
if artifact["bucket"] in buckets:
other_access, other_name = buckets[artifact["bucket"]]
if other_access != route["access"]:
raise ConfigError(f"bucket {artifact['bucket']} cannot be reused by protected and public routes")
raise ConfigError(f"bucket {artifact['bucket']} must belong to one artifact ({other_name})")
buckets[artifact["bucket"]] = (route["access"], artifact["name"])
if artifact["website_authority"] in authorities:
raise ConfigError(
f"website authority {artifact['website_authority']} must belong to one artifact"
)
authorities[artifact["website_authority"]] = artifact["name"]
for variable in artifact["credentials"].values():
if variable in credential_owners:
raise ConfigError(
f"publication credential {variable} is reused by artifacts "
f"{credential_owners[variable]} and {artifact['name']}"
)
credential_owners[variable] = artifact["name"]
for cache_rule in artifact["cache_rules"]:
directives = {part.strip().lower().split("=", 1)[0]
for part in cache_rule["cache_control"].split(",")}
if route["access"] == "protected" and "public" in directives:
raise ConfigError(f"protected route {route['name']} cannot use public cache policy")
if route["access"] == "protected" and not directives & {"private", "no-store"}:
raise ConfigError(f"protected route {route['name']} cache policy must be private or no-store")
if route["access"] == "protected" and "s-maxage" in directives:
raise ConfigError(f"protected route {route['name']} cannot use shared-cache max-age")
if route["access"] == "public" and "private" in directives:
raise ConfigError(f"public route {route['name']} cannot use private cache policy")
root = next(route for route in routes if route["path"] == "/")
if any(route["access"] == "protected" for route in routes) and root["access"] == "public":
raise ConfigError("a public '/' catch-all would expose unmatched protected content")
def normalize_site_config(raw, site_name):
raw = _mapping(raw, "site.yaml")
domain = _hostname(raw.get("domain"), "domain")
if not isinstance(raw.get("enabled", True), bool):
raise ConfigError("enabled must be a boolean")
if ("artifacts" in raw) != ("routes" in raw):
raise ConfigError("artifacts and routes must be declared together")
if "artifacts" not in raw:
return _legacy_config(raw, site_name)
legacy_fields = {"type", "content_dir", "tidy", "excludes", "middlewares"} & set(raw)
if legacy_fields:
raise ConfigError("legacy fields cannot be mixed with artifacts/routes: " + ", ".join(sorted(legacy_fields)))
_known_keys(raw, {"domain", "aliases", "enabled", "artifacts", "routes"}, "site.yaml")
artifacts = [_artifact(item, index) for index, item in enumerate(_list(raw["artifacts"], "artifacts"))]
routes = [_route(item, index) for index, item in enumerate(_list(raw["routes"], "routes"))]
if not artifacts or not routes:
raise ConfigError("artifacts and routes must not be empty")
cfg = {
"version": 2, "compatibility": None, "domain": domain,
"aliases": _aliases(raw.get("aliases"), domain),
"enabled": raw.get("enabled", True),
"artifacts": sorted(artifacts, key=lambda item: item["name"]),
"routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])),
}
_validate_multi(cfg)
for route in cfg["routes"]:
if len(f"{k8s_name(site_name)}-{route['name']}") > 63:
raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters")
return cfg
def validate_artifact_inputs(site_dir, cfg):
"""Reject source containment before a public or protected build starts."""
root = Path(site_dir).resolve()
sources = []
for artifact in cfg["artifacts"]:
source = (root / artifact["content_dir"]).resolve()
if source != root and root not in source.parents:
raise ConfigError(
f"artifact {artifact['name']} content_dir resolves outside the repository"
)
sources.append((artifact["name"], source))
for index, (name, source) in enumerate(sources):
for other_name, other_source in sources[index + 1:]:
if source == other_source or source in other_source.parents or other_source in source.parents:
raise ConfigError(
f"artifact build inputs overlap after resolution: {name} and {other_name}"
)
def parse_site_yaml(site_dir, site_name=None):
path = Path(site_dir) / "site.yaml"
if not path.exists():
die("site.yaml not found in repo root")
if site_name is None:
repo = os.environ.get("SITE_REPO", "")
site_name = repo.split("/", 1)[-1] if "/" in repo else Path(site_dir).name
try:
with open(path) as stream:
cfg = normalize_site_config(yaml.safe_load(stream), site_name)
except (ConfigError, yaml.YAMLError) as exc:
die(str(exc))
print("Site config:")
for k, v in site.items():
print(f" {k}: {v}")
return site
print(f" domain: {cfg['domain']}")
print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}")
print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}")
print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}")
return cfg
def clone_apps(token):
"""Clone Apps without placing the credential in argv or output."""
user = env("CI_BOT_USER", "ci-bot")
apps_dir = Path("/tmp/apps-deploy")
if apps_dir.exists():
shutil.rmtree(apps_dir)
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}")
run(f"git -C {apps_dir} config user.name {user}")
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
clone_env = git_auth_env(token)
url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git"
run(["git", "clone", "--depth", "1", url, str(apps_dir)],
display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env)
run(["git", "-C", str(apps_dir), "config", "user.name", user])
run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
return apps_dir
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
"""Render Jinja2 templates for a static-content site."""
templates_dir = Path(action_dir) / "templates"
jinja_env = Environment(
loader=FileSystemLoader(str(templates_dir)),
keep_trailing_newline=True,
)
tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2",
"kustomization.yaml.j2", "service.yaml.j2"]
for tmpl_name in tmpl_names:
tmpl = jinja_env.get_template(tmpl_name)
rendered = tmpl.render(**template_vars)
out_name = tmpl_name.replace(".j2", "")
dest = app_dir / out_name if tmpl_name == "app.yaml.j2" else manifests_dir / out_name
dest.write_text(rendered)
print(f" Rendered {tmpl_name} -> {dest}")
"""Render one certificate and deterministic per-route resources."""
jinja_env = Environment(loader=FileSystemLoader(str(Path(action_dir) / "templates")),
keep_trailing_newline=True, undefined=StrictUndefined)
manifests_dir.mkdir(parents=True, exist_ok=True)
for child in manifests_dir.iterdir():
if child.is_file() and child.suffix in {".yaml", ".yml"}:
child.unlink()
route_files = []
for route in template_vars["routes"]:
stem = "" if template_vars["compatibility"] else f"-{route['name']}"
for kind in ("service", "ingress"):
out_name = f"{kind}{stem}.yaml"
destination = manifests_dir / out_name
destination.write_text(jinja_env.get_template(f"{kind}.yaml.j2").render(
**template_vars, route=route
))
route_files.append(out_name)
print(f" Rendered {kind}.yaml.j2 -> {destination}")
common_vars = {**template_vars, "route_files": route_files}
for out_name, destination in {
"certificate.yaml": manifests_dir / "certificate.yaml",
"kustomization.yaml": manifests_dir / "kustomization.yaml",
"app.yaml": app_dir / "app.yaml",
}.items():
destination.write_text(jinja_env.get_template(f"{out_name}.j2").render(**common_vars))
print(f" Rendered {out_name}.j2 -> {destination}")
def commit_and_push(apps_dir, message):
run(f"git -C {apps_dir} add -A")
result = subprocess.run(
f"git -C {apps_dir} diff --cached --quiet",
shell=True, check=False,
)
def git_auth_env(token):
"""Return credential-safe Git authentication shared by clone and push."""
auth_env = os.environ.copy()
auth_env["CI_BOT_TOKEN"] = token
auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
auth_env["GIT_TERMINAL_PROMPT"] = "0"
return auth_env
def commit_and_push(apps_dir, message, token=None):
run(["git", "-C", str(apps_dir), "add", "-A"])
result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False)
if result.returncode == 0:
print("No manifest changes to commit")
return False
run(f"git -C {apps_dir} commit -m '{message}'")
run(f"git -C {apps_dir} push")
run(["git", "-C", str(apps_dir), "commit", "-m", message])
push_env = git_auth_env(token) if token else None
run(["git", "-C", str(apps_dir), "push"], env=push_env)
print("Manifests pushed — ArgoCD will sync")
return True