feat(delivery): publish multiple surfaces
Test / test (pull_request) Failing after 5s

Authored-By: OpenAI (GPT-5) <noreply@openai.com>
This commit is contained in:
Dave Kowalski
2026-08-29 21:21:47 +00:00
parent f1f780f5a3
commit 099a48f2b5
13 changed files with 826 additions and 309 deletions
+5 -3
View File
@@ -34,11 +34,11 @@ def build_static(site_dir, cfg):
elif cfg["type"] == "hugo":
print(f"Building Hugo site from {src}")
run(f"hugo --source {src} --destination {html_dir}")
run(["hugo", "--source", str(src), "--destination", str(html_dir)])
elif cfg["type"] == "mkdocs":
print(f"Building MkDocs site from {src}")
run(f"cd {src} && mkdocs build -d {html_dir}")
run(["mkdocs", "build", "-d", str(html_dir)], cwd=src)
if cfg.get("tidy", True):
print("Running tidy on HTML files...")
@@ -56,7 +56,9 @@ def build_static(site_dir, cfg):
def cmd_build():
site_dir = Path(env("SITE_DIR"))
cfg = parse_site_yaml(site_dir)
site_repo = env("SITE_REPO", "")
site_name = site_repo.split("/", 1)[1] if "/" in site_repo else None
cfg = parse_site_yaml(site_dir, site_name)
if not cfg["enabled"]:
print("Site disabled — skipping build")
+158 -109
View File
@@ -1,71 +1,132 @@
"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""
"""Deploy phase — artifact publication and split-route manifest rendering."""
import fnmatch
import hashlib
import json
import mimetypes
import os
import shlex
import shutil
import tempfile
import subprocess
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import quote
from urllib.request import Request, urlopen
try:
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
except ModuleNotFoundError: # aws-cli v1's bundled botocore layout
from awscli.botocore.auth import S3SigV4Auth
from awscli.botocore.awsrequest import AWSRequest
from awscli.botocore.credentials import Credentials
from utils import (
DEFAULT_S3_ENDPOINT,
GITEA_HOST,
NAMESPACE,
clone_apps,
commit_and_push,
die,
env,
k8s_name,
parse_site_yaml,
render_templates,
run,
DEFAULT_S3_ENDPOINT, GARAGE_WEBSITE_HOST, NAMESPACE, clone_apps,
commit_and_push, die, env, k8s_name, parse_site_yaml, render_templates, run,
)
GARAGE_ADMIN_ENDPOINT = os.environ.get(
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
)
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903")
CACHE_CONTROL = "public, max-age=0, must-revalidate"
def _aws_env(artifact):
credentials = artifact["credentials"]
access = env(credentials["access_key_env"])
secret = env(credentials["secret_key_env"])
child = os.environ.copy()
child.update({"AWS_ACCESS_KEY_ID": access, "AWS_SECRET_ACCESS_KEY": secret,
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001")})
return child
def s3_sync(site_name, site_dir, excludes=None):
def _aws(endpoint, operation):
return ["aws", "--endpoint-url", endpoint, *operation]
def _excluded(relative, patterns):
return any(fnmatch.fnmatch(relative, pattern) for pattern in patterns)
def _conditional_put(endpoint, artifact, key, path, content_type, digest):
"""Atomically create one S3 object without exposing signing credentials."""
selectors = artifact["credentials"]
credentials = Credentials(env(selectors["access_key_env"]),
env(selectors["secret_key_env"]))
region = os.environ.get("AWS_DEFAULT_REGION", "sjc001")
url = f"{endpoint.rstrip('/')}/{quote(artifact['bucket'], safe='')}/{quote(key)}"
data = path.read_bytes()
request = AWSRequest(method="PUT", url=url, data=data, headers={
"Content-Type": content_type,
"Cache-Control": artifact["cache_control"],
"x-amz-meta-sha256": digest,
"If-None-Match": "*",
})
S3SigV4Auth(credentials, "s3", region).add_auth(request)
signed = Request(url, data=data, method="PUT", headers=dict(request.headers.items()))
try:
with urlopen(signed) as response:
response.read()
except HTTPError as error:
if error.code == 412:
die(f"immutable artifact {artifact['name']}: concurrent object creation: {key}; "
"rerun to verify identical content")
die(f"immutable artifact {artifact['name']}: upload failed for {key}: {error}")
def _immutable_sync(source, artifact, endpoint, excludes):
"""Publish write-once objects; identical retries are no-ops."""
aws_env = _aws_env(artifact)
bucket = artifact["bucket"]
for path in sorted(item for item in source.rglob("*") if item.is_file()):
relative = path.relative_to(source).as_posix()
if _excluded(relative, excludes):
continue
key = "/".join(part for part in (artifact.get("key_prefix"), relative) if part)
digest = hashlib.sha256(path.read_bytes()).hexdigest()
head = subprocess.run(
_aws(endpoint, ["s3api", "head-object", "--bucket", bucket, "--key", key]),
env=aws_env, check=False, text=True, capture_output=True)
if head.returncode == 0:
metadata = json.loads(head.stdout)
if (metadata.get("Metadata") or {}).get("sha256") != digest:
die(f"immutable artifact {artifact['name']}: object changed: {key}")
if metadata.get("CacheControl") != artifact["cache_control"]:
die(f"immutable artifact {artifact['name']}: cache metadata changed: {key}")
print(f" Immutable object unchanged: s3://{bucket}/{key}")
continue
missing = head.stderr.lower()
if not any(marker in missing for marker in ("404", "not found", "nosuchkey")):
die(f"immutable artifact {artifact['name']}: head-object failed for {key}: "
f"{head.stderr.strip()}")
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
_conditional_put(endpoint, artifact, key, path, content_type, digest)
def _replaceable_sync(source, artifact, endpoint, excludes):
aws_env = _aws_env(artifact)
prefix = artifact.get("key_prefix")
destination = f"s3://{artifact['bucket']}/{prefix + '/' if prefix else ''}"
exclude_args = [part for pattern in excludes for part in ("--exclude", pattern)]
common = ["--only-show-errors", "--cache-control", artifact["cache_control"],
*exclude_args]
run(_aws(endpoint, ["s3", "sync", f"{source}/", destination,
"--delete", *common]), env=aws_env)
run(_aws(endpoint, ["s3", "cp", f"{source}/", destination,
"--recursive", *common]), env=aws_env)
def publish_artifact(build_root, artifact, excludes=None):
endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
html_dir = site_dir / "build" / "html"
if not html_dir.exists():
die(f"build/html not found — did the build step run? ({html_dir})")
env("AWS_ACCESS_KEY_ID")
env("AWS_SECRET_ACCESS_KEY")
os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
# `excludes` are patterns (site.yaml `excludes:` list) that should never
# be uploaded *and* should never be deleted from the bucket — escape hatch
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
if excludes:
print(f"Excluding patterns: {excludes}")
print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
# `sync --delete` handles new/changed/orphaned files. `cp --recursive`
# then re-uploads everything to refresh metadata (cache-control,
# content-type) on objects sync skipped because nothing changed.
# Cost: a no-op deploy still re-uploads every byte. Sites here are
# small enough that that's free; correctness wins over throughput.
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
# so a fresh upload always carries the right MIME type.
run(
f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
f"--delete --only-show-errors "
f"--cache-control '{CACHE_CONTROL}' "
f"{exclude_flags}".rstrip()
)
print("Re-stamping metadata on all objects...")
run(
f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
f"--recursive --only-show-errors "
f"--cache-control '{CACHE_CONTROL}' "
f"{exclude_flags}".rstrip()
)
source = (build_root / artifact["source"]).resolve()
if not source.is_relative_to(build_root.resolve()) or not source.is_dir():
die(f"artifact {artifact['name']}: source directory not found: {source}")
excludes = excludes or []
print(f"Publishing {artifact['name']} from {source} to s3://{artifact['bucket']}")
if artifact["immutable"]:
_immutable_sync(source, artifact, endpoint, excludes)
else:
_replaceable_sync(source, artifact, endpoint, excludes)
def garage_admin(method, path, token, body=None):
@@ -74,92 +135,83 @@ def garage_admin(method, path, token, body=None):
headers = {"Authorization": f"Bearer {token}"}
if data is not None:
headers["Content-Type"] = "application/json"
req = Request(url, data=data, method=method, headers=headers)
with urlopen(req) as resp:
raw = resp.read()
request = Request(url, data=data, method=method, headers=headers)
with urlopen(request) as response:
raw = response.read()
return json.loads(raw) if raw else {}
def ensure_bucket_aliases(site_name, aliases, admin_token):
"""Add cfg['aliases'] as Garage globalAliases on the site bucket.
Idempotent: skips aliases already present. Never removes aliases not in
the desired set (safety — orphan removal is manual).
"""
"""Fail closed while reconciling legacy Garage global aliases."""
if not aliases:
return
if not admin_token:
print(" (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
return
die("GARAGE_ADMIN_TOKEN is required when aliases are configured")
try:
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={quote(site_name)}",
admin_token)
except (HTTPError, URLError) as e:
print(f" WARNING: bucket lookup failed: {e}")
return
except (HTTPError, URLError) as error:
die(f"bucket lookup failed: {error}")
bucket_id = info.get("id")
if not bucket_id:
die(f"bucket lookup returned no id for {site_name}")
existing = set(info.get("globalAliases") or [])
print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
for alias in aliases:
if alias in existing:
continue
print(f" Adding globalAlias: {alias}")
try:
if alias not in existing:
garage_admin("POST", "/v2/AddBucketAlias", admin_token,
{"bucketId": bucket_id, "globalAlias": alias})
except HTTPError as e:
body = e.read().decode(errors="replace") if hasattr(e, "read") else ""
print(f" ERROR adding alias {alias}: {e} {body}")
raise
def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
"""Always re-render manifests from current site.yaml. Templates own
domain + aliases, so changes propagate without manual edits."""
manifests_dir.mkdir(parents=True, exist_ok=True)
template_vars = {
if manifests_dir.exists():
shutil.rmtree(manifests_dir)
routes = []
for route in cfg["routes"]:
artifact = cfg["artifacts"][route["artifact"]]
routes.append({
**route,
"artifact_config": artifact,
"resource_name": k8s_name(site_name, route["artifact"]),
"backend_host": (GARAGE_WEBSITE_HOST if cfg["compatibility"] else
f"{artifact['bucket']}.web.sjc001.fritzlab.net"),
"pass_host_header": cfg["compatibility"],
})
render_templates(action_dir, {
"site": site_name,
"site_k8s": k8s_name(site_name),
"domain": cfg["domain"],
"aliases": cfg["aliases"],
"namespace": NAMESPACE,
"middlewares": cfg["middlewares"],
}
render_templates(action_dir, template_vars, app_dir, manifests_dir)
"website_host": GARAGE_WEBSITE_HOST,
"routes": routes,
}, app_dir, manifests_dir)
def deploy_static(site_name, site_dir, action_dir, token, cfg):
s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
build_root = (site_dir / "build" / "html").resolve()
if not build_root.is_dir():
die(f"build/html not found — did the build step run? ({build_root})")
for artifact in cfg["artifacts"].values():
publish_artifact(build_root, artifact, cfg.get("excludes"))
if cfg["compatibility"]:
ensure_bucket_aliases(site_name, cfg["aliases"],
os.environ.get("GARAGE_ADMIN_TOKEN"))
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
manifests_dir = app_dir / "manifests"
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
commit_and_push(apps_dir, f"Deploy {site_name}")
render_site_manifests(site_name, action_dir, app_dir, app_dir / "manifests", cfg)
commit_and_push(apps_dir, f"Deploy {site_name}", token)
def decommission(site_name, token):
"""Remove manifests from apps repo."""
user = env("CI_BOT_USER", "ci-bot")
with tempfile.TemporaryDirectory() as tmp:
apps_dir = Path(tmp)
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
site_path = apps_dir / "sjc001" / "websites" / site_name
if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove")
return
shutil.rmtree(site_path)
run(f"git -C {apps_dir} config user.name {user}")
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
commit_and_push(apps_dir, f"Decommission {site_name}")
apps_dir = clone_apps(token)
site_path = apps_dir / "sjc001" / "websites" / site_name
if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove")
return
shutil.rmtree(site_path)
commit_and_push(apps_dir, f"Decommission {site_name}", token)
print(f"Bucket {site_name} and its objects are NOT purged automatically.")
print(f" garage bucket delete {site_name} --yes")
def cmd_deploy():
@@ -168,12 +220,9 @@ def cmd_deploy():
action_dir = Path(env("ACTION_DIR"))
token = env("CI_BOT_TOKEN")
site_name = site_repo.split("/", 1)[1]
cfg = parse_site_yaml(site_dir)
cfg = parse_site_yaml(site_dir, site_name)
if not cfg["enabled"]:
print("Site disabled — running decommission...")
decommission(site_name, token)
return
deploy_static(site_name, site_dir, action_dir, token, cfg)
+3 -2
View File
@@ -24,9 +24,10 @@ def ensure_aws():
subprocess.run(["aws", "--version"], check=True)
def ensure_jinja2():
def ensure_python_dependencies():
try:
import jinja2
import yaml
except ImportError:
print("Installing jinja2 + pyyaml...")
subprocess.run(
@@ -36,6 +37,6 @@ def ensure_jinja2():
if __name__ == "__main__":
ensure_jinja2()
ensure_python_dependencies()
ensure_aws()
print("Setup complete")
+279 -90
View File
@@ -1,154 +1,343 @@
"""Shared utilities for the site-publish action."""
import hashlib
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
import tempfile
from contextlib import contextmanager
from pathlib import Path, PurePosixPath
import yaml
from jinja2 import Environment, FileSystemLoader
from jinja2 import Environment, FileSystemLoader, StrictUndefined
APPS_REPO = "fritzlab/apps"
GITEA_HOST = "code.fritzlab.net"
NAMESPACE = "websites"
DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
GARAGE_WEBSITE_HOST = "garage-s3.storage.svc.k8s.sjc001.fritzlab.net"
EXCLUDE_FILES = {
".git", ".gitea", ".gitignore", "site.yaml",
"build", "Makefile", "README.md", "CLAUDE.md",
"Dockerfile", ".dockerignore", "go.mod", "go.sum",
".git", ".gitea", ".gitignore", "site.yaml", "build", "Makefile",
"README.md", "CLAUDE.md", "Dockerfile", ".dockerignore", "go.mod", "go.sum",
}
VALID_TYPES = {"static", "hugo", "mkdocs"}
CACHE_POLICIES = {
"immutable-release": ("public, max-age=31536000, immutable", True),
"revalidated-channel": ("public, max-age=0, must-revalidate", False),
"private": ("private, no-store", False),
}
ENV_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$")
BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$")
HOST_RE = BUCKET_RE
MIDDLEWARE_RE = NAME_RE
DOCKER_DEPRECATION_MSG = """\
type: docker is no longer supported by action/site-publish.
site-publish handles only static-content sites (static, hugo, mkdocs)
that ship to Garage S3. For containerized web apps, use the standard
image-producer chain:
- uses: action/image-build@v1 # build + smoke-test
- uses: action/image-push@v1 # push + prune
- uses: action/image-deploy@v1 # apps repo image-pin
Hand-author your apps-repo manifests once (Deployment, Service, Ingress,
Certificate, kustomization with images: block) under
sjc001/websites/<repo>/manifests/. image-deploy will pin the tag on
every CI run. See action/image-deploy README and
sjc001/websites/rainsounds.vino.network/manifests/ for the canonical
example.\
site-publish handles only static-content sites (static, hugo, mkdocs).
Use action/image-build, action/image-push, and action/image-deploy for images.\
"""
def k8s_name(name):
"""Sanitize for DNS-1035 label (dots → dashes)."""
return name.replace(".", "-")
def k8s_name(*parts):
raw = re.sub(r"[^a-z0-9-]", "-", "-".join(parts).replace(".", "-").lower())
raw = raw.strip("-")
if len(raw) <= 63:
return raw
return f"{raw[:54].rstrip('-')}-{hashlib.sha256(raw.encode()).hexdigest()[:8]}"
def env(key, default=None):
val = os.environ.get(key, default)
if val is None:
value = os.environ.get(key, default)
if value is None or value == "":
die(f"Missing required env var: {key}")
return val
return value
def die(msg):
print(f"ERROR: {msg}", file=sys.stderr)
sys.exit(1)
def die(message):
print(f"ERROR: {message}", file=sys.stderr)
raise SystemExit(1)
def run(cmd, **kwargs):
print(f" $ {cmd}")
return subprocess.run(cmd, shell=True, check=True, **kwargs)
def run(cmd, *, display=None, **kwargs):
"""Run argv without a shell and print only a credential-free display."""
if isinstance(cmd, str):
raise TypeError("run() requires an argv sequence")
print(f" $ {display or ' '.join(str(part) for part in cmd)}")
return subprocess.run(cmd, check=True, **kwargs)
def parse_site_yaml(site_dir):
path = Path(site_dir) / "site.yaml"
if not path.exists():
die("site.yaml not found in repo root")
def _string_list(cfg, key):
value = cfg.get(key) or []
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
die(f"{key} must be a list of strings")
return value
with open(path) as f:
cfg = yaml.safe_load(f)
if not cfg.get("domain"):
die("domain is required in site.yaml")
def _common(cfg):
domain = cfg.get("domain")
if not isinstance(domain, str) or not HOST_RE.fullmatch(domain):
die("domain must be a lowercase hostname")
aliases = _string_list(cfg, "aliases")
if any(not HOST_RE.fullmatch(alias) for alias in aliases):
die("aliases must contain lowercase hostnames")
site_type = cfg.get("type", "static")
if site_type == "docker":
die(DOCKER_DEPRECATION_MSG)
if site_type not in VALID_TYPES:
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
excludes = cfg.get("excludes") or []
if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes):
die("excludes must be a list of string patterns")
middlewares = cfg.get("middlewares") or []
if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares):
die("middlewares must be a list of Traefik file-provider middleware names")
site = {
"domain": cfg["domain"],
middlewares = _string_list(cfg, "middlewares")
if any(not MIDDLEWARE_RE.fullmatch(item) for item in middlewares):
die("middlewares must contain Traefik file-provider names")
if not isinstance(cfg.get("enabled", True), bool):
die("enabled must be a boolean")
if not isinstance(cfg.get("tidy", True), bool):
die("tidy must be a boolean")
content_dir = cfg.get("content_dir", "")
content_path = PurePosixPath(content_dir)
if not isinstance(content_dir, str) or content_path.is_absolute() or ".." in content_path.parts:
die("content_dir must be a relative path inside the repository")
return {
"domain": domain,
"type": site_type,
"enabled": cfg.get("enabled", True),
"aliases": cfg.get("aliases") or [],
"content_dir": cfg.get("content_dir", ""),
"aliases": aliases,
"content_dir": content_dir,
"tidy": cfg.get("tidy", True),
"excludes": excludes,
"excludes": _string_list(cfg, "excludes"),
"middlewares": middlewares,
}
print("Site config:")
for k, v in site.items():
print(f" {k}: {v}")
def _source(value, name):
if not isinstance(value, str) or not value:
die(f"artifact {name}: source must be a non-empty relative path")
path = PurePosixPath(value)
if path.is_absolute() or ".." in path.parts:
die(f"artifact {name}: source must stay under build/html")
return value
def _v2(cfg, site):
if cfg.get("schema") != "v2":
die("multi-surface configs must set schema: v2")
raw_artifacts, raw_routes = cfg.get("artifacts"), cfg.get("routes")
if not isinstance(raw_artifacts, dict) or not raw_artifacts:
die("artifacts must be a non-empty mapping")
if not isinstance(raw_routes, list) or not raw_routes:
die("routes must be a non-empty list")
artifacts = {}
for name, raw in raw_artifacts.items():
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
die("artifact names must be lowercase kebab-case")
if not isinstance(raw, dict):
die(f"artifact {name}: definition must be a mapping")
unknown = set(raw) - {"source", "bucket", "credentials", "cache"}
if unknown:
die(f"artifact {name}: unknown keys: {', '.join(sorted(unknown))}")
bucket = raw.get("bucket")
if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket):
die(f"artifact {name}: bucket must be a lowercase Garage bucket name")
credentials = raw.get("credentials")
if not isinstance(credentials, dict) or set(credentials) != {
"access_key_env", "secret_key_env"}:
die(f"artifact {name}: credentials require access_key_env and secret_key_env")
if any(not isinstance(value, str) or not ENV_RE.fullmatch(value)
for value in credentials.values()):
die(f"artifact {name}: credential selectors must be environment variable names")
cache = raw.get("cache")
if cache not in CACHE_POLICIES:
die(f"artifact {name}: unknown cache policy {cache!r}")
cache_control, immutable = CACHE_POLICIES[cache]
artifacts[name] = {
"name": name, "source": _source(raw.get("source"), name),
"bucket": bucket, "credentials": credentials, "cache": cache,
"cache_control": cache_control, "immutable": immutable,
}
routes, paths, used = [], set(), set()
buckets, credentials = {}, {}
for raw in raw_routes:
if not isinstance(raw, dict):
die("each route must be a mapping")
unknown = set(raw) - {"path", "artifact", "access", "middlewares"}
if unknown:
die(f"route has unknown keys: {', '.join(sorted(unknown))}")
path = raw.get("path")
if (not isinstance(path, str) or not path.startswith("/") or "//" in path
or (path != "/" and path.endswith("/"))):
die("route paths must be normalized absolute prefixes")
if path in paths:
die(f"duplicate route path: {path}")
paths.add(path)
artifact_name = raw.get("artifact")
if artifact_name not in artifacts:
die(f"route {path}: unknown artifact {artifact_name!r}")
if artifact_name in used:
die(f"artifact {artifact_name} may be routed only once")
used.add(artifact_name)
access = raw.get("access")
if access not in {"public", "authenticated"}:
die(f"route {path}: access must be public or authenticated")
if path == "/" and access == "public":
die("public catch-all route / is forbidden in schema v2")
middlewares = raw.get("middlewares") or []
if not isinstance(middlewares, list) or any(
not isinstance(item, str) or not MIDDLEWARE_RE.fullmatch(item)
for item in middlewares):
die(f"route {path}: middlewares must contain file-provider names")
if access == "authenticated" and not middlewares:
die(f"route {path}: authenticated access requires middleware")
artifact = artifacts[artifact_name]
if access == "authenticated" and artifact["cache"] != "private":
die(f"route {path}: authenticated artifacts must use private cache")
if access == "public" and artifact["cache"] == "private":
die(f"route {path}: public artifacts cannot use private cache")
prior_access = buckets.setdefault(artifact["bucket"], access)
if prior_access != access:
die(f"route {path}: bucket is reused across access classes")
credential_pair = tuple(artifact["credentials"].values())
prior_artifact = credentials.setdefault(credential_pair, artifact_name)
if prior_artifact != artifact_name:
die(f"route {path}: publication credentials are reused by artifacts "
f"{prior_artifact} and {artifact_name}")
artifact["access"] = access
routes.append({"path": path, "artifact": artifact_name,
"access": access, "middlewares": middlewares})
artifact["key_prefix"] = path.lstrip("/")
missing = set(artifacts) - used
if missing:
die(f"unrouted artifacts: {', '.join(sorted(missing))}")
if "/" not in paths:
die("schema v2 requires an explicit / access policy")
artifact_list = list(artifacts.values())
for index, left in enumerate(artifact_list):
left_parts = PurePosixPath(left["source"]).parts
for right in artifact_list[index + 1:]:
right_parts = PurePosixPath(right["source"]).parts
overlaps = (left_parts == right_parts[:len(left_parts)] or
right_parts == left_parts[:len(right_parts)])
if overlaps and left["access"] != right["access"]:
die(f"artifact sources overlap across access classes: "
f"{left['name']} and {right['name']}")
site.update({"schema": "v2", "compatibility": False, "artifacts": artifacts})
site["routes"] = sorted(routes, key=lambda route: (-len(route["path"]), route["path"]))
return site
def parse_site_yaml(site_dir, site_name=None):
path = Path(site_dir) / "site.yaml"
if not path.exists():
die("site.yaml not found in repo root")
with open(path, encoding="utf-8") as handle:
cfg = yaml.safe_load(handle)
if not isinstance(cfg, dict):
die("site.yaml must contain a mapping")
allowed = {"domain", "type", "enabled", "aliases", "content_dir", "tidy",
"excludes", "middlewares", "schema", "artifacts", "routes"}
unknown = set(cfg) - allowed
if unknown:
die(f"unknown site.yaml keys: {', '.join(sorted(unknown))}")
site = _common(cfg)
has_v2 = any(key in cfg for key in ("schema", "artifacts", "routes"))
if has_v2:
if not all(key in cfg for key in ("schema", "artifacts", "routes")):
die("schema, artifacts, and routes must be declared together")
if site["middlewares"]:
die("schema v2 middlewares belong on individual routes")
site = _v2(cfg, site)
else:
site_name = site_name or site["domain"]
cache_control, immutable = CACHE_POLICIES["revalidated-channel"]
site.update({
"schema": "single-surface-v1", "compatibility": True,
"artifacts": {"site": {
"name": "site", "source": ".", "bucket": site_name,
"credentials": {"access_key_env": "AWS_ACCESS_KEY_ID",
"secret_key_env": "AWS_SECRET_ACCESS_KEY"},
"cache": "revalidated-channel", "cache_control": cache_control,
"immutable": immutable, "key_prefix": "",
}},
"routes": [{"path": "/", "artifact": "site", "access": "legacy",
"middlewares": site["middlewares"]}],
})
print(f"Site config: {site['schema']} {site['domain']} ({len(site['routes'])} route(s))")
return site
@contextmanager
def git_auth(token, user):
"""Provide an HTTPS token through an inherited FD, never argv or output."""
read_fd, write_fd = os.pipe()
with tempfile.TemporaryDirectory() as temp_dir:
askpass = Path(temp_dir) / "askpass.sh"
askpass.write_text(
"#!/bin/sh\ncase \"$1\" in\n"
" *Username*) printf '%s\\n' \"$SITE_PUBLISH_GIT_USER\" ;;\n"
f" *Password*) cat <&{read_fd} ;;\nesac\n", encoding="utf-8")
askpass.chmod(0o700)
os.write(write_fd, token.encode())
os.close(write_fd)
child_env = os.environ.copy()
child_env.update({"GIT_ASKPASS": str(askpass), "GIT_TERMINAL_PROMPT": "0",
"SITE_PUBLISH_GIT_USER": user})
try:
yield {"env": child_env, "pass_fds": (read_fd,)}
finally:
os.close(read_fd)
def clone_apps(token):
user = env("CI_BOT_USER", "ci-bot")
apps_dir = Path("/tmp/apps-deploy")
if apps_dir.exists():
shutil.rmtree(apps_dir)
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}")
run(f"git -C {apps_dir} config user.name {user}")
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
url = f"https://{GITEA_HOST}/{APPS_REPO}.git"
with git_auth(token, user) as auth:
run(["git", "clone", "--depth", "1", url, str(apps_dir)], **auth)
run(["git", "-C", str(apps_dir), "config", "user.name", user])
run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
return apps_dir
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
"""Render Jinja2 templates for a static-content site."""
templates_dir = Path(action_dir) / "templates"
jinja_env = Environment(
loader=FileSystemLoader(str(templates_dir)),
keep_trailing_newline=True,
)
tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2",
"kustomization.yaml.j2", "service.yaml.j2"]
for tmpl_name in tmpl_names:
tmpl = jinja_env.get_template(tmpl_name)
rendered = tmpl.render(**template_vars)
out_name = tmpl_name.replace(".j2", "")
dest = app_dir / out_name if tmpl_name == "app.yaml.j2" else manifests_dir / out_name
dest.write_text(rendered)
print(f" Rendered {tmpl_name} -> {dest}")
templates = Path(action_dir) / "templates"
jinja = Environment(loader=FileSystemLoader(str(templates)),
keep_trailing_newline=True, undefined=StrictUndefined)
app_dir.mkdir(parents=True, exist_ok=True)
manifests_dir.mkdir(parents=True, exist_ok=True)
(app_dir / "app.yaml").write_text(
jinja.get_template("app.yaml.j2").render(**template_vars), encoding="utf-8")
(manifests_dir / "certificate.yaml").write_text(
jinja.get_template("certificate.yaml.j2").render(**template_vars), encoding="utf-8")
resources = ["certificate.yaml"]
for route in template_vars["routes"]:
values = {**template_vars, "route": route}
for kind in ("service", "ingress"):
filename = f"{kind}-{route['resource_name']}.yaml"
(manifests_dir / filename).write_text(
jinja.get_template(f"{kind}.yaml.j2").render(**values), encoding="utf-8")
resources.append(filename)
(manifests_dir / "kustomization.yaml").write_text(
jinja.get_template("kustomization.yaml.j2").render(
**template_vars, resources=resources), encoding="utf-8")
def commit_and_push(apps_dir, message):
run(f"git -C {apps_dir} add -A")
result = subprocess.run(
f"git -C {apps_dir} diff --cached --quiet",
shell=True, check=False,
)
if result.returncode == 0:
def commit_and_push(apps_dir, message, token):
user = env("CI_BOT_USER", "ci-bot")
run(["git", "-C", str(apps_dir), "add", "-A"])
clean = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"],
check=False).returncode == 0
if clean:
print("No manifest changes to commit")
return False
run(f"git -C {apps_dir} commit -m '{message}'")
run(f"git -C {apps_dir} push")
run(["git", "-C", str(apps_dir), "commit", "-m", message])
with git_auth(token, user) as auth:
run(["git", "-C", str(apps_dir), "push"], **auth)
print("Manifests pushed — ArgoCD will sync")
return True