feat: publish isolated site artifacts
Test / test (pull_request) Successful in 5s

This commit is contained in:
Evelyn Chen
2026-08-29 21:28:28 +00:00
parent f1f780f5a3
commit 7aca96e320
10 changed files with 1032 additions and 109 deletions
+57 -4
View File
@@ -5,7 +5,7 @@ import subprocess
import tempfile
from pathlib import Path
from utils import EXCLUDE_FILES, env, parse_site_yaml, run
from utils import EXCLUDE_FILES, env, parse_site_yaml, run_args
def build_static(site_dir, cfg):
@@ -34,11 +34,11 @@ def build_static(site_dir, cfg):
elif cfg["type"] == "hugo":
print(f"Building Hugo site from {src}")
run(f"hugo --source {src} --destination {html_dir}")
run_args(["hugo", "--source", str(src), "--destination", str(html_dir)])
elif cfg["type"] == "mkdocs":
print(f"Building MkDocs site from {src}")
run(f"cd {src} && mkdocs build -d {html_dir}")
run_args(["mkdocs", "build", "-d", str(html_dir)], cwd=src)
if cfg.get("tidy", True):
print("Running tidy on HTML files...")
@@ -54,6 +54,56 @@ def build_static(site_dir, cfg):
print(f"Build complete — content at {html_dir}")
def stage_artifacts(site_dir, cfg):
"""Snapshot caller-built outputs for the deploy phase."""
staging_dir = site_dir / ".site-publish"
if staging_dir.is_symlink():
raise SystemExit("ERROR: .site-publish must not be a symbolic link")
if staging_dir.exists():
shutil.rmtree(staging_dir)
staging_dir.mkdir()
root = site_dir.resolve()
for name, artifact in cfg["artifacts"].items():
source = (site_dir / artifact["source"]).resolve()
try:
source.relative_to(root)
except ValueError:
raise SystemExit(f"ERROR: artifact {name}.source escapes the repository")
if not source.is_dir():
raise SystemExit(f"ERROR: artifact {name}.source is not a directory: {artifact['source']}")
files = [entry for entry in source.rglob("*") if entry.is_file()]
if not files:
raise SystemExit(f"ERROR: artifact {name}.source has no files: {artifact['source']}")
for entry in source.rglob("*"):
if entry.is_symlink():
try:
entry.resolve(strict=True).relative_to(root)
except (FileNotFoundError, ValueError):
raise SystemExit(
f"ERROR: artifact {name}.source contains an unsafe symlink: {entry.relative_to(source)}"
)
destination = staging_dir / name
destination.mkdir()
routes = [route for route in cfg["routes"] if route["artifact"] == name]
targets = set()
print(f"Staging artifact {name} from {artifact['source']}")
for route in routes:
route_root = destination / route["path"].lstrip("/")
for entry in files:
relative = entry.relative_to(source)
target = route_root / relative
if target in targets:
raise SystemExit(
f"ERROR: artifact {name} routes map more than once to {target.relative_to(destination)}"
)
targets.add(target)
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(entry, target, follow_symlinks=True)
print(f"Build complete — staged artifacts at {staging_dir}")
def cmd_build():
site_dir = Path(env("SITE_DIR"))
cfg = parse_site_yaml(site_dir)
@@ -62,4 +112,7 @@ def cmd_build():
print("Site disabled — skipping build")
return
build_static(site_dir, cfg)
if cfg["mode"] == "multi":
stage_artifacts(site_dir, cfg)
else:
build_static(site_dir, cfg)
+197 -40
View File
@@ -2,16 +2,15 @@
import json
import os
import shlex
import shutil
import tempfile
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import Request, urlopen
from utils import (
DEFAULT_S3_ENDPOINT,
GITEA_HOST,
NAMESPACE,
clone_apps,
commit_and_push,
@@ -20,7 +19,7 @@ from utils import (
k8s_name,
parse_site_yaml,
render_templates,
run,
run_args,
)
GARAGE_ADMIN_ENDPOINT = os.environ.get(
@@ -31,21 +30,66 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
CACHE_CONTROL = "public, max-age=0, must-revalidate"
def s3_sync(site_name, site_dir, excludes=None):
endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
html_dir = site_dir / "build" / "html"
if not html_dir.exists():
die(f"build/html not found — did the build step run? ({html_dir})")
env("AWS_ACCESS_KEY_ID")
env("AWS_SECRET_ACCESS_KEY")
os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
def credential_environment(profile):
if profile == "default":
access_key = env("AWS_ACCESS_KEY_ID")
secret_key = env("AWS_SECRET_ACCESS_KEY")
else:
prefix = f"SITE_PUBLISH_{profile.upper().replace('-', '_')}"
access_key = env(f"{prefix}_S3_ACCESS_KEY_ID")
secret_key = env(f"{prefix}_S3_SECRET_ACCESS_KEY")
child_env = os.environ.copy()
child_env["AWS_ACCESS_KEY_ID"] = access_key
child_env["AWS_SECRET_ACCESS_KEY"] = secret_key
child_env.setdefault("AWS_DEFAULT_REGION", "sjc001")
return child_env
def configure_cors(bucket, origins, endpoint, child_env):
if origins is None:
return
if not origins:
run_args(
["aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
"--bucket", bucket],
env=child_env,
)
return
config = {
"CORSRules": [{
"AllowedOrigins": origins,
"AllowedMethods": ["GET", "HEAD"],
"AllowedHeaders": ["*"],
"ExposeHeaders": ["ETag"],
"MaxAgeSeconds": 3600,
}]
}
with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
json.dump(config, handle)
handle.flush()
run_args(
["aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
"--bucket", bucket, "--cors-configuration", f"file://{handle.name}"],
env=child_env,
)
def s3_sync(bucket, source, *, credential="default", endpoint=None,
cache=None, cors_origins=None, excludes=None):
endpoint = endpoint or os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
if not source.exists():
die(f"staged artifact not found — did the build step run? ({source})")
child_env = credential_environment(credential)
cache = cache or {"default": CACHE_CONTROL, "rules": []}
# `excludes` are patterns (site.yaml `excludes:` list) that should never
# be uploaded *and* should never be deleted from the bucket — escape hatch
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
exclude_args = []
for pattern in excludes or []:
exclude_args.extend(["--exclude", pattern])
if excludes:
print(f"Excluding patterns: {excludes}")
print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
print(f"Syncing {source} → s3://{bucket} via {endpoint}")
# `sync --delete` handles new/changed/orphaned files. `cp --recursive`
# then re-uploads everything to refresh metadata (cache-control,
# content-type) on objects sync skipped because nothing changed.
@@ -53,22 +97,63 @@ def s3_sync(site_name, site_dir, excludes=None):
# small enough that that's free; correctness wins over throughput.
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
# so a fresh upload always carries the right MIME type.
run(
f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
f"--delete --only-show-errors "
f"--cache-control '{CACHE_CONTROL}' "
f"{exclude_flags}".rstrip()
run_args(
["aws", "--endpoint-url", endpoint, "s3", "sync", f"{source}/", f"s3://{bucket}/",
"--delete", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
env=child_env,
)
print("Re-stamping metadata on all objects...")
run(
f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
f"--recursive --only-show-errors "
f"--cache-control '{CACHE_CONTROL}' "
f"{exclude_flags}".rstrip()
run_args(
["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
"--recursive", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
env=child_env,
)
for rule in cache["rules"]:
print(f"Applying cache metadata for {rule['match']}")
run_args(
["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
"--recursive", "--only-show-errors", "--exclude", "*",
"--include", rule["match"], "--cache-control", rule["value"], *exclude_args],
env=child_env,
)
configure_cors(bucket, cors_origins, endpoint, child_env)
def preflight_artifacts(site_dir, cfg):
"""Validate every source, credential, and bucket before the first write."""
for artifact in cfg["artifacts"].values():
source = site_dir / ".site-publish" / artifact["name"]
if not source.is_dir() or not any(entry.is_file() for entry in source.rglob("*")):
die(f"staged artifact is absent or empty: {source}")
child_env = credential_environment(artifact["credential"])
endpoint = artifact["endpoint"] or os.environ.get(
"GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT
)
run_args(
["aws", "--endpoint-url", endpoint, "s3api", "head-bucket",
"--bucket", artifact["bucket"]],
env=child_env,
)
def routed_cache(cache, routes):
"""Translate source-relative cache patterns to their public object keys."""
rules = []
for route in routes:
prefix = route["path"].lstrip("/")
for rule in cache["rules"]:
pattern = f"{prefix}/{rule['match']}" if prefix else rule["match"]
rules.append({"match": pattern, "value": rule["value"]})
return {"default": cache["default"], "rules": rules}
def garage_admin(method, path, token, body=None):
parsed = urlsplit(GARAGE_ADMIN_ENDPOINT)
if (
parsed.scheme not in {"http", "https"} or not parsed.netloc
or parsed.username or parsed.password or parsed.query or parsed.fragment
):
die("GARAGE_ADMIN_ENDPOINT must be an HTTP URL without credentials")
url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
data = json.dumps(body).encode() if body is not None else None
headers = {"Authorization": f"Bearer {token}"}
@@ -89,17 +174,17 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
if not aliases:
return
if not admin_token:
print(" (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
return
die("GARAGE_ADMIN_TOKEN is required when aliases are configured")
try:
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
admin_token)
except (HTTPError, URLError) as e:
print(f" WARNING: bucket lookup failed: {e}")
return
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
bucket_id = info.get("id")
if not isinstance(bucket_id, str) or not bucket_id:
die(f"bucket lookup for {site_name} returned no bucket id")
existing = set(info.get("globalAliases") or [])
print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
@@ -119,21 +204,92 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
"""Always re-render manifests from current site.yaml. Templates own
domain + aliases, so changes propagate without manual edits."""
manifests_dir.mkdir(parents=True, exist_ok=True)
if manifests_dir.exists():
shutil.rmtree(manifests_dir)
manifests_dir.mkdir(parents=True)
site_k8s = k8s_name(site_name)
if cfg["mode"] == "legacy":
artifacts = [{
"name": "site",
"bucket": site_name,
"service_name": site_k8s,
"external_name": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
"virtual_host": False,
}]
routes = [{
"name": "site",
"path": "/",
"artifact": "site",
"service_name": site_k8s,
"ingress_name": site_k8s,
"middlewares": cfg["middlewares"],
"middleware_refs": [f"{name}@file" for name in cfg["middlewares"]],
}]
else:
artifacts = []
artifact_by_name = {}
for name, artifact in cfg["artifacts"].items():
service_name = k8s_name(f"{site_name}-{name}")
view = {
**artifact,
"service_name": service_name,
"external_name": f"{artifact['bucket']}.web.sjc001.fritzlab.net",
"virtual_host": True,
}
artifacts.append(view)
artifact_by_name[name] = view
routes = []
for route in cfg["routes"]:
artifact = artifact_by_name[route["artifact"]]
ingress_name = k8s_name(f"{site_name}-{route['name']}")
middleware_refs = [f"{name}@file" for name in route["middlewares"]]
routes.append({
**route,
"service_name": artifact["service_name"],
"ingress_name": ingress_name,
"middleware_refs": middleware_refs,
})
template_vars = {
"site": site_name,
"site_k8s": k8s_name(site_name),
"site_k8s": site_k8s,
"domain": cfg["domain"],
"aliases": cfg["aliases"],
"namespace": NAMESPACE,
"middlewares": cfg["middlewares"],
"artifacts": artifacts,
"routes": routes,
"modern": cfg["mode"] == "multi",
}
render_templates(action_dir, template_vars, app_dir, manifests_dir)
def deploy_static(site_name, site_dir, action_dir, token, cfg):
s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
if cfg["mode"] == "legacy":
s3_sync(
site_name,
site_dir / "build" / "html",
excludes=cfg.get("excludes"),
)
ensure_bucket_aliases(
site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")
)
else:
preflight_artifacts(site_dir, cfg)
for artifact in cfg["artifacts"].values():
routes = [
route for route in cfg["routes"]
if route["artifact"] == artifact["name"]
]
s3_sync(
artifact["bucket"],
site_dir / ".site-publish" / artifact["name"],
credential=artifact["credential"],
endpoint=artifact["endpoint"],
cache=routed_cache(artifact["cache"], routes),
cors_origins=artifact["cors_origins"],
excludes=artifact["excludes"],
)
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
@@ -141,23 +297,24 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
commit_and_push(apps_dir, f"Deploy {site_name}")
try:
commit_and_push(apps_dir, f"Deploy {site_name}", token)
finally:
shutil.rmtree(apps_dir.parent, ignore_errors=True)
def decommission(site_name, token):
"""Remove manifests from apps repo."""
user = env("CI_BOT_USER", "ci-bot")
with tempfile.TemporaryDirectory() as tmp:
apps_dir = Path(tmp)
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
apps_dir = clone_apps(token)
try:
site_path = apps_dir / "sjc001" / "websites" / site_name
if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove")
return
shutil.rmtree(site_path)
run(f"git -C {apps_dir} config user.name {user}")
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
commit_and_push(apps_dir, f"Decommission {site_name}")
commit_and_push(apps_dir, f"Decommission {site_name}", token)
finally:
shutil.rmtree(apps_dir.parent, ignore_errors=True)
print(f"Bucket {site_name} and its objects are NOT purged automatically.")
print(f" garage bucket delete {site_name} --yes")
+3 -2
View File
@@ -24,9 +24,10 @@ def ensure_aws():
subprocess.run(["aws", "--version"], check=True)
def ensure_jinja2():
def ensure_python_modules():
try:
import jinja2
import yaml
except ImportError:
print("Installing jinja2 + pyyaml...")
subprocess.run(
@@ -36,6 +37,6 @@ def ensure_jinja2():
if __name__ == "__main__":
ensure_jinja2()
ensure_python_modules()
ensure_aws()
print("Setup complete")
+369 -32
View File
@@ -1,13 +1,18 @@
"""Shared utilities for the site-publish action."""
import hashlib
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from urllib.parse import urlsplit
import yaml
from jinja2 import Environment, FileSystemLoader
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from yaml import YAMLError
APPS_REPO = "fritzlab/apps"
GITEA_HOST = "code.fritzlab.net"
@@ -16,11 +21,15 @@ DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
EXCLUDE_FILES = {
".git", ".gitea", ".gitignore", "site.yaml",
"build", "Makefile", "README.md", "CLAUDE.md",
".site-publish", "build", "Makefile", "README.md", "CLAUDE.md",
"Dockerfile", ".dockerignore", "go.mod", "go.sum",
}
VALID_TYPES = {"static", "hugo", "mkdocs"}
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
PROFILE_RE = re.compile(r"^[a-z][a-z0-9-]*$")
AUTH_MIDDLEWARE = "authentik-forwardauth"
DOCKER_DEPRECATION_MSG = """\
type: docker is no longer supported by action/site-publish.
@@ -43,13 +52,18 @@ example.\
def k8s_name(name):
"""Sanitize for DNS-1035 label (dots → dashes)."""
return name.replace(".", "-")
"""Return a stable DNS-1035 label, including for long host names."""
normalized = re.sub(r"[^a-z0-9-]", "-", name.lower()).strip("-")
normalized = re.sub(r"-+", "-", normalized)
if len(normalized) <= 63:
return normalized
digest = hashlib.sha256(normalized.encode()).hexdigest()[:8]
return f"{normalized[:54].rstrip('-')}-{digest}"
def env(key, default=None):
val = os.environ.get(key, default)
if val is None:
if val is None or val == "":
die(f"Missing required env var: {key}")
return val
@@ -59,9 +73,258 @@ def die(msg):
sys.exit(1)
def run(cmd, **kwargs):
print(f" $ {cmd}")
return subprocess.run(cmd, shell=True, check=True, **kwargs)
def run_args(args, *, display=None, **kwargs):
"""Run an argv vector without shell interpolation or credential logging."""
print(f" $ {display or ' '.join(args)}")
return subprocess.run(args, check=True, **kwargs)
def _string_list(cfg, key):
value = cfg.get(key) or []
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
die(f"{key} must be a list of strings")
return value
def _patterns(cfg, key, field):
values = _string_list(cfg, key)
if any(not value or "\n" in value or "\r" in value for value in values):
die(f"{field} must contain non-empty single-line patterns")
return values
def _cors_origins(raw, artifact_name):
origins = _string_list(raw, "cors_origins")
for origin in origins:
if origin == "*":
continue
parsed = urlsplit(origin)
if (
parsed.scheme != "https" or not parsed.netloc or parsed.path not in {"", "/"}
or parsed.query or parsed.fragment or parsed.username or parsed.password
):
die(f"artifact {artifact_name}.cors_origins must contain * or HTTPS origins")
return origins
def _validate_domain(value, field):
if not isinstance(value, str) or not value or len(value) > 253:
die(f"{field} must be a DNS name")
labels = value.rstrip(".").split(".")
if any(not NAME_RE.fullmatch(label) for label in labels):
die(f"{field} must be a DNS name")
return value.rstrip(".")
def _validate_middlewares(value, field):
if not isinstance(value, list) or any(
not isinstance(item, str) or not MIDDLEWARE_RE.fullmatch(item)
for item in value
):
die(f"{field} must contain file-provider middleware names")
return value
def _known_keys(value, allowed, field):
unknown = sorted(set(value) - set(allowed))
if unknown:
die(f"{field} has unknown fields: {', '.join(unknown)}")
def _cache_header(value, field):
if not isinstance(value, str) or not value or "\n" in value or "\r" in value:
die(f"{field} must be one Cache-Control header value")
return value
def _cache_config(raw, artifact_name):
cache = raw.get("cache") or {}
if not isinstance(cache, dict):
die(f"artifact {artifact_name}.cache must be a mapping")
_known_keys(cache, {"default", "rules"}, f"artifact {artifact_name}.cache")
default = _cache_header(
cache.get("default", "public, max-age=0, must-revalidate"),
f"artifact {artifact_name}.cache.default",
)
rules_raw = cache.get("rules") or []
if not isinstance(rules_raw, list):
die(f"artifact {artifact_name}.cache.rules must be a list")
rules = []
patterns = set()
for index, rule in enumerate(rules_raw):
field = f"artifact {artifact_name}.cache.rules[{index}]"
if not isinstance(rule, dict):
die(f"{field} must be a mapping")
_known_keys(rule, {"match", "value"}, field)
pattern = rule.get("match")
if (
not isinstance(pattern, str) or not pattern or pattern.startswith("/")
or "\n" in pattern or "\r" in pattern or ".." in Path(pattern).parts
):
die(f"{field}.match must be a relative aws-cli include pattern")
if pattern in patterns:
die(f"artifact {artifact_name} repeats cache match {pattern}")
patterns.add(pattern)
rules.append({
"match": pattern,
"value": _cache_header(rule.get("value"), f"{field}.value"),
})
return {"default": default, "rules": rules}
def _cache_directives(value):
return {part.strip().lower().split("=", 1)[0] for part in value.split(",")}
def _modern_config(cfg, domain, aliases):
artifacts_cfg = cfg.get("artifacts")
routes_cfg = cfg.get("routes")
if not isinstance(artifacts_cfg, dict) or not artifacts_cfg:
die("artifacts must be a non-empty mapping")
if not isinstance(routes_cfg, list) or not routes_cfg:
die("routes must be a non-empty list")
artifacts = {}
buckets = set()
for name, raw in artifacts_cfg.items():
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
die(f"artifact name {name!r} must be a DNS label")
if not isinstance(raw, dict):
die(f"artifact {name} must be a mapping")
_known_keys(
raw,
{"source", "bucket", "credential", "endpoint", "cache", "cors_origins", "excludes"},
f"artifact {name}",
)
source = raw.get("source")
bucket = raw.get("bucket")
credential = raw.get("credential", "default")
if not isinstance(source, str) or not source or Path(source).is_absolute():
die(f"artifact {name}.source must be a relative path")
if source in {".", "./"} or ".." in Path(source).parts:
die(f"artifact {name}.source must name a build output inside the repository")
bucket = _validate_domain(bucket, f"artifact {name}.bucket")
if bucket in buckets:
die(f"artifact bucket {bucket} is used more than once")
buckets.add(bucket)
if not isinstance(credential, str) or not PROFILE_RE.fullmatch(credential):
die(f"artifact {name}.credential must be a lowercase profile name")
endpoint = raw.get("endpoint")
if endpoint is not None:
parsed_endpoint = urlsplit(endpoint) if isinstance(endpoint, str) else None
if (
parsed_endpoint is None or parsed_endpoint.scheme not in {"http", "https"}
or not parsed_endpoint.netloc or parsed_endpoint.username or parsed_endpoint.password
or parsed_endpoint.query or parsed_endpoint.fragment
):
die(f"artifact {name}.endpoint must be an HTTP URL without credentials")
cors_origins = _cors_origins(raw, name)
artifacts[name] = {
"name": name,
"source": source,
"bucket": bucket,
"credential": credential,
"cache": _cache_config(raw, name),
"endpoint": endpoint,
"cors_origins": cors_origins,
"excludes": _patterns(raw, "excludes", f"artifact {name}.excludes"),
}
routes = []
route_names = set()
route_paths = set()
referenced = set()
for raw in routes_cfg:
if not isinstance(raw, dict):
die("each route must be a mapping")
_known_keys(
raw,
{"name", "path", "artifact", "access", "middlewares"},
"route",
)
name = raw.get("name")
path = raw.get("path")
artifact = raw.get("artifact")
access = raw.get("access")
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
die("each route.name must be a DNS label")
if name in route_names:
die(f"route name {name} is used more than once")
route_names.add(name)
if not isinstance(path, str) or not re.fullmatch(
r"/(?:[A-Za-z0-9._~-]+(?:/[A-Za-z0-9._~-]+)*)?", path
):
die(f"route {name}.path must be a canonical absolute URL path")
if path != "/" and path.endswith("/"):
die(f"route {name}.path must not end with /")
if "//" in path or ".." in Path(path).parts or "?" in path or "#" in path:
die(f"route {name}.path is not a canonical URL path")
if path in route_paths:
die(f"route path {path} is used more than once")
route_paths.add(path)
if artifact not in artifacts:
die(f"route {name} references unknown artifact {artifact!r}")
referenced.add(artifact)
if access not in {"public", "protected"}:
die(f"route {name}.access must be public or protected")
route_middlewares = _validate_middlewares(
raw.get("middlewares") or [], f"route {name}.middlewares"
)
if access == "protected":
route_middlewares = [AUTH_MIDDLEWARE, *route_middlewares]
if len(set(route_middlewares)) != len(route_middlewares):
die(f"route {name}.middlewares contains a duplicate")
routes.append({
"name": name,
"path": path,
"artifact": artifact,
"access": access,
"middlewares": route_middlewares,
})
unreferenced = sorted(set(artifacts) - referenced)
if unreferenced:
die(f"artifacts without routes: {', '.join(unreferenced)}")
for artifact_name, artifact in artifacts.items():
access_modes = {
route["access"] for route in routes if route["artifact"] == artifact_name
}
if len(access_modes) != 1:
die(f"artifact {artifact_name} cannot cross public and protected routes")
access = next(iter(access_modes))
policies = [artifact["cache"]["default"], *(
rule["value"] for rule in artifact["cache"]["rules"]
)]
for policy in policies:
directives = _cache_directives(policy)
if access == "protected" and not ({"private", "no-store"} & directives):
die(f"protected artifact {artifact_name} cache policy must be private or no-store")
if access == "protected" and ({"public", "s-maxage"} & directives):
die(f"protected artifact {artifact_name} cannot use shared-cache directives")
if access == "public" and "private" in directives:
die(f"public artifact {artifact_name} cannot use private cache metadata")
if access == "protected" and "*" in artifact["cors_origins"]:
die(f"protected artifact {artifact_name} cannot allow wildcard CORS")
profile_access = {}
for artifact_name, artifact in artifacts.items():
access = next(
route["access"] for route in routes if route["artifact"] == artifact_name
)
existing = profile_access.setdefault(artifact["credential"], access)
if existing != access:
die(
f"credential profile {artifact['credential']} cannot cross public and protected artifacts"
)
return {
"mode": "multi",
"domain": domain,
"aliases": aliases,
"artifacts": artifacts,
"routes": routes,
}
def parse_site_yaml(site_dir):
@@ -69,12 +332,50 @@ def parse_site_yaml(site_dir):
if not path.exists():
die("site.yaml not found in repo root")
with open(path) as f:
cfg = yaml.safe_load(f)
try:
with open(path, encoding="utf-8") as f:
cfg = yaml.safe_load(f)
except YAMLError as error:
die(f"site.yaml is not valid YAML: {error}")
if not isinstance(cfg, dict):
die("site.yaml must contain a mapping")
if not cfg.get("domain"):
die("domain is required in site.yaml")
domain = _validate_domain(cfg["domain"], "domain")
aliases = [
_validate_domain(value, "aliases entry")
for value in _string_list(cfg, "aliases")
]
if len(set([domain, *aliases])) != 1 + len(aliases):
die("domain and aliases must be unique")
has_artifacts = "artifacts" in cfg
has_routes = "routes" in cfg
if has_artifacts != has_routes:
die("artifacts and routes must be declared together")
if has_artifacts:
_known_keys(
cfg,
{"domain", "aliases", "enabled", "artifacts", "routes"},
"site.yaml",
)
site = _modern_config(cfg, domain, aliases)
site["type"] = "artifacts"
site["enabled"] = cfg.get("enabled", True)
if not isinstance(site["enabled"], bool):
die("enabled must be true or false")
site["tidy"] = False
site["content_dir"] = ""
site["excludes"] = []
site["middlewares"] = []
print("Site config:")
print(f" domain: {site['domain']}")
print(f" artifacts: {', '.join(site['artifacts'])}")
print(f" routes: {', '.join(route['path'] for route in site['routes'])}")
return site
site_type = cfg.get("type", "static")
if site_type == "docker":
@@ -83,20 +384,26 @@ def parse_site_yaml(site_dir):
if site_type not in VALID_TYPES:
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
excludes = cfg.get("excludes") or []
if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes):
die("excludes must be a list of string patterns")
excludes = _string_list(cfg, "excludes")
middlewares = _validate_middlewares(cfg.get("middlewares") or [], "middlewares")
middlewares = cfg.get("middlewares") or []
if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares):
die("middlewares must be a list of Traefik file-provider middleware names")
content_dir = cfg.get("content_dir", "")
if not isinstance(content_dir, str) or Path(content_dir).is_absolute():
die("content_dir must be a relative path")
if ".." in Path(content_dir).parts:
die("content_dir cannot escape the repository")
enabled = cfg.get("enabled", True)
if not isinstance(enabled, bool):
die("enabled must be true or false")
site = {
"domain": cfg["domain"],
"mode": "legacy",
"domain": domain,
"type": site_type,
"enabled": cfg.get("enabled", True),
"aliases": cfg.get("aliases") or [],
"content_dir": cfg.get("content_dir", ""),
"enabled": enabled,
"aliases": aliases,
"content_dir": content_dir,
"tidy": cfg.get("tidy", True),
"excludes": excludes,
"middlewares": middlewares,
@@ -110,21 +417,46 @@ def parse_site_yaml(site_dir):
def clone_apps(token):
user = env("CI_BOT_USER", "ci-bot")
apps_dir = Path("/tmp/apps-deploy")
if apps_dir.exists():
shutil.rmtree(apps_dir)
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}")
run(f"git -C {apps_dir} config user.name {user}")
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
clone_root = Path(tempfile.mkdtemp(prefix="apps-deploy-"))
apps_dir = clone_root / "repo"
askpass = clone_root / "askpass"
askpass.write_text(
"#!/bin/sh\ncase \"$1\" in *Username*) printf '%s\\n' \"$GIT_AUTH_USER\" ;; "
"*) printf '%s\\n' \"$GIT_AUTH_TOKEN\" ;; esac\n",
encoding="utf-8",
)
askpass.chmod(0o700)
git_env = git_auth_environment(token, user, askpass)
try:
run_args(
["git", "clone", "--depth", "1", f"https://{GITEA_HOST}/{APPS_REPO}.git", str(apps_dir)],
display=f"git clone --depth 1 https://{GITEA_HOST}/{APPS_REPO}.git {apps_dir}",
env=git_env,
)
except Exception:
shutil.rmtree(clone_root, ignore_errors=True)
raise
run_args(["git", "-C", str(apps_dir), "config", "user.name", user])
run_args(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
return apps_dir
def git_auth_environment(token, user, askpass):
child_env = os.environ.copy()
child_env["GIT_ASKPASS"] = str(askpass)
child_env["GIT_TERMINAL_PROMPT"] = "0"
child_env["GIT_AUTH_USER"] = user
child_env["GIT_AUTH_TOKEN"] = token
return child_env
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
"""Render Jinja2 templates for a static-content site."""
templates_dir = Path(action_dir) / "templates"
jinja_env = Environment(
loader=FileSystemLoader(str(templates_dir)),
keep_trailing_newline=True,
undefined=StrictUndefined,
)
tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2",
@@ -139,16 +471,21 @@ def render_templates(action_dir, template_vars, app_dir, manifests_dir):
print(f" Rendered {tmpl_name} -> {dest}")
def commit_and_push(apps_dir, message):
run(f"git -C {apps_dir} add -A")
def commit_and_push(apps_dir, message, token):
run_args(["git", "-C", str(apps_dir), "add", "-A"])
result = subprocess.run(
f"git -C {apps_dir} diff --cached --quiet",
shell=True, check=False,
["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"],
check=False,
)
if result.returncode == 0:
print("No manifest changes to commit")
return False
run(f"git -C {apps_dir} commit -m '{message}'")
run(f"git -C {apps_dir} push")
run_args(["git", "-C", str(apps_dir), "commit", "-m", message])
git_env = git_auth_environment(
token,
env("CI_BOT_USER", "ci-bot"),
apps_dir.parent / "askpass",
)
run_args(["git", "-C", str(apps_dir), "push"], env=git_env)
print("Manifests pushed — ArgoCD will sync")
return True