Dave Kowalski
099a48f2b5
Test / test (pull_request) Failing after 5s
Authored-By: OpenAI (GPT-5) <noreply@openai.com>
344 lines
15 KiB
Python
344 lines
15 KiB
Python
"""Shared utilities for the site-publish action."""
|
|
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from contextlib import contextmanager
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import yaml
|
|
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
|
|
|
APPS_REPO = "fritzlab/apps"
|
|
GITEA_HOST = "code.fritzlab.net"
|
|
NAMESPACE = "websites"
|
|
DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
|
|
GARAGE_WEBSITE_HOST = "garage-s3.storage.svc.k8s.sjc001.fritzlab.net"
|
|
EXCLUDE_FILES = {
|
|
".git", ".gitea", ".gitignore", "site.yaml", "build", "Makefile",
|
|
"README.md", "CLAUDE.md", "Dockerfile", ".dockerignore", "go.mod", "go.sum",
|
|
}
|
|
VALID_TYPES = {"static", "hugo", "mkdocs"}
|
|
CACHE_POLICIES = {
|
|
"immutable-release": ("public, max-age=31536000, immutable", True),
|
|
"revalidated-channel": ("public, max-age=0, must-revalidate", False),
|
|
"private": ("private, no-store", False),
|
|
}
|
|
ENV_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
|
|
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$")
|
|
BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$")
|
|
HOST_RE = BUCKET_RE
|
|
MIDDLEWARE_RE = NAME_RE
|
|
|
|
DOCKER_DEPRECATION_MSG = """\
|
|
type: docker is no longer supported by action/site-publish.
|
|
|
|
site-publish handles only static-content sites (static, hugo, mkdocs).
|
|
Use action/image-build, action/image-push, and action/image-deploy for images.\
|
|
"""
|
|
|
|
|
|
def k8s_name(*parts):
|
|
raw = re.sub(r"[^a-z0-9-]", "-", "-".join(parts).replace(".", "-").lower())
|
|
raw = raw.strip("-")
|
|
if len(raw) <= 63:
|
|
return raw
|
|
return f"{raw[:54].rstrip('-')}-{hashlib.sha256(raw.encode()).hexdigest()[:8]}"
|
|
|
|
|
|
def env(key, default=None):
|
|
value = os.environ.get(key, default)
|
|
if value is None or value == "":
|
|
die(f"Missing required env var: {key}")
|
|
return value
|
|
|
|
|
|
def die(message):
|
|
print(f"ERROR: {message}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def run(cmd, *, display=None, **kwargs):
|
|
"""Run argv without a shell and print only a credential-free display."""
|
|
if isinstance(cmd, str):
|
|
raise TypeError("run() requires an argv sequence")
|
|
print(f" $ {display or ' '.join(str(part) for part in cmd)}")
|
|
return subprocess.run(cmd, check=True, **kwargs)
|
|
|
|
|
|
def _string_list(cfg, key):
|
|
value = cfg.get(key) or []
|
|
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
|
die(f"{key} must be a list of strings")
|
|
return value
|
|
|
|
|
|
def _common(cfg):
|
|
domain = cfg.get("domain")
|
|
if not isinstance(domain, str) or not HOST_RE.fullmatch(domain):
|
|
die("domain must be a lowercase hostname")
|
|
aliases = _string_list(cfg, "aliases")
|
|
if any(not HOST_RE.fullmatch(alias) for alias in aliases):
|
|
die("aliases must contain lowercase hostnames")
|
|
site_type = cfg.get("type", "static")
|
|
if site_type == "docker":
|
|
die(DOCKER_DEPRECATION_MSG)
|
|
if site_type not in VALID_TYPES:
|
|
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
|
|
middlewares = _string_list(cfg, "middlewares")
|
|
if any(not MIDDLEWARE_RE.fullmatch(item) for item in middlewares):
|
|
die("middlewares must contain Traefik file-provider names")
|
|
if not isinstance(cfg.get("enabled", True), bool):
|
|
die("enabled must be a boolean")
|
|
if not isinstance(cfg.get("tidy", True), bool):
|
|
die("tidy must be a boolean")
|
|
content_dir = cfg.get("content_dir", "")
|
|
content_path = PurePosixPath(content_dir)
|
|
if not isinstance(content_dir, str) or content_path.is_absolute() or ".." in content_path.parts:
|
|
die("content_dir must be a relative path inside the repository")
|
|
return {
|
|
"domain": domain,
|
|
"type": site_type,
|
|
"enabled": cfg.get("enabled", True),
|
|
"aliases": aliases,
|
|
"content_dir": content_dir,
|
|
"tidy": cfg.get("tidy", True),
|
|
"excludes": _string_list(cfg, "excludes"),
|
|
"middlewares": middlewares,
|
|
}
|
|
|
|
|
|
def _source(value, name):
|
|
if not isinstance(value, str) or not value:
|
|
die(f"artifact {name}: source must be a non-empty relative path")
|
|
path = PurePosixPath(value)
|
|
if path.is_absolute() or ".." in path.parts:
|
|
die(f"artifact {name}: source must stay under build/html")
|
|
return value
|
|
|
|
|
|
def _v2(cfg, site):
|
|
if cfg.get("schema") != "v2":
|
|
die("multi-surface configs must set schema: v2")
|
|
raw_artifacts, raw_routes = cfg.get("artifacts"), cfg.get("routes")
|
|
if not isinstance(raw_artifacts, dict) or not raw_artifacts:
|
|
die("artifacts must be a non-empty mapping")
|
|
if not isinstance(raw_routes, list) or not raw_routes:
|
|
die("routes must be a non-empty list")
|
|
|
|
artifacts = {}
|
|
for name, raw in raw_artifacts.items():
|
|
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
|
|
die("artifact names must be lowercase kebab-case")
|
|
if not isinstance(raw, dict):
|
|
die(f"artifact {name}: definition must be a mapping")
|
|
unknown = set(raw) - {"source", "bucket", "credentials", "cache"}
|
|
if unknown:
|
|
die(f"artifact {name}: unknown keys: {', '.join(sorted(unknown))}")
|
|
bucket = raw.get("bucket")
|
|
if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket):
|
|
die(f"artifact {name}: bucket must be a lowercase Garage bucket name")
|
|
credentials = raw.get("credentials")
|
|
if not isinstance(credentials, dict) or set(credentials) != {
|
|
"access_key_env", "secret_key_env"}:
|
|
die(f"artifact {name}: credentials require access_key_env and secret_key_env")
|
|
if any(not isinstance(value, str) or not ENV_RE.fullmatch(value)
|
|
for value in credentials.values()):
|
|
die(f"artifact {name}: credential selectors must be environment variable names")
|
|
cache = raw.get("cache")
|
|
if cache not in CACHE_POLICIES:
|
|
die(f"artifact {name}: unknown cache policy {cache!r}")
|
|
cache_control, immutable = CACHE_POLICIES[cache]
|
|
artifacts[name] = {
|
|
"name": name, "source": _source(raw.get("source"), name),
|
|
"bucket": bucket, "credentials": credentials, "cache": cache,
|
|
"cache_control": cache_control, "immutable": immutable,
|
|
}
|
|
|
|
routes, paths, used = [], set(), set()
|
|
buckets, credentials = {}, {}
|
|
for raw in raw_routes:
|
|
if not isinstance(raw, dict):
|
|
die("each route must be a mapping")
|
|
unknown = set(raw) - {"path", "artifact", "access", "middlewares"}
|
|
if unknown:
|
|
die(f"route has unknown keys: {', '.join(sorted(unknown))}")
|
|
path = raw.get("path")
|
|
if (not isinstance(path, str) or not path.startswith("/") or "//" in path
|
|
or (path != "/" and path.endswith("/"))):
|
|
die("route paths must be normalized absolute prefixes")
|
|
if path in paths:
|
|
die(f"duplicate route path: {path}")
|
|
paths.add(path)
|
|
artifact_name = raw.get("artifact")
|
|
if artifact_name not in artifacts:
|
|
die(f"route {path}: unknown artifact {artifact_name!r}")
|
|
if artifact_name in used:
|
|
die(f"artifact {artifact_name} may be routed only once")
|
|
used.add(artifact_name)
|
|
access = raw.get("access")
|
|
if access not in {"public", "authenticated"}:
|
|
die(f"route {path}: access must be public or authenticated")
|
|
if path == "/" and access == "public":
|
|
die("public catch-all route / is forbidden in schema v2")
|
|
middlewares = raw.get("middlewares") or []
|
|
if not isinstance(middlewares, list) or any(
|
|
not isinstance(item, str) or not MIDDLEWARE_RE.fullmatch(item)
|
|
for item in middlewares):
|
|
die(f"route {path}: middlewares must contain file-provider names")
|
|
if access == "authenticated" and not middlewares:
|
|
die(f"route {path}: authenticated access requires middleware")
|
|
artifact = artifacts[artifact_name]
|
|
if access == "authenticated" and artifact["cache"] != "private":
|
|
die(f"route {path}: authenticated artifacts must use private cache")
|
|
if access == "public" and artifact["cache"] == "private":
|
|
die(f"route {path}: public artifacts cannot use private cache")
|
|
prior_access = buckets.setdefault(artifact["bucket"], access)
|
|
if prior_access != access:
|
|
die(f"route {path}: bucket is reused across access classes")
|
|
credential_pair = tuple(artifact["credentials"].values())
|
|
prior_artifact = credentials.setdefault(credential_pair, artifact_name)
|
|
if prior_artifact != artifact_name:
|
|
die(f"route {path}: publication credentials are reused by artifacts "
|
|
f"{prior_artifact} and {artifact_name}")
|
|
artifact["access"] = access
|
|
routes.append({"path": path, "artifact": artifact_name,
|
|
"access": access, "middlewares": middlewares})
|
|
artifact["key_prefix"] = path.lstrip("/")
|
|
|
|
missing = set(artifacts) - used
|
|
if missing:
|
|
die(f"unrouted artifacts: {', '.join(sorted(missing))}")
|
|
if "/" not in paths:
|
|
die("schema v2 requires an explicit / access policy")
|
|
artifact_list = list(artifacts.values())
|
|
for index, left in enumerate(artifact_list):
|
|
left_parts = PurePosixPath(left["source"]).parts
|
|
for right in artifact_list[index + 1:]:
|
|
right_parts = PurePosixPath(right["source"]).parts
|
|
overlaps = (left_parts == right_parts[:len(left_parts)] or
|
|
right_parts == left_parts[:len(right_parts)])
|
|
if overlaps and left["access"] != right["access"]:
|
|
die(f"artifact sources overlap across access classes: "
|
|
f"{left['name']} and {right['name']}")
|
|
site.update({"schema": "v2", "compatibility": False, "artifacts": artifacts})
|
|
site["routes"] = sorted(routes, key=lambda route: (-len(route["path"]), route["path"]))
|
|
return site
|
|
|
|
|
|
def parse_site_yaml(site_dir, site_name=None):
|
|
path = Path(site_dir) / "site.yaml"
|
|
if not path.exists():
|
|
die("site.yaml not found in repo root")
|
|
with open(path, encoding="utf-8") as handle:
|
|
cfg = yaml.safe_load(handle)
|
|
if not isinstance(cfg, dict):
|
|
die("site.yaml must contain a mapping")
|
|
allowed = {"domain", "type", "enabled", "aliases", "content_dir", "tidy",
|
|
"excludes", "middlewares", "schema", "artifacts", "routes"}
|
|
unknown = set(cfg) - allowed
|
|
if unknown:
|
|
die(f"unknown site.yaml keys: {', '.join(sorted(unknown))}")
|
|
site = _common(cfg)
|
|
has_v2 = any(key in cfg for key in ("schema", "artifacts", "routes"))
|
|
if has_v2:
|
|
if not all(key in cfg for key in ("schema", "artifacts", "routes")):
|
|
die("schema, artifacts, and routes must be declared together")
|
|
if site["middlewares"]:
|
|
die("schema v2 middlewares belong on individual routes")
|
|
site = _v2(cfg, site)
|
|
else:
|
|
site_name = site_name or site["domain"]
|
|
cache_control, immutable = CACHE_POLICIES["revalidated-channel"]
|
|
site.update({
|
|
"schema": "single-surface-v1", "compatibility": True,
|
|
"artifacts": {"site": {
|
|
"name": "site", "source": ".", "bucket": site_name,
|
|
"credentials": {"access_key_env": "AWS_ACCESS_KEY_ID",
|
|
"secret_key_env": "AWS_SECRET_ACCESS_KEY"},
|
|
"cache": "revalidated-channel", "cache_control": cache_control,
|
|
"immutable": immutable, "key_prefix": "",
|
|
}},
|
|
"routes": [{"path": "/", "artifact": "site", "access": "legacy",
|
|
"middlewares": site["middlewares"]}],
|
|
})
|
|
print(f"Site config: {site['schema']} {site['domain']} ({len(site['routes'])} route(s))")
|
|
return site
|
|
|
|
|
|
@contextmanager
|
|
def git_auth(token, user):
|
|
"""Provide an HTTPS token through an inherited FD, never argv or output."""
|
|
read_fd, write_fd = os.pipe()
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
askpass = Path(temp_dir) / "askpass.sh"
|
|
askpass.write_text(
|
|
"#!/bin/sh\ncase \"$1\" in\n"
|
|
" *Username*) printf '%s\\n' \"$SITE_PUBLISH_GIT_USER\" ;;\n"
|
|
f" *Password*) cat <&{read_fd} ;;\nesac\n", encoding="utf-8")
|
|
askpass.chmod(0o700)
|
|
os.write(write_fd, token.encode())
|
|
os.close(write_fd)
|
|
child_env = os.environ.copy()
|
|
child_env.update({"GIT_ASKPASS": str(askpass), "GIT_TERMINAL_PROMPT": "0",
|
|
"SITE_PUBLISH_GIT_USER": user})
|
|
try:
|
|
yield {"env": child_env, "pass_fds": (read_fd,)}
|
|
finally:
|
|
os.close(read_fd)
|
|
|
|
|
|
def clone_apps(token):
|
|
user = env("CI_BOT_USER", "ci-bot")
|
|
apps_dir = Path("/tmp/apps-deploy")
|
|
if apps_dir.exists():
|
|
shutil.rmtree(apps_dir)
|
|
url = f"https://{GITEA_HOST}/{APPS_REPO}.git"
|
|
with git_auth(token, user) as auth:
|
|
run(["git", "clone", "--depth", "1", url, str(apps_dir)], **auth)
|
|
run(["git", "-C", str(apps_dir), "config", "user.name", user])
|
|
run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
|
|
return apps_dir
|
|
|
|
|
|
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
|
|
templates = Path(action_dir) / "templates"
|
|
jinja = Environment(loader=FileSystemLoader(str(templates)),
|
|
keep_trailing_newline=True, undefined=StrictUndefined)
|
|
app_dir.mkdir(parents=True, exist_ok=True)
|
|
manifests_dir.mkdir(parents=True, exist_ok=True)
|
|
(app_dir / "app.yaml").write_text(
|
|
jinja.get_template("app.yaml.j2").render(**template_vars), encoding="utf-8")
|
|
(manifests_dir / "certificate.yaml").write_text(
|
|
jinja.get_template("certificate.yaml.j2").render(**template_vars), encoding="utf-8")
|
|
resources = ["certificate.yaml"]
|
|
for route in template_vars["routes"]:
|
|
values = {**template_vars, "route": route}
|
|
for kind in ("service", "ingress"):
|
|
filename = f"{kind}-{route['resource_name']}.yaml"
|
|
(manifests_dir / filename).write_text(
|
|
jinja.get_template(f"{kind}.yaml.j2").render(**values), encoding="utf-8")
|
|
resources.append(filename)
|
|
(manifests_dir / "kustomization.yaml").write_text(
|
|
jinja.get_template("kustomization.yaml.j2").render(
|
|
**template_vars, resources=resources), encoding="utf-8")
|
|
|
|
|
|
def commit_and_push(apps_dir, message, token):
|
|
user = env("CI_BOT_USER", "ci-bot")
|
|
run(["git", "-C", str(apps_dir), "add", "-A"])
|
|
clean = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"],
|
|
check=False).returncode == 0
|
|
if clean:
|
|
print("No manifest changes to commit")
|
|
return False
|
|
run(["git", "-C", str(apps_dir), "commit", "-m", message])
|
|
with git_auth(token, user) as auth:
|
|
run(["git", "-C", str(apps_dir), "push"], **auth)
|
|
print("Manifests pushed — ArgoCD will sync")
|
|
return True
|