Authored-By: OpenAI (GPT-5) <noreply@openai.com>
This commit is contained in:
Dave Kowalski
+279
-90
@@ -1,154 +1,343 @@
|
||||
"""Shared utilities for the site-publish action."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
||||
|
||||
APPS_REPO = "fritzlab/apps"
|
||||
GITEA_HOST = "code.fritzlab.net"
|
||||
NAMESPACE = "websites"
|
||||
DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
|
||||
|
||||
GARAGE_WEBSITE_HOST = "garage-s3.storage.svc.k8s.sjc001.fritzlab.net"
|
||||
EXCLUDE_FILES = {
|
||||
".git", ".gitea", ".gitignore", "site.yaml",
|
||||
"build", "Makefile", "README.md", "CLAUDE.md",
|
||||
"Dockerfile", ".dockerignore", "go.mod", "go.sum",
|
||||
".git", ".gitea", ".gitignore", "site.yaml", "build", "Makefile",
|
||||
"README.md", "CLAUDE.md", "Dockerfile", ".dockerignore", "go.mod", "go.sum",
|
||||
}
|
||||
|
||||
VALID_TYPES = {"static", "hugo", "mkdocs"}
|
||||
CACHE_POLICIES = {
|
||||
"immutable-release": ("public, max-age=31536000, immutable", True),
|
||||
"revalidated-channel": ("public, max-age=0, must-revalidate", False),
|
||||
"private": ("private, no-store", False),
|
||||
}
|
||||
ENV_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
|
||||
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$")
|
||||
BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$")
|
||||
HOST_RE = BUCKET_RE
|
||||
MIDDLEWARE_RE = NAME_RE
|
||||
|
||||
DOCKER_DEPRECATION_MSG = """\
|
||||
type: docker is no longer supported by action/site-publish.
|
||||
|
||||
site-publish handles only static-content sites (static, hugo, mkdocs)
|
||||
that ship to Garage S3. For containerized web apps, use the standard
|
||||
image-producer chain:
|
||||
|
||||
- uses: action/image-build@v1 # build + smoke-test
|
||||
- uses: action/image-push@v1 # push + prune
|
||||
- uses: action/image-deploy@v1 # apps repo image-pin
|
||||
|
||||
Hand-author your apps-repo manifests once (Deployment, Service, Ingress,
|
||||
Certificate, kustomization with images: block) under
|
||||
sjc001/websites/<repo>/manifests/. image-deploy will pin the tag on
|
||||
every CI run. See action/image-deploy README and
|
||||
sjc001/websites/rainsounds.vino.network/manifests/ for the canonical
|
||||
example.\
|
||||
site-publish handles only static-content sites (static, hugo, mkdocs).
|
||||
Use action/image-build, action/image-push, and action/image-deploy for images.\
|
||||
"""
|
||||
|
||||
|
||||
def k8s_name(name):
|
||||
"""Sanitize for DNS-1035 label (dots → dashes)."""
|
||||
return name.replace(".", "-")
|
||||
def k8s_name(*parts):
|
||||
raw = re.sub(r"[^a-z0-9-]", "-", "-".join(parts).replace(".", "-").lower())
|
||||
raw = raw.strip("-")
|
||||
if len(raw) <= 63:
|
||||
return raw
|
||||
return f"{raw[:54].rstrip('-')}-{hashlib.sha256(raw.encode()).hexdigest()[:8]}"
|
||||
|
||||
|
||||
def env(key, default=None):
|
||||
val = os.environ.get(key, default)
|
||||
if val is None:
|
||||
value = os.environ.get(key, default)
|
||||
if value is None or value == "":
|
||||
die(f"Missing required env var: {key}")
|
||||
return val
|
||||
return value
|
||||
|
||||
|
||||
def die(msg):
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
def die(message):
|
||||
print(f"ERROR: {message}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
print(f" $ {cmd}")
|
||||
return subprocess.run(cmd, shell=True, check=True, **kwargs)
|
||||
def run(cmd, *, display=None, **kwargs):
|
||||
"""Run argv without a shell and print only a credential-free display."""
|
||||
if isinstance(cmd, str):
|
||||
raise TypeError("run() requires an argv sequence")
|
||||
print(f" $ {display or ' '.join(str(part) for part in cmd)}")
|
||||
return subprocess.run(cmd, check=True, **kwargs)
|
||||
|
||||
|
||||
def parse_site_yaml(site_dir):
|
||||
path = Path(site_dir) / "site.yaml"
|
||||
if not path.exists():
|
||||
die("site.yaml not found in repo root")
|
||||
def _string_list(cfg, key):
|
||||
value = cfg.get(key) or []
|
||||
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
||||
die(f"{key} must be a list of strings")
|
||||
return value
|
||||
|
||||
with open(path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
|
||||
if not cfg.get("domain"):
|
||||
die("domain is required in site.yaml")
|
||||
|
||||
def _common(cfg):
|
||||
domain = cfg.get("domain")
|
||||
if not isinstance(domain, str) or not HOST_RE.fullmatch(domain):
|
||||
die("domain must be a lowercase hostname")
|
||||
aliases = _string_list(cfg, "aliases")
|
||||
if any(not HOST_RE.fullmatch(alias) for alias in aliases):
|
||||
die("aliases must contain lowercase hostnames")
|
||||
site_type = cfg.get("type", "static")
|
||||
|
||||
if site_type == "docker":
|
||||
die(DOCKER_DEPRECATION_MSG)
|
||||
|
||||
if site_type not in VALID_TYPES:
|
||||
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
|
||||
|
||||
excludes = cfg.get("excludes") or []
|
||||
if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes):
|
||||
die("excludes must be a list of string patterns")
|
||||
|
||||
middlewares = cfg.get("middlewares") or []
|
||||
if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares):
|
||||
die("middlewares must be a list of Traefik file-provider middleware names")
|
||||
|
||||
site = {
|
||||
"domain": cfg["domain"],
|
||||
middlewares = _string_list(cfg, "middlewares")
|
||||
if any(not MIDDLEWARE_RE.fullmatch(item) for item in middlewares):
|
||||
die("middlewares must contain Traefik file-provider names")
|
||||
if not isinstance(cfg.get("enabled", True), bool):
|
||||
die("enabled must be a boolean")
|
||||
if not isinstance(cfg.get("tidy", True), bool):
|
||||
die("tidy must be a boolean")
|
||||
content_dir = cfg.get("content_dir", "")
|
||||
content_path = PurePosixPath(content_dir)
|
||||
if not isinstance(content_dir, str) or content_path.is_absolute() or ".." in content_path.parts:
|
||||
die("content_dir must be a relative path inside the repository")
|
||||
return {
|
||||
"domain": domain,
|
||||
"type": site_type,
|
||||
"enabled": cfg.get("enabled", True),
|
||||
"aliases": cfg.get("aliases") or [],
|
||||
"content_dir": cfg.get("content_dir", ""),
|
||||
"aliases": aliases,
|
||||
"content_dir": content_dir,
|
||||
"tidy": cfg.get("tidy", True),
|
||||
"excludes": excludes,
|
||||
"excludes": _string_list(cfg, "excludes"),
|
||||
"middlewares": middlewares,
|
||||
}
|
||||
|
||||
print("Site config:")
|
||||
for k, v in site.items():
|
||||
print(f" {k}: {v}")
|
||||
|
||||
def _source(value, name):
|
||||
if not isinstance(value, str) or not value:
|
||||
die(f"artifact {name}: source must be a non-empty relative path")
|
||||
path = PurePosixPath(value)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
die(f"artifact {name}: source must stay under build/html")
|
||||
return value
|
||||
|
||||
|
||||
def _v2(cfg, site):
|
||||
if cfg.get("schema") != "v2":
|
||||
die("multi-surface configs must set schema: v2")
|
||||
raw_artifacts, raw_routes = cfg.get("artifacts"), cfg.get("routes")
|
||||
if not isinstance(raw_artifacts, dict) or not raw_artifacts:
|
||||
die("artifacts must be a non-empty mapping")
|
||||
if not isinstance(raw_routes, list) or not raw_routes:
|
||||
die("routes must be a non-empty list")
|
||||
|
||||
artifacts = {}
|
||||
for name, raw in raw_artifacts.items():
|
||||
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
|
||||
die("artifact names must be lowercase kebab-case")
|
||||
if not isinstance(raw, dict):
|
||||
die(f"artifact {name}: definition must be a mapping")
|
||||
unknown = set(raw) - {"source", "bucket", "credentials", "cache"}
|
||||
if unknown:
|
||||
die(f"artifact {name}: unknown keys: {', '.join(sorted(unknown))}")
|
||||
bucket = raw.get("bucket")
|
||||
if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket):
|
||||
die(f"artifact {name}: bucket must be a lowercase Garage bucket name")
|
||||
credentials = raw.get("credentials")
|
||||
if not isinstance(credentials, dict) or set(credentials) != {
|
||||
"access_key_env", "secret_key_env"}:
|
||||
die(f"artifact {name}: credentials require access_key_env and secret_key_env")
|
||||
if any(not isinstance(value, str) or not ENV_RE.fullmatch(value)
|
||||
for value in credentials.values()):
|
||||
die(f"artifact {name}: credential selectors must be environment variable names")
|
||||
cache = raw.get("cache")
|
||||
if cache not in CACHE_POLICIES:
|
||||
die(f"artifact {name}: unknown cache policy {cache!r}")
|
||||
cache_control, immutable = CACHE_POLICIES[cache]
|
||||
artifacts[name] = {
|
||||
"name": name, "source": _source(raw.get("source"), name),
|
||||
"bucket": bucket, "credentials": credentials, "cache": cache,
|
||||
"cache_control": cache_control, "immutable": immutable,
|
||||
}
|
||||
|
||||
routes, paths, used = [], set(), set()
|
||||
buckets, credentials = {}, {}
|
||||
for raw in raw_routes:
|
||||
if not isinstance(raw, dict):
|
||||
die("each route must be a mapping")
|
||||
unknown = set(raw) - {"path", "artifact", "access", "middlewares"}
|
||||
if unknown:
|
||||
die(f"route has unknown keys: {', '.join(sorted(unknown))}")
|
||||
path = raw.get("path")
|
||||
if (not isinstance(path, str) or not path.startswith("/") or "//" in path
|
||||
or (path != "/" and path.endswith("/"))):
|
||||
die("route paths must be normalized absolute prefixes")
|
||||
if path in paths:
|
||||
die(f"duplicate route path: {path}")
|
||||
paths.add(path)
|
||||
artifact_name = raw.get("artifact")
|
||||
if artifact_name not in artifacts:
|
||||
die(f"route {path}: unknown artifact {artifact_name!r}")
|
||||
if artifact_name in used:
|
||||
die(f"artifact {artifact_name} may be routed only once")
|
||||
used.add(artifact_name)
|
||||
access = raw.get("access")
|
||||
if access not in {"public", "authenticated"}:
|
||||
die(f"route {path}: access must be public or authenticated")
|
||||
if path == "/" and access == "public":
|
||||
die("public catch-all route / is forbidden in schema v2")
|
||||
middlewares = raw.get("middlewares") or []
|
||||
if not isinstance(middlewares, list) or any(
|
||||
not isinstance(item, str) or not MIDDLEWARE_RE.fullmatch(item)
|
||||
for item in middlewares):
|
||||
die(f"route {path}: middlewares must contain file-provider names")
|
||||
if access == "authenticated" and not middlewares:
|
||||
die(f"route {path}: authenticated access requires middleware")
|
||||
artifact = artifacts[artifact_name]
|
||||
if access == "authenticated" and artifact["cache"] != "private":
|
||||
die(f"route {path}: authenticated artifacts must use private cache")
|
||||
if access == "public" and artifact["cache"] == "private":
|
||||
die(f"route {path}: public artifacts cannot use private cache")
|
||||
prior_access = buckets.setdefault(artifact["bucket"], access)
|
||||
if prior_access != access:
|
||||
die(f"route {path}: bucket is reused across access classes")
|
||||
credential_pair = tuple(artifact["credentials"].values())
|
||||
prior_artifact = credentials.setdefault(credential_pair, artifact_name)
|
||||
if prior_artifact != artifact_name:
|
||||
die(f"route {path}: publication credentials are reused by artifacts "
|
||||
f"{prior_artifact} and {artifact_name}")
|
||||
artifact["access"] = access
|
||||
routes.append({"path": path, "artifact": artifact_name,
|
||||
"access": access, "middlewares": middlewares})
|
||||
artifact["key_prefix"] = path.lstrip("/")
|
||||
|
||||
missing = set(artifacts) - used
|
||||
if missing:
|
||||
die(f"unrouted artifacts: {', '.join(sorted(missing))}")
|
||||
if "/" not in paths:
|
||||
die("schema v2 requires an explicit / access policy")
|
||||
artifact_list = list(artifacts.values())
|
||||
for index, left in enumerate(artifact_list):
|
||||
left_parts = PurePosixPath(left["source"]).parts
|
||||
for right in artifact_list[index + 1:]:
|
||||
right_parts = PurePosixPath(right["source"]).parts
|
||||
overlaps = (left_parts == right_parts[:len(left_parts)] or
|
||||
right_parts == left_parts[:len(right_parts)])
|
||||
if overlaps and left["access"] != right["access"]:
|
||||
die(f"artifact sources overlap across access classes: "
|
||||
f"{left['name']} and {right['name']}")
|
||||
site.update({"schema": "v2", "compatibility": False, "artifacts": artifacts})
|
||||
site["routes"] = sorted(routes, key=lambda route: (-len(route["path"]), route["path"]))
|
||||
return site
|
||||
|
||||
|
||||
def parse_site_yaml(site_dir, site_name=None):
|
||||
path = Path(site_dir) / "site.yaml"
|
||||
if not path.exists():
|
||||
die("site.yaml not found in repo root")
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
cfg = yaml.safe_load(handle)
|
||||
if not isinstance(cfg, dict):
|
||||
die("site.yaml must contain a mapping")
|
||||
allowed = {"domain", "type", "enabled", "aliases", "content_dir", "tidy",
|
||||
"excludes", "middlewares", "schema", "artifacts", "routes"}
|
||||
unknown = set(cfg) - allowed
|
||||
if unknown:
|
||||
die(f"unknown site.yaml keys: {', '.join(sorted(unknown))}")
|
||||
site = _common(cfg)
|
||||
has_v2 = any(key in cfg for key in ("schema", "artifacts", "routes"))
|
||||
if has_v2:
|
||||
if not all(key in cfg for key in ("schema", "artifacts", "routes")):
|
||||
die("schema, artifacts, and routes must be declared together")
|
||||
if site["middlewares"]:
|
||||
die("schema v2 middlewares belong on individual routes")
|
||||
site = _v2(cfg, site)
|
||||
else:
|
||||
site_name = site_name or site["domain"]
|
||||
cache_control, immutable = CACHE_POLICIES["revalidated-channel"]
|
||||
site.update({
|
||||
"schema": "single-surface-v1", "compatibility": True,
|
||||
"artifacts": {"site": {
|
||||
"name": "site", "source": ".", "bucket": site_name,
|
||||
"credentials": {"access_key_env": "AWS_ACCESS_KEY_ID",
|
||||
"secret_key_env": "AWS_SECRET_ACCESS_KEY"},
|
||||
"cache": "revalidated-channel", "cache_control": cache_control,
|
||||
"immutable": immutable, "key_prefix": "",
|
||||
}},
|
||||
"routes": [{"path": "/", "artifact": "site", "access": "legacy",
|
||||
"middlewares": site["middlewares"]}],
|
||||
})
|
||||
print(f"Site config: {site['schema']} {site['domain']} ({len(site['routes'])} route(s))")
|
||||
return site
|
||||
|
||||
|
||||
@contextmanager
|
||||
def git_auth(token, user):
|
||||
"""Provide an HTTPS token through an inherited FD, never argv or output."""
|
||||
read_fd, write_fd = os.pipe()
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
askpass = Path(temp_dir) / "askpass.sh"
|
||||
askpass.write_text(
|
||||
"#!/bin/sh\ncase \"$1\" in\n"
|
||||
" *Username*) printf '%s\\n' \"$SITE_PUBLISH_GIT_USER\" ;;\n"
|
||||
f" *Password*) cat <&{read_fd} ;;\nesac\n", encoding="utf-8")
|
||||
askpass.chmod(0o700)
|
||||
os.write(write_fd, token.encode())
|
||||
os.close(write_fd)
|
||||
child_env = os.environ.copy()
|
||||
child_env.update({"GIT_ASKPASS": str(askpass), "GIT_TERMINAL_PROMPT": "0",
|
||||
"SITE_PUBLISH_GIT_USER": user})
|
||||
try:
|
||||
yield {"env": child_env, "pass_fds": (read_fd,)}
|
||||
finally:
|
||||
os.close(read_fd)
|
||||
|
||||
|
||||
def clone_apps(token):
|
||||
user = env("CI_BOT_USER", "ci-bot")
|
||||
apps_dir = Path("/tmp/apps-deploy")
|
||||
if apps_dir.exists():
|
||||
shutil.rmtree(apps_dir)
|
||||
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}")
|
||||
run(f"git -C {apps_dir} config user.name {user}")
|
||||
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
|
||||
url = f"https://{GITEA_HOST}/{APPS_REPO}.git"
|
||||
with git_auth(token, user) as auth:
|
||||
run(["git", "clone", "--depth", "1", url, str(apps_dir)], **auth)
|
||||
run(["git", "-C", str(apps_dir), "config", "user.name", user])
|
||||
run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
|
||||
return apps_dir
|
||||
|
||||
|
||||
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
|
||||
"""Render Jinja2 templates for a static-content site."""
|
||||
templates_dir = Path(action_dir) / "templates"
|
||||
jinja_env = Environment(
|
||||
loader=FileSystemLoader(str(templates_dir)),
|
||||
keep_trailing_newline=True,
|
||||
)
|
||||
|
||||
tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2",
|
||||
"kustomization.yaml.j2", "service.yaml.j2"]
|
||||
|
||||
for tmpl_name in tmpl_names:
|
||||
tmpl = jinja_env.get_template(tmpl_name)
|
||||
rendered = tmpl.render(**template_vars)
|
||||
out_name = tmpl_name.replace(".j2", "")
|
||||
dest = app_dir / out_name if tmpl_name == "app.yaml.j2" else manifests_dir / out_name
|
||||
dest.write_text(rendered)
|
||||
print(f" Rendered {tmpl_name} -> {dest}")
|
||||
templates = Path(action_dir) / "templates"
|
||||
jinja = Environment(loader=FileSystemLoader(str(templates)),
|
||||
keep_trailing_newline=True, undefined=StrictUndefined)
|
||||
app_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifests_dir.mkdir(parents=True, exist_ok=True)
|
||||
(app_dir / "app.yaml").write_text(
|
||||
jinja.get_template("app.yaml.j2").render(**template_vars), encoding="utf-8")
|
||||
(manifests_dir / "certificate.yaml").write_text(
|
||||
jinja.get_template("certificate.yaml.j2").render(**template_vars), encoding="utf-8")
|
||||
resources = ["certificate.yaml"]
|
||||
for route in template_vars["routes"]:
|
||||
values = {**template_vars, "route": route}
|
||||
for kind in ("service", "ingress"):
|
||||
filename = f"{kind}-{route['resource_name']}.yaml"
|
||||
(manifests_dir / filename).write_text(
|
||||
jinja.get_template(f"{kind}.yaml.j2").render(**values), encoding="utf-8")
|
||||
resources.append(filename)
|
||||
(manifests_dir / "kustomization.yaml").write_text(
|
||||
jinja.get_template("kustomization.yaml.j2").render(
|
||||
**template_vars, resources=resources), encoding="utf-8")
|
||||
|
||||
|
||||
def commit_and_push(apps_dir, message):
|
||||
run(f"git -C {apps_dir} add -A")
|
||||
result = subprocess.run(
|
||||
f"git -C {apps_dir} diff --cached --quiet",
|
||||
shell=True, check=False,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
def commit_and_push(apps_dir, message, token):
|
||||
user = env("CI_BOT_USER", "ci-bot")
|
||||
run(["git", "-C", str(apps_dir), "add", "-A"])
|
||||
clean = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"],
|
||||
check=False).returncode == 0
|
||||
if clean:
|
||||
print("No manifest changes to commit")
|
||||
return False
|
||||
run(f"git -C {apps_dir} commit -m '{message}'")
|
||||
run(f"git -C {apps_dir} push")
|
||||
run(["git", "-C", str(apps_dir), "commit", "-m", message])
|
||||
with git_auth(token, user) as auth:
|
||||
run(["git", "-C", str(apps_dir), "push"], **auth)
|
||||
print("Manifests pushed — ArgoCD will sync")
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user