"""Shared utilities for the site-publish action.""" import ipaddress import os import re import shutil import subprocess import sys from pathlib import Path, PurePosixPath from urllib.parse import urlparse import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined APPS_REPO = "fritzlab/apps" GITEA_HOST = "code.fritzlab.net" NAMESPACE = "websites" DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900" DEFAULT_WEBSITE_SUFFIX = "web.sjc001.fritzlab.net" DEFAULT_CACHE_CONTROL = "public, max-age=0, must-revalidate" EXCLUDE_FILES = { ".git", ".gitea", ".gitignore", "site.yaml", "build", ".site-publish", "Makefile", "README.md", "CLAUDE.md", "Dockerfile", ".dockerignore", "go.mod", "go.sum", } VALID_TYPES = {"static", "hugo", "mkdocs"} NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") ENV_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") ACCESS_KEY_ENV_RE = re.compile(r"^([A-Z][A-Z0-9_]*)_S3_ACCESS_KEY$") BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?$") MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9_-]{0,61}[A-Za-z0-9])?$") DOCKER_DEPRECATION_MSG = """\ type: docker is no longer supported by action/site-publish. site-publish handles only static-content sites (static, hugo, mkdocs) that ship to Garage S3. For containerized web apps, use the standard image-producer chain: - uses: action/image-build@v1 # build + smoke-test - uses: action/image-push@v1 # push + prune - uses: action/image-deploy@v1 # apps repo image-pin Hand-author your apps-repo manifests once (Deployment, Service, Ingress, Certificate, kustomization with images: block) under sjc001/websites//manifests/. image-deploy will pin the tag on every CI run. See action/image-deploy README and sjc001/websites/rainsounds.vino.network/manifests/ for the canonical example.\ """ class ConfigError(ValueError): """A site.yaml contract violation.""" def k8s_name(name): return name.replace(".", "-") def env(key, default=None): val = os.environ.get(key, default) if val is None or val == "": die(f"Missing required env var: {key}") return val def die(msg): print(f"ERROR: {msg}", file=sys.stderr) raise SystemExit(1) def run(cmd, *, display=None, **kwargs): """Run an argv command, printing only a safe display form.""" if not isinstance(cmd, (list, tuple)): raise TypeError("run() requires an argv list") shown = display if display is not None else " ".join(str(part) for part in cmd) print(f" $ {shown}") return subprocess.run(cmd, check=True, **kwargs) def _mapping(value, label): if not isinstance(value, dict): raise ConfigError(f"{label} must be a mapping") return value def _list(value, label): if not isinstance(value, list): raise ConfigError(f"{label} must be a list") return value def _known_keys(value, allowed, label): unknown = set(value) - set(allowed) if unknown: raise ConfigError(f"{label} has unknown fields: {', '.join(sorted(unknown))}") def _strings(value, label): values = _list(value or [], label) if any(not isinstance(item, str) or not item for item in values): raise ConfigError(f"{label} must be a list of non-empty strings") return values def _cors_origins(value, label): origins = _list(value, label) if any(not isinstance(item, str) or not item for item in origins): raise ConfigError(f"{label} must be a list of non-empty strings") canonical = [] for origin in origins: if origin == "*": canonical.append(origin) continue parsed = urlparse(origin) try: port = parsed.port except ValueError: raise ConfigError(f"{label} must contain '*' or canonical HTTPS origins") from None if parsed.scheme != "https" or not parsed.hostname or parsed.path or parsed.params or ( parsed.query or parsed.fragment or parsed.username or parsed.password ): raise ConfigError(f"{label} must contain '*' or canonical HTTPS origins") try: address = ipaddress.ip_address(parsed.hostname) except ValueError: try: hostname = parsed.hostname.encode("idna").decode("ascii") except UnicodeError: raise ConfigError( f"{label} must contain '*' or canonical HTTPS origins" ) from None try: _hostname(hostname, f"{label} hostname") except ConfigError: raise ConfigError( f"{label} must contain '*' or canonical HTTPS origins" ) from None else: hostname = f"[{address.compressed}]" if address.version == 6 else address.compressed canonical.append(f"https://{hostname}{f':{port}' if port not in (None, 443) else ''}") if len(canonical) != len(set(canonical)): raise ConfigError(f"{label} must not contain duplicate canonical origins") if "*" in canonical and len(canonical) != 1: raise ConfigError(f"{label} wildcard must be the only origin") if origins != canonical: raise ConfigError(f"{label} must contain '*' or canonical HTTPS origins") return origins def _hostname(value, label): if not isinstance(value, str) or len(value) > 253 or value.endswith("."): raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot") labels = value.split(".") if len(labels) < 2 or any(not NAME_RE.fullmatch(part) for part in labels): raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot") return value def _aliases(value, domain): aliases = _strings(value or [], "aliases") aliases = [_hostname(alias, "aliases entry") for alias in aliases] if len(aliases) != len(set(aliases)) or domain in aliases: raise ConfigError("aliases must be unique and cannot repeat domain") return aliases def _relative_path(value, label, *, allow_root=False): value = "" if value is None else value if not isinstance(value, str): raise ConfigError(f"{label} must be a string") if value == "/" and allow_root: return "" path = PurePosixPath(value) if path.is_absolute() or ".." in path.parts: raise ConfigError(f"{label} must be a relative path without '..'") normalized = str(path).strip("/") return "" if normalized == "." else normalized def _route_path(value, label): if not isinstance(value, str) or not value.startswith("/"): raise ConfigError(f"{label} must start with '/'") if "//" in value or "?" in value or "#" in value or ".." in value.split("/"): raise ConfigError(f"{label} is not a canonical URL path") normalized = value.rstrip("/") or "/" if not re.fullmatch(r"/[A-Za-z0-9._~/-]*", normalized): raise ConfigError(f"{label} contains unsupported URL path characters") return normalized def _middlewares(value, label): names = _strings(value, label) if any(not MIDDLEWARE_RE.fullmatch(name) for name in names): raise ConfigError(f"{label} contains an invalid middleware name") if len(names) != len(set(names)): raise ConfigError(f"{label} contains duplicate middleware names") return names def _cache_control(value, label): if not isinstance(value, str) or not value.strip(): raise ConfigError(f"{label} must be a non-empty Cache-Control value") directives = [part.strip().lower() for part in value.split(",")] names = [part.split("=", 1)[0] for part in directives] if len(names) != len(set(names)): raise ConfigError(f"{label} repeats a Cache-Control directive") present = set(names) if {"public", "private"} <= present: raise ConfigError(f"{label} cannot be both public and private") ages = {} for part in directives: name = part.split("=", 1)[0] if name in {"max-age", "s-maxage"}: raw = part.split("=", 1)[1] if "=" in part else "" if not raw.isdigit(): raise ConfigError(f"{label} {name} must be a non-negative integer") ages[name] = int(raw) max_age = ages.get("max-age") if "immutable" in present and (not max_age or present & {"no-store", "no-cache", "must-revalidate"}): raise ConfigError(f"{label} immutable requires positive max-age without revalidation") if "no-store" in present and any(ages.values()): raise ConfigError(f"{label} no-store contradicts positive caching") if "private" in present and ages.get("s-maxage"): raise ConfigError(f"{label} private contradicts shared-cache max-age") return ", ".join(part.strip() for part in value.split(",")) def _site_type(value, label): site_type = value or "static" if site_type == "docker": raise ConfigError(DOCKER_DEPRECATION_MSG) if site_type not in VALID_TYPES: raise ConfigError(f"Unknown {label}: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})") return site_type def _endpoint(value, label): parsed = urlparse(value) if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.path not in {"", "/"}: raise ConfigError(f"{label} must be an http(s) origin without a path") return value.rstrip("/") def _legacy_config(raw, site_name): if not isinstance(raw.get("tidy", True), bool): raise ConfigError("tidy must be a boolean") if not isinstance(raw.get("enabled", True), bool): raise ConfigError("enabled must be a boolean") artifact = { "name": "site", "type": _site_type(raw.get("type", "static"), "site type"), "content_dir": _relative_path(raw.get("content_dir", ""), "content_dir"), "tidy": raw.get("tidy", True), "excludes": _strings(raw.get("excludes") or [], "excludes"), "build_dir": "build/html", "bucket": site_name, "s3_endpoint": os.environ.get("GARAGE_S3_ENDPOINT") or DEFAULT_S3_ENDPOINT, "website_authority": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net", "credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", "secret_key_env": "AWS_SECRET_ACCESS_KEY"}, "cache_rules": [{"path": "", "cache_control": DEFAULT_CACHE_CONTROL}], "cors_origins": None, } return { "version": 1, "compatibility": "single-surface-v1", "domain": raw["domain"], "aliases": _aliases(raw.get("aliases"), raw["domain"]), "enabled": raw.get("enabled", True), "artifacts": [artifact], "routes": [{ "name": "site", "path": "/", "artifact": "site", "access": "legacy", "access_middleware": None, "middlewares": _middlewares(raw.get("middlewares") or [], "middlewares"), }], } def _artifact(item, index): label = f"artifacts[{index}]" item = _mapping(item, label) _known_keys( item, {"name", "type", "content_dir", "tidy", "excludes", "publish", "cache", "cors_origins"}, label, ) name = item.get("name") if not isinstance(name, str) or not NAME_RE.fullmatch(name): raise ConfigError(f"{label}.name must be a DNS label") publish = _mapping(item.get("publish"), f"{label}.publish") _known_keys(publish, {"bucket", "credentials"}, f"{label}.publish") bucket = publish.get("bucket") if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket): raise ConfigError(f"{label}.publish.bucket is not a valid bucket name") credentials = _mapping(publish.get("credentials"), f"{label}.publish.credentials") _known_keys(credentials, {"access_key_env", "secret_key_env"}, f"{label}.publish.credentials") normalized_credentials = {} for key in ("access_key_env", "secret_key_env"): value = credentials.get(key) if not isinstance(value, str) or not ENV_RE.fullmatch(value): raise ConfigError(f"{label}.publish.credentials.{key} must name an environment variable") normalized_credentials[key] = value access_match = ACCESS_KEY_ENV_RE.fullmatch(normalized_credentials["access_key_env"]) expected_secret = ( f"{access_match.group(1)}_S3_SECRET_KEY" if access_match else None ) if normalized_credentials["secret_key_env"] != expected_secret: raise ConfigError( f"{label}.publish.credentials must be a matched " "_S3_ACCESS_KEY and _S3_SECRET_KEY pair" ) cache = _mapping(item.get("cache"), f"{label}.cache") _known_keys(cache, {"rules"}, f"{label}.cache") rules = _list(cache.get("rules"), f"{label}.cache.rules") if not rules: raise ConfigError(f"{label}.cache.rules must declare a '/' default") cache_rules, paths = [], set() for rule_index, rule in enumerate(rules): rule_label = f"{label}.cache.rules[{rule_index}]" rule = _mapping(rule, rule_label) _known_keys(rule, {"path", "cache_control"}, rule_label) path = _relative_path(rule.get("path"), f"{rule_label}.path", allow_root=True) if path in paths: raise ConfigError(f"{label}.cache.rules has duplicate path /{path}") paths.add(path) cache_rules.append({"path": path, "cache_control": _cache_control( rule.get("cache_control"), f"{rule_label}.cache_control" )}) if "" not in paths: raise ConfigError(f"{label}.cache.rules must declare a '/' default") cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"])) immutable_paths = [rule["path"] for rule in cache_rules if "immutable" in {part.strip().lower().split("=", 1)[0] for part in rule["cache_control"].split(",")}] if "" in immutable_paths: raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'") for path in immutable_paths: if any(other.startswith(f"{path}/") for other in paths): raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}") authority = f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}" if not isinstance(item.get("tidy", True), bool): raise ConfigError(f"{label}.tidy must be a boolean") return { "name": name, "type": _site_type(item.get("type", "static"), f"{label}.type"), "content_dir": _relative_path(item.get("content_dir", ""), f"{label}.content_dir"), "tidy": item.get("tidy", True), "excludes": _strings(item.get("excludes") or [], f"{label}.excludes"), "build_dir": f".site-publish/{name}/html", "bucket": bucket, "s3_endpoint": DEFAULT_S3_ENDPOINT, "website_authority": authority, "credentials": normalized_credentials, "cache_rules": cache_rules, "cors_origins": ( _cors_origins(item["cors_origins"], f"{label}.cors_origins") if "cors_origins" in item else [] ), } def _route(item, index): label = f"routes[{index}]" item = _mapping(item, label) _known_keys(item, {"name", "path", "artifact", "access", "middlewares"}, label) name = item.get("name") if not isinstance(name, str) or not NAME_RE.fullmatch(name): raise ConfigError(f"{label}.name must be a DNS label") access = _mapping(item.get("access"), f"{label}.access") _known_keys(access, {"mode", "middleware"}, f"{label}.access") mode, middleware = access.get("mode"), access.get("middleware") if mode not in {"public", "protected"}: raise ConfigError(f"{label}.access.mode must be public or protected") if mode == "protected" and (not isinstance(middleware, str) or not MIDDLEWARE_RE.fullmatch(middleware)): raise ConfigError(f"{label}.access.middleware is required for protected access") if mode == "public" and middleware is not None: raise ConfigError(f"{label}.access.middleware is forbidden for public access") middlewares = _middlewares(item.get("middlewares") or [], f"{label}.middlewares") if middleware in middlewares: raise ConfigError(f"{label} repeats its access middleware") artifact = item.get("artifact") if not isinstance(artifact, str): raise ConfigError(f"{label}.artifact must name an artifact") return { "name": name, "path": _route_path(item.get("path"), f"{label}.path"), "artifact": artifact, "access": mode, "access_middleware": middleware, "middlewares": middlewares, } def _validate_multi(cfg): artifacts, routes = cfg["artifacts"], cfg["routes"] artifact_names = [item["name"] for item in artifacts] route_names = [item["name"] for item in routes] route_paths = [item["path"] for item in routes] if len(artifact_names) != len(set(artifact_names)): raise ConfigError("artifact names must be unique") if len(route_names) != len(set(route_names)): raise ConfigError("route names must be unique") if len(route_paths) != len(set(route_paths)): raise ConfigError("route paths are ambiguous after normalization") if "/" not in route_paths: raise ConfigError("routes must declare a '/' catch-all") artifact_by_name = {item["name"]: item for item in artifacts} references = {name: [] for name in artifact_by_name} for route in routes: if route["artifact"] not in artifact_by_name: raise ConfigError(f"route {route['name']} references unknown artifact {route['artifact']}") references[route["artifact"]].append(route) for name, used_by in references.items(): if len(used_by) != 1: raise ConfigError(f"artifact {name} must be referenced by exactly one route") buckets, authorities, credential_owners = {}, {}, {} for route in routes: artifact = artifact_by_name[route["artifact"]] if artifact["bucket"] in buckets: other_access, other_name = buckets[artifact["bucket"]] if other_access != route["access"]: raise ConfigError(f"bucket {artifact['bucket']} cannot be reused by protected and public routes") raise ConfigError(f"bucket {artifact['bucket']} must belong to one artifact ({other_name})") buckets[artifact["bucket"]] = (route["access"], artifact["name"]) if artifact["website_authority"] in authorities: raise ConfigError( f"website authority {artifact['website_authority']} must belong to one artifact" ) authorities[artifact["website_authority"]] = artifact["name"] for variable in artifact["credentials"].values(): if variable in credential_owners: raise ConfigError( f"publication credential {variable} is reused by artifacts " f"{credential_owners[variable]} and {artifact['name']}" ) credential_owners[variable] = artifact["name"] for cache_rule in artifact["cache_rules"]: directives = {part.strip().lower().split("=", 1)[0] for part in cache_rule["cache_control"].split(",")} if route["access"] == "protected" and "public" in directives: raise ConfigError(f"protected route {route['name']} cannot use public cache policy") if route["access"] == "protected" and not directives & {"private", "no-store"}: raise ConfigError(f"protected route {route['name']} cache policy must be private or no-store") if route["access"] == "protected" and "s-maxage" in directives: raise ConfigError(f"protected route {route['name']} cannot use shared-cache max-age") if route["access"] == "public" and "private" in directives: raise ConfigError(f"public route {route['name']} cannot use private cache policy") if route["access"] == "protected" and "*" in artifact["cors_origins"]: raise ConfigError( f"protected route {route['name']} cannot allow wildcard CORS" ) root = next(route for route in routes if route["path"] == "/") if any(route["access"] == "protected" for route in routes) and root["access"] == "public": raise ConfigError("a public '/' catch-all would expose unmatched protected content") def normalize_site_config(raw, site_name): raw = _mapping(raw, "site.yaml") domain = _hostname(raw.get("domain"), "domain") if not isinstance(raw.get("enabled", True), bool): raise ConfigError("enabled must be a boolean") if ("artifacts" in raw) != ("routes" in raw): raise ConfigError("artifacts and routes must be declared together") if "artifacts" not in raw: return _legacy_config(raw, site_name) legacy_fields = {"type", "content_dir", "tidy", "excludes", "middlewares"} & set(raw) if legacy_fields: raise ConfigError("legacy fields cannot be mixed with artifacts/routes: " + ", ".join(sorted(legacy_fields))) _known_keys(raw, {"domain", "aliases", "enabled", "artifacts", "routes"}, "site.yaml") artifacts = [_artifact(item, index) for index, item in enumerate(_list(raw["artifacts"], "artifacts"))] routes = [_route(item, index) for index, item in enumerate(_list(raw["routes"], "routes"))] if not artifacts or not routes: raise ConfigError("artifacts and routes must not be empty") cfg = { "version": 2, "compatibility": None, "domain": domain, "aliases": _aliases(raw.get("aliases"), domain), "enabled": raw.get("enabled", True), "artifacts": sorted(artifacts, key=lambda item: item["name"]), "routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])), } _validate_multi(cfg) for route in cfg["routes"]: if len(f"{k8s_name(site_name)}-{route['name']}") > 63: raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters") return cfg def validate_artifact_inputs(site_dir, cfg): """Reject source containment before a public or protected build starts.""" if cfg["compatibility"]: return root = Path(site_dir).resolve() sources = [] for artifact in cfg["artifacts"]: declared = root for component in Path(artifact["content_dir"]).parts: declared /= component if declared.is_symlink(): raise ConfigError( f"artifact {artifact['name']} content_dir contains symlink component: " f"{declared.relative_to(root)}" ) source = declared.resolve() if source != root and root not in source.parents: raise ConfigError( f"artifact {artifact['name']} content_dir resolves outside the repository" ) if source.exists(): symlink = next((path for path in source.rglob("*") if path.is_symlink()), None) if symlink is not None: raise ConfigError( f"artifact {artifact['name']} build input contains symlink: " f"{symlink.relative_to(root)}" ) sources.append((artifact["name"], source)) for index, (name, source) in enumerate(sources): for other_name, other_source in sources[index + 1:]: if source == other_source or source in other_source.parents or other_source in source.parents: raise ConfigError( f"artifact build inputs overlap after resolution: {name} and {other_name}" ) def parse_site_yaml(site_dir, site_name=None): path = Path(site_dir) / "site.yaml" if not path.exists(): die("site.yaml not found in repo root") if site_name is None: repo = os.environ.get("SITE_REPO", "") site_name = repo.split("/", 1)[-1] if "/" in repo else Path(site_dir).name try: with open(path) as stream: cfg = normalize_site_config(yaml.safe_load(stream), site_name) except (ConfigError, yaml.YAMLError) as exc: die(str(exc)) print("Site config:") print(f" domain: {cfg['domain']}") print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}") print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}") print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}") return cfg def clone_apps(token): """Clone Apps without placing the credential in argv or output.""" user = env("CI_BOT_USER", "ci-bot") apps_dir = Path("/tmp/apps-deploy") if apps_dir.exists(): shutil.rmtree(apps_dir) clone_env = git_auth_env(token) url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git" run(["git", "clone", "--depth", "1", url, str(apps_dir)], display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env) run(["git", "-C", str(apps_dir), "config", "user.name", user]) run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"]) return apps_dir def render_templates(action_dir, template_vars, app_dir, manifests_dir): """Render one certificate and deterministic per-route resources.""" jinja_env = Environment(loader=FileSystemLoader(str(Path(action_dir) / "templates")), keep_trailing_newline=True, undefined=StrictUndefined) manifests_dir.mkdir(parents=True, exist_ok=True) for child in manifests_dir.iterdir(): if child.is_file() and child.suffix in {".yaml", ".yml"}: child.unlink() route_files = [] for route in template_vars["routes"]: stem = "" if template_vars["compatibility"] else f"-{route['name']}" for kind in ("service", "ingress"): out_name = f"{kind}{stem}.yaml" destination = manifests_dir / out_name destination.write_text(jinja_env.get_template(f"{kind}.yaml.j2").render( **template_vars, route=route )) route_files.append(out_name) print(f" Rendered {kind}.yaml.j2 -> {destination}") common_vars = {**template_vars, "route_files": route_files} for out_name, destination in { "certificate.yaml": manifests_dir / "certificate.yaml", "kustomization.yaml": manifests_dir / "kustomization.yaml", "app.yaml": app_dir / "app.yaml", }.items(): destination.write_text(jinja_env.get_template(f"{out_name}.j2").render(**common_vars)) print(f" Rendered {out_name}.j2 -> {destination}") def git_auth_env(token): """Return credential-safe Git authentication shared by clone and push.""" auth_env = os.environ.copy() auth_env["CI_BOT_TOKEN"] = token auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh")) auth_env["GIT_TERMINAL_PROMPT"] = "0" return auth_env def commit_and_push(apps_dir, message, token=None): run(["git", "-C", str(apps_dir), "add", "-A"]) result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False) if result.returncode == 0: print("No manifest changes to commit") return False run(["git", "-C", str(apps_dir), "commit", "-m", message]) push_env = git_auth_env(token) if token else None run(["git", "-C", str(apps_dir), "push"], env=push_env) print("Manifests pushed — ArgoCD will sync") return True