Authored-By: OpenAI (GPT-5) <noreply@openai.com>
This commit is contained in:
+26
-21
@@ -1,25 +1,25 @@
|
||||
"""Build phase — content prep for static-content sites."""
|
||||
"""Build each declared static-content artifact independently."""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from utils import EXCLUDE_FILES, env, parse_site_yaml, run
|
||||
from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs
|
||||
|
||||
|
||||
def build_static(site_dir, cfg):
|
||||
build_dir = site_dir / "build"
|
||||
html_dir = build_dir / "html"
|
||||
def build_artifact(site_dir, artifact):
|
||||
html_dir = site_dir / artifact["build_dir"]
|
||||
if html_dir.parent.exists():
|
||||
shutil.rmtree(html_dir.parent)
|
||||
|
||||
if build_dir.exists():
|
||||
shutil.rmtree(build_dir)
|
||||
|
||||
content_dir = cfg["content_dir"]
|
||||
content_dir = artifact["content_dir"]
|
||||
src = site_dir / content_dir if content_dir else site_dir
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(f"artifact {artifact['name']} content_dir not found: {src}")
|
||||
|
||||
if cfg["type"] == "static":
|
||||
print(f"Copying static content from {src}")
|
||||
if artifact["type"] == "static":
|
||||
print(f"Copying artifact {artifact['name']} from {src}")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp) / "html"
|
||||
shutil.copytree(src, tmp_path, dirs_exist_ok=True)
|
||||
@@ -29,18 +29,18 @@ def build_static(site_dir, cfg):
|
||||
shutil.rmtree(p)
|
||||
elif p.exists():
|
||||
p.unlink()
|
||||
build_dir.mkdir(parents=True)
|
||||
html_dir.parent.mkdir(parents=True)
|
||||
shutil.move(str(tmp_path), str(html_dir))
|
||||
|
||||
elif cfg["type"] == "hugo":
|
||||
print(f"Building Hugo site from {src}")
|
||||
run(f"hugo --source {src} --destination {html_dir}")
|
||||
elif artifact["type"] == "hugo":
|
||||
print(f"Building Hugo artifact {artifact['name']} from {src}")
|
||||
run(["hugo", "--source", str(src), "--destination", str(html_dir)])
|
||||
|
||||
elif cfg["type"] == "mkdocs":
|
||||
print(f"Building MkDocs site from {src}")
|
||||
run(f"cd {src} && mkdocs build -d {html_dir}")
|
||||
elif artifact["type"] == "mkdocs":
|
||||
print(f"Building MkDocs artifact {artifact['name']} from {src}")
|
||||
run(["mkdocs", "build", "-d", str(html_dir)], cwd=src)
|
||||
|
||||
if cfg.get("tidy", True):
|
||||
if artifact["tidy"]:
|
||||
print("Running tidy on HTML files...")
|
||||
for html_file in html_dir.rglob("*.html"):
|
||||
subprocess.run(
|
||||
@@ -51,7 +51,9 @@ def build_static(site_dir, cfg):
|
||||
check=False,
|
||||
)
|
||||
|
||||
print(f"Build complete — content at {html_dir}")
|
||||
if not any(path.is_file() for path in html_dir.rglob("*")):
|
||||
raise FileNotFoundError(f"artifact {artifact['name']} produced no files in {html_dir}")
|
||||
print(f"Artifact {artifact['name']} complete — content at {html_dir}")
|
||||
|
||||
|
||||
def cmd_build():
|
||||
@@ -62,4 +64,7 @@ def cmd_build():
|
||||
print("Site disabled — skipping build")
|
||||
return
|
||||
|
||||
build_static(site_dir, cfg)
|
||||
validate_artifact_inputs(site_dir, cfg)
|
||||
|
||||
for artifact in cfg["artifacts"]:
|
||||
build_artifact(site_dir, artifact)
|
||||
|
||||
+215
-58
@@ -1,17 +1,18 @@
|
||||
"""Deploy phase — S3 sync, manifest rendering, alias reconcile."""
|
||||
|
||||
import fnmatch
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import shlex
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from utils import (
|
||||
DEFAULT_S3_ENDPOINT,
|
||||
GITEA_HOST,
|
||||
NAMESPACE,
|
||||
clone_apps,
|
||||
commit_and_push,
|
||||
@@ -21,6 +22,7 @@ from utils import (
|
||||
parse_site_yaml,
|
||||
render_templates,
|
||||
run,
|
||||
validate_artifact_inputs,
|
||||
)
|
||||
|
||||
GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
||||
@@ -28,44 +30,188 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
||||
)
|
||||
|
||||
|
||||
CACHE_CONTROL = "public, max-age=0, must-revalidate"
|
||||
def validate_artifact_output(site_dir, artifact):
|
||||
"""Prove every artifact and declared cache prefix exists before publishing any."""
|
||||
html_dir = site_dir / artifact["build_dir"]
|
||||
if not html_dir.is_dir() or not any(path.is_file() for path in html_dir.rglob("*")):
|
||||
die(f"artifact {artifact['name']} build output is absent or empty: {html_dir}")
|
||||
for rule in artifact["cache_rules"]:
|
||||
if not rule["path"]:
|
||||
continue
|
||||
cache_root = html_dir / rule["path"]
|
||||
if not cache_root.exists() or not any(path.is_file() for path in cache_root.rglob("*")):
|
||||
die(f"artifact {artifact['name']} cache path /{rule['path']} has no built files")
|
||||
|
||||
|
||||
def s3_sync(site_name, site_dir, excludes=None):
|
||||
endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
|
||||
html_dir = site_dir / "build" / "html"
|
||||
if not html_dir.exists():
|
||||
die(f"build/html not found — did the build step run? ({html_dir})")
|
||||
env("AWS_ACCESS_KEY_ID")
|
||||
env("AWS_SECRET_ACCESS_KEY")
|
||||
os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
|
||||
def validate_publication_environment(cfg):
|
||||
"""Resolve every declared credential before the first bucket is changed."""
|
||||
for artifact in cfg["artifacts"]:
|
||||
env(artifact["credentials"]["access_key_env"])
|
||||
env(artifact["credentials"]["secret_key_env"])
|
||||
if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
|
||||
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
|
||||
|
||||
|
||||
def _is_immutable(rule):
|
||||
return "immutable" in {
|
||||
part.strip().lower().split("=", 1)[0]
|
||||
for part in rule["cache_control"].split(",")
|
||||
}
|
||||
|
||||
|
||||
def _aws_capture(args, aws_env):
|
||||
"""Run a non-streaming AWS request without exposing environment credentials."""
|
||||
print(f" $ {' '.join(str(part) for part in args)}")
|
||||
return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
|
||||
|
||||
|
||||
def _immutable_head(endpoint, bucket, key, aws_env):
|
||||
args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
|
||||
"--bucket", bucket, "--key", key, "--output", "json"]
|
||||
result = _aws_capture(args, aws_env)
|
||||
if result.returncode == 0:
|
||||
return json.loads(result.stdout)
|
||||
error = f"{result.stdout}\n{result.stderr}"
|
||||
if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
|
||||
return None
|
||||
raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
|
||||
|
||||
|
||||
def _immutable_digests(source, cache_control, content_type):
|
||||
with source.open("rb") as stream:
|
||||
content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
publication = hashlib.sha256()
|
||||
publication.update(cache_control.encode())
|
||||
publication.update(b"\0")
|
||||
publication.update(content_type.encode())
|
||||
publication.update(b"\0")
|
||||
with source.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
publication.update(block)
|
||||
return content_digest, publication.hexdigest()
|
||||
|
||||
|
||||
def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
|
||||
metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
|
||||
return (
|
||||
metadata.get("sha256") == content_digest
|
||||
and metadata.get("publication-sha256") == publication_digest
|
||||
and info.get("CacheControl") == cache_control
|
||||
and info.get("ContentType") == content_type
|
||||
)
|
||||
|
||||
|
||||
def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
|
||||
"""Publish a content-addressed key; identical retries converge."""
|
||||
content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
|
||||
content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
|
||||
address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
|
||||
if address_digests != [publication_digest]:
|
||||
raise RuntimeError(
|
||||
f"immutable key must contain its one publication SHA-256 {publication_digest}: "
|
||||
f"s3://{bucket}/{key}"
|
||||
)
|
||||
existing = _immutable_head(endpoint, bucket, key, aws_env)
|
||||
if existing is not None:
|
||||
if _same_immutable_object(
|
||||
existing, content_digest, publication_digest, cache_control, content_type,
|
||||
):
|
||||
print(f" Immutable object already matches: s3://{bucket}/{key}")
|
||||
return False
|
||||
raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
|
||||
args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
|
||||
"--bucket", bucket, "--key", key, "--body", str(source),
|
||||
"--content-type", content_type, "--cache-control", cache_control,
|
||||
"--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
|
||||
result = _aws_capture(args, aws_env)
|
||||
if result.returncode == 0:
|
||||
return True
|
||||
error = f"{result.stdout}\n{result.stderr}"
|
||||
raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
|
||||
|
||||
|
||||
def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
|
||||
"""Publish one immutable cache partition without overwrite or deletion."""
|
||||
rule_root = html_dir / rule["path"]
|
||||
child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
|
||||
if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
|
||||
object_prefix = route["path"].strip("/")
|
||||
for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
|
||||
relative = source.relative_to(html_dir).as_posix()
|
||||
if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
|
||||
continue
|
||||
if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
|
||||
continue
|
||||
key = "/".join(part for part in (object_prefix, relative) if part)
|
||||
publish_immutable_file(
|
||||
artifact["s3_endpoint"], artifact["bucket"], key, source,
|
||||
rule["cache_control"], aws_env,
|
||||
)
|
||||
|
||||
|
||||
def s3_sync(artifact, route, site_dir, credential_env_names=None):
|
||||
endpoint = artifact["s3_endpoint"]
|
||||
html_dir = site_dir / artifact["build_dir"]
|
||||
access_key = env(artifact["credentials"]["access_key_env"])
|
||||
secret_key = env(artifact["credentials"]["secret_key_env"])
|
||||
aws_env = os.environ.copy()
|
||||
for name in credential_env_names or artifact["credentials"].values():
|
||||
aws_env.pop(name, None)
|
||||
for name in ("CI_BOT_TOKEN", "GARAGE_ADMIN_TOKEN", "AWS_PROFILE",
|
||||
"AWS_SHARED_CREDENTIALS_FILE", "AWS_SESSION_TOKEN"):
|
||||
aws_env.pop(name, None)
|
||||
aws_env.update({
|
||||
"AWS_ACCESS_KEY_ID": access_key,
|
||||
"AWS_SECRET_ACCESS_KEY": secret_key,
|
||||
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
|
||||
})
|
||||
bucket = artifact["bucket"]
|
||||
object_prefix = route["path"].strip("/")
|
||||
destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}"
|
||||
default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
|
||||
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
|
||||
# `excludes` are patterns (site.yaml `excludes:` list) that should never
|
||||
# be uploaded *and* should never be deleted from the bucket — escape hatch
|
||||
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
|
||||
exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
|
||||
if excludes:
|
||||
print(f"Excluding patterns: {excludes}")
|
||||
print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
|
||||
# `sync --delete` handles new/changed/orphaned files. `cp --recursive`
|
||||
# then re-uploads everything to refresh metadata (cache-control,
|
||||
# content-type) on objects sync skipped because nothing changed.
|
||||
# Cost: a no-op deploy still re-uploads every byte. Sites here are
|
||||
# small enough that that's free; correctness wins over throughput.
|
||||
exclude_args = [arg for pattern in artifact["excludes"] for arg in ("--exclude", pattern)]
|
||||
if artifact["excludes"]:
|
||||
print(f"Excluding patterns: {artifact['excludes']}")
|
||||
print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
|
||||
# Sync and deletion are scoped to the current route prefix. A route move
|
||||
# leaves its old bucket partition intact but unreachable after the old
|
||||
# Ingress disappears, while stale mutable keys on the serving prefix are
|
||||
# deleted. Immutable subtrees are structurally excluded. Partitioned
|
||||
# `cp --recursive` calls then re-upload each file once to refresh metadata
|
||||
# (cache-control, content-type) on objects sync skipped as unchanged.
|
||||
# A no-op deploy therefore transfers the artifact bytes once.
|
||||
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
|
||||
# so a fresh upload always carries the right MIME type.
|
||||
run(
|
||||
f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
|
||||
f"--delete --only-show-errors "
|
||||
f"--cache-control '{CACHE_CONTROL}' "
|
||||
f"{exclude_flags}".rstrip()
|
||||
)
|
||||
sync_excludes = [*exclude_args,
|
||||
*(arg for path in immutable_paths for arg in ("--exclude", f"{path}/*"))]
|
||||
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
|
||||
"--delete", "--only-show-errors", "--cache-control", default_cache,
|
||||
*sync_excludes], env=aws_env)
|
||||
print("Re-stamping metadata on all objects...")
|
||||
run(
|
||||
f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
|
||||
f"--recursive --only-show-errors "
|
||||
f"--cache-control '{CACHE_CONTROL}' "
|
||||
f"{exclude_flags}".rstrip()
|
||||
)
|
||||
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
|
||||
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
|
||||
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
|
||||
"--recursive", "--only-show-errors", "--cache-control", default_cache,
|
||||
*default_filters, *exclude_args], env=aws_env)
|
||||
for rule in artifact["cache_rules"]:
|
||||
if not rule["path"]:
|
||||
continue
|
||||
if _is_immutable(rule):
|
||||
publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
|
||||
continue
|
||||
include = f"{rule['path'].rstrip('/')}/*"
|
||||
child_filters = [arg for path in specific_paths
|
||||
if path.startswith(f"{rule['path'].rstrip('/')}/")
|
||||
for arg in ("--exclude", f"{path}/*")]
|
||||
# Apply rules from the artifact root so artifact-level exclusions keep
|
||||
# their original meaning under every cache override.
|
||||
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
|
||||
"--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
|
||||
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
|
||||
|
||||
|
||||
def garage_admin(method, path, token, body=None):
|
||||
@@ -89,15 +235,13 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
|
||||
if not aliases:
|
||||
return
|
||||
if not admin_token:
|
||||
print(" (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
|
||||
return
|
||||
die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
|
||||
|
||||
try:
|
||||
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
|
||||
admin_token)
|
||||
except (HTTPError, URLError) as e:
|
||||
print(f" WARNING: bucket lookup failed: {e}")
|
||||
return
|
||||
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
|
||||
|
||||
bucket_id = info.get("id")
|
||||
existing = set(info.get("globalAliases") or [])
|
||||
@@ -120,20 +264,36 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
|
||||
"""Always re-render manifests from current site.yaml. Templates own
|
||||
domain + aliases, so changes propagate without manual edits."""
|
||||
manifests_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
||||
routes = []
|
||||
for route in cfg["routes"]:
|
||||
artifact = artifact_by_name[route["artifact"]]
|
||||
resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
|
||||
routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
|
||||
template_vars = {
|
||||
"site": site_name,
|
||||
"site_k8s": k8s_name(site_name),
|
||||
"domain": cfg["domain"],
|
||||
"aliases": cfg["aliases"],
|
||||
"namespace": NAMESPACE,
|
||||
"middlewares": cfg["middlewares"],
|
||||
"compatibility": cfg["compatibility"],
|
||||
"routes": routes,
|
||||
}
|
||||
render_templates(action_dir, template_vars, app_dir, manifests_dir)
|
||||
|
||||
|
||||
def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
||||
s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
|
||||
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
|
||||
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
||||
credential_env_names = {
|
||||
name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
|
||||
}
|
||||
validate_publication_environment(cfg)
|
||||
for artifact in cfg["artifacts"]:
|
||||
validate_artifact_output(site_dir, artifact)
|
||||
for route in cfg["routes"]:
|
||||
s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
|
||||
if cfg["compatibility"]:
|
||||
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
|
||||
|
||||
apps_dir = clone_apps(token)
|
||||
app_dir = apps_dir / "sjc001" / "websites" / site_name
|
||||
@@ -141,25 +301,21 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
||||
|
||||
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
|
||||
|
||||
commit_and_push(apps_dir, f"Deploy {site_name}")
|
||||
commit_and_push(apps_dir, f"Deploy {site_name}", token)
|
||||
|
||||
|
||||
def decommission(site_name, token):
|
||||
def decommission(site_name, token, buckets=None):
|
||||
"""Remove manifests from apps repo."""
|
||||
user = env("CI_BOT_USER", "ci-bot")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
apps_dir = Path(tmp)
|
||||
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
|
||||
site_path = apps_dir / "sjc001" / "websites" / site_name
|
||||
if not site_path.exists():
|
||||
print(f"No manifests for {site_name} — nothing to remove")
|
||||
return
|
||||
shutil.rmtree(site_path)
|
||||
run(f"git -C {apps_dir} config user.name {user}")
|
||||
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
|
||||
commit_and_push(apps_dir, f"Decommission {site_name}")
|
||||
print(f"Bucket {site_name} and its objects are NOT purged automatically.")
|
||||
print(f" garage bucket delete {site_name} --yes")
|
||||
apps_dir = clone_apps(token)
|
||||
site_path = apps_dir / "sjc001" / "websites" / site_name
|
||||
if not site_path.exists():
|
||||
print(f"No manifests for {site_name} — nothing to remove")
|
||||
return
|
||||
shutil.rmtree(site_path)
|
||||
commit_and_push(apps_dir, f"Decommission {site_name}", token)
|
||||
for bucket in buckets or [site_name]:
|
||||
print(f"Bucket {bucket} and its objects are NOT purged automatically.")
|
||||
print(f" garage bucket delete {bucket} --yes")
|
||||
|
||||
|
||||
def cmd_deploy():
|
||||
@@ -173,7 +329,8 @@ def cmd_deploy():
|
||||
|
||||
if not cfg["enabled"]:
|
||||
print("Site disabled — running decommission...")
|
||||
decommission(site_name, token)
|
||||
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
|
||||
return
|
||||
|
||||
validate_artifact_inputs(site_dir, cfg)
|
||||
deploy_static(site_name, site_dir, action_dir, token, cfg)
|
||||
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
case "$1" in
|
||||
*Username*) printf '%s\n' "$CI_BOT_USER" ;;
|
||||
*) printf '%s\n' "$CI_BOT_TOKEN" ;;
|
||||
esac
|
||||
+3
-2
@@ -24,9 +24,10 @@ def ensure_aws():
|
||||
subprocess.run(["aws", "--version"], check=True)
|
||||
|
||||
|
||||
def ensure_jinja2():
|
||||
def ensure_python_dependencies():
|
||||
try:
|
||||
import jinja2
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("Installing jinja2 + pyyaml...")
|
||||
subprocess.run(
|
||||
@@ -36,6 +37,6 @@ def ensure_jinja2():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ensure_jinja2()
|
||||
ensure_python_dependencies()
|
||||
ensure_aws()
|
||||
print("Setup complete")
|
||||
|
||||
+445
-69
@@ -1,26 +1,34 @@
|
||||
"""Shared utilities for the site-publish action."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
||||
|
||||
APPS_REPO = "fritzlab/apps"
|
||||
GITEA_HOST = "code.fritzlab.net"
|
||||
NAMESPACE = "websites"
|
||||
DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
|
||||
DEFAULT_WEBSITE_SUFFIX = "web.sjc001.fritzlab.net"
|
||||
DEFAULT_CACHE_CONTROL = "public, max-age=0, must-revalidate"
|
||||
|
||||
EXCLUDE_FILES = {
|
||||
".git", ".gitea", ".gitignore", "site.yaml",
|
||||
"build", "Makefile", "README.md", "CLAUDE.md",
|
||||
"Dockerfile", ".dockerignore", "go.mod", "go.sum",
|
||||
".git", ".gitea", ".gitignore", "site.yaml", "build", ".site-publish",
|
||||
"Makefile", "README.md", "CLAUDE.md", "Dockerfile", ".dockerignore",
|
||||
"go.mod", "go.sum",
|
||||
}
|
||||
|
||||
VALID_TYPES = {"static", "hugo", "mkdocs"}
|
||||
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
||||
ENV_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
||||
ACCESS_KEY_ENV_RE = re.compile(r"^([A-Z][A-Z0-9_]*)_S3_ACCESS_KEY$")
|
||||
BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?$")
|
||||
MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9_-]{0,61}[A-Za-z0-9])?$")
|
||||
|
||||
DOCKER_DEPRECATION_MSG = """\
|
||||
type: docker is no longer supported by action/site-publish.
|
||||
@@ -42,113 +50,481 @@ example.\
|
||||
"""
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""A site.yaml contract violation."""
|
||||
|
||||
|
||||
def k8s_name(name):
|
||||
"""Sanitize for DNS-1035 label (dots → dashes)."""
|
||||
return name.replace(".", "-")
|
||||
|
||||
|
||||
def env(key, default=None):
|
||||
val = os.environ.get(key, default)
|
||||
if val is None:
|
||||
if val is None or val == "":
|
||||
die(f"Missing required env var: {key}")
|
||||
return val
|
||||
|
||||
|
||||
def die(msg):
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
print(f" $ {cmd}")
|
||||
return subprocess.run(cmd, shell=True, check=True, **kwargs)
|
||||
def run(cmd, *, display=None, **kwargs):
|
||||
"""Run an argv command, printing only a safe display form."""
|
||||
if not isinstance(cmd, (list, tuple)):
|
||||
raise TypeError("run() requires an argv list")
|
||||
shown = display if display is not None else " ".join(str(part) for part in cmd)
|
||||
print(f" $ {shown}")
|
||||
return subprocess.run(cmd, check=True, **kwargs)
|
||||
|
||||
|
||||
def parse_site_yaml(site_dir):
|
||||
path = Path(site_dir) / "site.yaml"
|
||||
if not path.exists():
|
||||
die("site.yaml not found in repo root")
|
||||
def _mapping(value, label):
|
||||
if not isinstance(value, dict):
|
||||
raise ConfigError(f"{label} must be a mapping")
|
||||
return value
|
||||
|
||||
with open(path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
|
||||
if not cfg.get("domain"):
|
||||
die("domain is required in site.yaml")
|
||||
def _list(value, label):
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError(f"{label} must be a list")
|
||||
return value
|
||||
|
||||
site_type = cfg.get("type", "static")
|
||||
|
||||
def _known_keys(value, allowed, label):
|
||||
unknown = set(value) - set(allowed)
|
||||
if unknown:
|
||||
raise ConfigError(f"{label} has unknown fields: {', '.join(sorted(unknown))}")
|
||||
|
||||
|
||||
def _strings(value, label):
|
||||
values = _list(value or [], label)
|
||||
if any(not isinstance(item, str) or not item for item in values):
|
||||
raise ConfigError(f"{label} must be a list of non-empty strings")
|
||||
return values
|
||||
|
||||
|
||||
def _hostname(value, label):
|
||||
if not isinstance(value, str) or len(value) > 253 or value.endswith("."):
|
||||
raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot")
|
||||
labels = value.split(".")
|
||||
if len(labels) < 2 or any(not NAME_RE.fullmatch(part) for part in labels):
|
||||
raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot")
|
||||
return value
|
||||
|
||||
|
||||
def _aliases(value, domain):
|
||||
aliases = _strings(value or [], "aliases")
|
||||
aliases = [_hostname(alias, "aliases entry") for alias in aliases]
|
||||
if len(aliases) != len(set(aliases)) or domain in aliases:
|
||||
raise ConfigError("aliases must be unique and cannot repeat domain")
|
||||
return aliases
|
||||
|
||||
|
||||
def _relative_path(value, label, *, allow_root=False):
|
||||
value = "" if value is None else value
|
||||
if not isinstance(value, str):
|
||||
raise ConfigError(f"{label} must be a string")
|
||||
if value == "/" and allow_root:
|
||||
return ""
|
||||
path = PurePosixPath(value)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise ConfigError(f"{label} must be a relative path without '..'")
|
||||
normalized = str(path).strip("/")
|
||||
return "" if normalized == "." else normalized
|
||||
|
||||
|
||||
def _route_path(value, label):
|
||||
if not isinstance(value, str) or not value.startswith("/"):
|
||||
raise ConfigError(f"{label} must start with '/'")
|
||||
if "//" in value or "?" in value or "#" in value or ".." in value.split("/"):
|
||||
raise ConfigError(f"{label} is not a canonical URL path")
|
||||
normalized = value.rstrip("/") or "/"
|
||||
if not re.fullmatch(r"/[A-Za-z0-9._~/-]*", normalized):
|
||||
raise ConfigError(f"{label} contains unsupported URL path characters")
|
||||
return normalized
|
||||
|
||||
|
||||
def _middlewares(value, label):
|
||||
names = _strings(value, label)
|
||||
if any(not MIDDLEWARE_RE.fullmatch(name) for name in names):
|
||||
raise ConfigError(f"{label} contains an invalid middleware name")
|
||||
if len(names) != len(set(names)):
|
||||
raise ConfigError(f"{label} contains duplicate middleware names")
|
||||
return names
|
||||
|
||||
|
||||
def _cache_control(value, label):
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ConfigError(f"{label} must be a non-empty Cache-Control value")
|
||||
directives = [part.strip().lower() for part in value.split(",")]
|
||||
names = [part.split("=", 1)[0] for part in directives]
|
||||
if len(names) != len(set(names)):
|
||||
raise ConfigError(f"{label} repeats a Cache-Control directive")
|
||||
present = set(names)
|
||||
if {"public", "private"} <= present:
|
||||
raise ConfigError(f"{label} cannot be both public and private")
|
||||
ages = {}
|
||||
for part in directives:
|
||||
name = part.split("=", 1)[0]
|
||||
if name in {"max-age", "s-maxage"}:
|
||||
raw = part.split("=", 1)[1] if "=" in part else ""
|
||||
if not raw.isdigit():
|
||||
raise ConfigError(f"{label} {name} must be a non-negative integer")
|
||||
ages[name] = int(raw)
|
||||
max_age = ages.get("max-age")
|
||||
if "immutable" in present and (not max_age or present & {"no-store", "no-cache", "must-revalidate"}):
|
||||
raise ConfigError(f"{label} immutable requires positive max-age without revalidation")
|
||||
if "no-store" in present and any(ages.values()):
|
||||
raise ConfigError(f"{label} no-store contradicts positive caching")
|
||||
if "private" in present and ages.get("s-maxage"):
|
||||
raise ConfigError(f"{label} private contradicts shared-cache max-age")
|
||||
return ", ".join(part.strip() for part in value.split(","))
|
||||
|
||||
|
||||
def _site_type(value, label):
|
||||
site_type = value or "static"
|
||||
if site_type == "docker":
|
||||
die(DOCKER_DEPRECATION_MSG)
|
||||
|
||||
raise ConfigError(DOCKER_DEPRECATION_MSG)
|
||||
if site_type not in VALID_TYPES:
|
||||
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
|
||||
raise ConfigError(f"Unknown {label}: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
|
||||
return site_type
|
||||
|
||||
excludes = cfg.get("excludes") or []
|
||||
if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes):
|
||||
die("excludes must be a list of string patterns")
|
||||
|
||||
middlewares = cfg.get("middlewares") or []
|
||||
if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares):
|
||||
die("middlewares must be a list of Traefik file-provider middleware names")
|
||||
def _endpoint(value, label):
|
||||
parsed = urlparse(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.path not in {"", "/"}:
|
||||
raise ConfigError(f"{label} must be an http(s) origin without a path")
|
||||
return value.rstrip("/")
|
||||
|
||||
site = {
|
||||
"domain": cfg["domain"],
|
||||
"type": site_type,
|
||||
"enabled": cfg.get("enabled", True),
|
||||
"aliases": cfg.get("aliases") or [],
|
||||
"content_dir": cfg.get("content_dir", ""),
|
||||
"tidy": cfg.get("tidy", True),
|
||||
"excludes": excludes,
|
||||
|
||||
def _legacy_config(raw, site_name):
|
||||
if not isinstance(raw.get("tidy", True), bool):
|
||||
raise ConfigError("tidy must be a boolean")
|
||||
if not isinstance(raw.get("enabled", True), bool):
|
||||
raise ConfigError("enabled must be a boolean")
|
||||
artifact = {
|
||||
"name": "site",
|
||||
"type": _site_type(raw.get("type", "static"), "site type"),
|
||||
"content_dir": _relative_path(raw.get("content_dir", ""), "content_dir"),
|
||||
"tidy": raw.get("tidy", True),
|
||||
"excludes": _strings(raw.get("excludes") or [], "excludes"),
|
||||
"build_dir": "build/html",
|
||||
"bucket": site_name,
|
||||
"s3_endpoint": os.environ.get("GARAGE_S3_ENDPOINT") or DEFAULT_S3_ENDPOINT,
|
||||
"website_authority": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
|
||||
"credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", "secret_key_env": "AWS_SECRET_ACCESS_KEY"},
|
||||
"cache_rules": [{"path": "", "cache_control": DEFAULT_CACHE_CONTROL}],
|
||||
}
|
||||
return {
|
||||
"version": 1,
|
||||
"compatibility": "single-surface-v1",
|
||||
"domain": raw["domain"],
|
||||
"aliases": _aliases(raw.get("aliases"), raw["domain"]),
|
||||
"enabled": raw.get("enabled", True),
|
||||
"artifacts": [artifact],
|
||||
"routes": [{
|
||||
"name": "site", "path": "/", "artifact": "site", "access": "legacy",
|
||||
"access_middleware": None,
|
||||
"middlewares": _middlewares(raw.get("middlewares") or [], "middlewares"),
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def _artifact(item, index):
|
||||
label = f"artifacts[{index}]"
|
||||
item = _mapping(item, label)
|
||||
_known_keys(item, {"name", "type", "content_dir", "tidy", "excludes", "publish", "cache"}, label)
|
||||
name = item.get("name")
|
||||
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
|
||||
raise ConfigError(f"{label}.name must be a DNS label")
|
||||
publish = _mapping(item.get("publish"), f"{label}.publish")
|
||||
_known_keys(publish, {"bucket", "credentials"}, f"{label}.publish")
|
||||
bucket = publish.get("bucket")
|
||||
if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket):
|
||||
raise ConfigError(f"{label}.publish.bucket is not a valid bucket name")
|
||||
credentials = _mapping(publish.get("credentials"), f"{label}.publish.credentials")
|
||||
_known_keys(credentials, {"access_key_env", "secret_key_env"}, f"{label}.publish.credentials")
|
||||
normalized_credentials = {}
|
||||
for key in ("access_key_env", "secret_key_env"):
|
||||
value = credentials.get(key)
|
||||
if not isinstance(value, str) or not ENV_RE.fullmatch(value):
|
||||
raise ConfigError(f"{label}.publish.credentials.{key} must name an environment variable")
|
||||
normalized_credentials[key] = value
|
||||
access_match = ACCESS_KEY_ENV_RE.fullmatch(normalized_credentials["access_key_env"])
|
||||
expected_secret = (
|
||||
f"{access_match.group(1)}_S3_SECRET_KEY" if access_match else None
|
||||
)
|
||||
if normalized_credentials["secret_key_env"] != expected_secret:
|
||||
raise ConfigError(
|
||||
f"{label}.publish.credentials must be a matched "
|
||||
"<NAME>_S3_ACCESS_KEY and <NAME>_S3_SECRET_KEY pair"
|
||||
)
|
||||
cache = _mapping(item.get("cache"), f"{label}.cache")
|
||||
_known_keys(cache, {"rules"}, f"{label}.cache")
|
||||
rules = _list(cache.get("rules"), f"{label}.cache.rules")
|
||||
if not rules:
|
||||
raise ConfigError(f"{label}.cache.rules must declare a '/' default")
|
||||
cache_rules, paths = [], set()
|
||||
for rule_index, rule in enumerate(rules):
|
||||
rule_label = f"{label}.cache.rules[{rule_index}]"
|
||||
rule = _mapping(rule, rule_label)
|
||||
_known_keys(rule, {"path", "cache_control"}, rule_label)
|
||||
path = _relative_path(rule.get("path"), f"{rule_label}.path", allow_root=True)
|
||||
if path in paths:
|
||||
raise ConfigError(f"{label}.cache.rules has duplicate path /{path}")
|
||||
paths.add(path)
|
||||
cache_rules.append({"path": path, "cache_control": _cache_control(
|
||||
rule.get("cache_control"), f"{rule_label}.cache_control"
|
||||
)})
|
||||
if "" not in paths:
|
||||
raise ConfigError(f"{label}.cache.rules must declare a '/' default")
|
||||
cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"]))
|
||||
immutable_paths = [rule["path"] for rule in cache_rules
|
||||
if "immutable" in {part.strip().lower().split("=", 1)[0]
|
||||
for part in rule["cache_control"].split(",")}]
|
||||
if "" in immutable_paths:
|
||||
raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'")
|
||||
for path in immutable_paths:
|
||||
if any(other.startswith(f"{path}/") for other in paths):
|
||||
raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}")
|
||||
authority = f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}"
|
||||
if not isinstance(item.get("tidy", True), bool):
|
||||
raise ConfigError(f"{label}.tidy must be a boolean")
|
||||
return {
|
||||
"name": name,
|
||||
"type": _site_type(item.get("type", "static"), f"{label}.type"),
|
||||
"content_dir": _relative_path(item.get("content_dir", ""), f"{label}.content_dir"),
|
||||
"tidy": item.get("tidy", True),
|
||||
"excludes": _strings(item.get("excludes") or [], f"{label}.excludes"),
|
||||
"build_dir": f".site-publish/{name}/html",
|
||||
"bucket": bucket,
|
||||
"s3_endpoint": DEFAULT_S3_ENDPOINT,
|
||||
"website_authority": authority,
|
||||
"credentials": normalized_credentials,
|
||||
"cache_rules": cache_rules,
|
||||
}
|
||||
|
||||
|
||||
def _route(item, index):
|
||||
label = f"routes[{index}]"
|
||||
item = _mapping(item, label)
|
||||
_known_keys(item, {"name", "path", "artifact", "access", "middlewares"}, label)
|
||||
name = item.get("name")
|
||||
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
|
||||
raise ConfigError(f"{label}.name must be a DNS label")
|
||||
access = _mapping(item.get("access"), f"{label}.access")
|
||||
_known_keys(access, {"mode", "middleware"}, f"{label}.access")
|
||||
mode, middleware = access.get("mode"), access.get("middleware")
|
||||
if mode not in {"public", "protected"}:
|
||||
raise ConfigError(f"{label}.access.mode must be public or protected")
|
||||
if mode == "protected" and (not isinstance(middleware, str) or not MIDDLEWARE_RE.fullmatch(middleware)):
|
||||
raise ConfigError(f"{label}.access.middleware is required for protected access")
|
||||
if mode == "public" and middleware is not None:
|
||||
raise ConfigError(f"{label}.access.middleware is forbidden for public access")
|
||||
middlewares = _middlewares(item.get("middlewares") or [], f"{label}.middlewares")
|
||||
if middleware in middlewares:
|
||||
raise ConfigError(f"{label} repeats its access middleware")
|
||||
artifact = item.get("artifact")
|
||||
if not isinstance(artifact, str):
|
||||
raise ConfigError(f"{label}.artifact must name an artifact")
|
||||
return {
|
||||
"name": name, "path": _route_path(item.get("path"), f"{label}.path"),
|
||||
"artifact": artifact, "access": mode, "access_middleware": middleware,
|
||||
"middlewares": middlewares,
|
||||
}
|
||||
|
||||
|
||||
def _validate_multi(cfg):
|
||||
artifacts, routes = cfg["artifacts"], cfg["routes"]
|
||||
artifact_names = [item["name"] for item in artifacts]
|
||||
route_names = [item["name"] for item in routes]
|
||||
route_paths = [item["path"] for item in routes]
|
||||
if len(artifact_names) != len(set(artifact_names)):
|
||||
raise ConfigError("artifact names must be unique")
|
||||
if len(route_names) != len(set(route_names)):
|
||||
raise ConfigError("route names must be unique")
|
||||
if len(route_paths) != len(set(route_paths)):
|
||||
raise ConfigError("route paths are ambiguous after normalization")
|
||||
if "/" not in route_paths:
|
||||
raise ConfigError("routes must declare a '/' catch-all")
|
||||
artifact_by_name = {item["name"]: item for item in artifacts}
|
||||
references = {name: [] for name in artifact_by_name}
|
||||
for route in routes:
|
||||
if route["artifact"] not in artifact_by_name:
|
||||
raise ConfigError(f"route {route['name']} references unknown artifact {route['artifact']}")
|
||||
references[route["artifact"]].append(route)
|
||||
for name, used_by in references.items():
|
||||
if len(used_by) != 1:
|
||||
raise ConfigError(f"artifact {name} must be referenced by exactly one route")
|
||||
buckets, authorities, credential_owners = {}, {}, {}
|
||||
for route in routes:
|
||||
artifact = artifact_by_name[route["artifact"]]
|
||||
if artifact["bucket"] in buckets:
|
||||
other_access, other_name = buckets[artifact["bucket"]]
|
||||
if other_access != route["access"]:
|
||||
raise ConfigError(f"bucket {artifact['bucket']} cannot be reused by protected and public routes")
|
||||
raise ConfigError(f"bucket {artifact['bucket']} must belong to one artifact ({other_name})")
|
||||
buckets[artifact["bucket"]] = (route["access"], artifact["name"])
|
||||
if artifact["website_authority"] in authorities:
|
||||
raise ConfigError(
|
||||
f"website authority {artifact['website_authority']} must belong to one artifact"
|
||||
)
|
||||
authorities[artifact["website_authority"]] = artifact["name"]
|
||||
for variable in artifact["credentials"].values():
|
||||
if variable in credential_owners:
|
||||
raise ConfigError(
|
||||
f"publication credential {variable} is reused by artifacts "
|
||||
f"{credential_owners[variable]} and {artifact['name']}"
|
||||
)
|
||||
credential_owners[variable] = artifact["name"]
|
||||
for cache_rule in artifact["cache_rules"]:
|
||||
directives = {part.strip().lower().split("=", 1)[0]
|
||||
for part in cache_rule["cache_control"].split(",")}
|
||||
if route["access"] == "protected" and "public" in directives:
|
||||
raise ConfigError(f"protected route {route['name']} cannot use public cache policy")
|
||||
if route["access"] == "protected" and not directives & {"private", "no-store"}:
|
||||
raise ConfigError(f"protected route {route['name']} cache policy must be private or no-store")
|
||||
if route["access"] == "protected" and "s-maxage" in directives:
|
||||
raise ConfigError(f"protected route {route['name']} cannot use shared-cache max-age")
|
||||
if route["access"] == "public" and "private" in directives:
|
||||
raise ConfigError(f"public route {route['name']} cannot use private cache policy")
|
||||
root = next(route for route in routes if route["path"] == "/")
|
||||
if any(route["access"] == "protected" for route in routes) and root["access"] == "public":
|
||||
raise ConfigError("a public '/' catch-all would expose unmatched protected content")
|
||||
|
||||
|
||||
def normalize_site_config(raw, site_name):
|
||||
raw = _mapping(raw, "site.yaml")
|
||||
domain = _hostname(raw.get("domain"), "domain")
|
||||
if not isinstance(raw.get("enabled", True), bool):
|
||||
raise ConfigError("enabled must be a boolean")
|
||||
if ("artifacts" in raw) != ("routes" in raw):
|
||||
raise ConfigError("artifacts and routes must be declared together")
|
||||
if "artifacts" not in raw:
|
||||
return _legacy_config(raw, site_name)
|
||||
legacy_fields = {"type", "content_dir", "tidy", "excludes", "middlewares"} & set(raw)
|
||||
if legacy_fields:
|
||||
raise ConfigError("legacy fields cannot be mixed with artifacts/routes: " + ", ".join(sorted(legacy_fields)))
|
||||
_known_keys(raw, {"domain", "aliases", "enabled", "artifacts", "routes"}, "site.yaml")
|
||||
artifacts = [_artifact(item, index) for index, item in enumerate(_list(raw["artifacts"], "artifacts"))]
|
||||
routes = [_route(item, index) for index, item in enumerate(_list(raw["routes"], "routes"))]
|
||||
if not artifacts or not routes:
|
||||
raise ConfigError("artifacts and routes must not be empty")
|
||||
cfg = {
|
||||
"version": 2, "compatibility": None, "domain": domain,
|
||||
"aliases": _aliases(raw.get("aliases"), domain),
|
||||
"enabled": raw.get("enabled", True),
|
||||
"artifacts": sorted(artifacts, key=lambda item: item["name"]),
|
||||
"routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])),
|
||||
}
|
||||
_validate_multi(cfg)
|
||||
for route in cfg["routes"]:
|
||||
if len(f"{k8s_name(site_name)}-{route['name']}") > 63:
|
||||
raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters")
|
||||
return cfg
|
||||
|
||||
|
||||
def validate_artifact_inputs(site_dir, cfg):
|
||||
"""Reject source containment before a public or protected build starts."""
|
||||
root = Path(site_dir).resolve()
|
||||
sources = []
|
||||
for artifact in cfg["artifacts"]:
|
||||
source = (root / artifact["content_dir"]).resolve()
|
||||
if source != root and root not in source.parents:
|
||||
raise ConfigError(
|
||||
f"artifact {artifact['name']} content_dir resolves outside the repository"
|
||||
)
|
||||
sources.append((artifact["name"], source))
|
||||
for index, (name, source) in enumerate(sources):
|
||||
for other_name, other_source in sources[index + 1:]:
|
||||
if source == other_source or source in other_source.parents or other_source in source.parents:
|
||||
raise ConfigError(
|
||||
f"artifact build inputs overlap after resolution: {name} and {other_name}"
|
||||
)
|
||||
|
||||
|
||||
def parse_site_yaml(site_dir, site_name=None):
|
||||
path = Path(site_dir) / "site.yaml"
|
||||
if not path.exists():
|
||||
die("site.yaml not found in repo root")
|
||||
if site_name is None:
|
||||
repo = os.environ.get("SITE_REPO", "")
|
||||
site_name = repo.split("/", 1)[-1] if "/" in repo else Path(site_dir).name
|
||||
try:
|
||||
with open(path) as stream:
|
||||
cfg = normalize_site_config(yaml.safe_load(stream), site_name)
|
||||
except (ConfigError, yaml.YAMLError) as exc:
|
||||
die(str(exc))
|
||||
print("Site config:")
|
||||
for k, v in site.items():
|
||||
print(f" {k}: {v}")
|
||||
return site
|
||||
print(f" domain: {cfg['domain']}")
|
||||
print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}")
|
||||
print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}")
|
||||
print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}")
|
||||
return cfg
|
||||
|
||||
|
||||
def clone_apps(token):
|
||||
"""Clone Apps without placing the credential in argv or output."""
|
||||
user = env("CI_BOT_USER", "ci-bot")
|
||||
apps_dir = Path("/tmp/apps-deploy")
|
||||
if apps_dir.exists():
|
||||
shutil.rmtree(apps_dir)
|
||||
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}")
|
||||
run(f"git -C {apps_dir} config user.name {user}")
|
||||
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
|
||||
clone_env = git_auth_env(token)
|
||||
url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git"
|
||||
run(["git", "clone", "--depth", "1", url, str(apps_dir)],
|
||||
display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env)
|
||||
run(["git", "-C", str(apps_dir), "config", "user.name", user])
|
||||
run(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
|
||||
return apps_dir
|
||||
|
||||
|
||||
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
|
||||
"""Render Jinja2 templates for a static-content site."""
|
||||
templates_dir = Path(action_dir) / "templates"
|
||||
jinja_env = Environment(
|
||||
loader=FileSystemLoader(str(templates_dir)),
|
||||
keep_trailing_newline=True,
|
||||
)
|
||||
|
||||
tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2",
|
||||
"kustomization.yaml.j2", "service.yaml.j2"]
|
||||
|
||||
for tmpl_name in tmpl_names:
|
||||
tmpl = jinja_env.get_template(tmpl_name)
|
||||
rendered = tmpl.render(**template_vars)
|
||||
out_name = tmpl_name.replace(".j2", "")
|
||||
dest = app_dir / out_name if tmpl_name == "app.yaml.j2" else manifests_dir / out_name
|
||||
dest.write_text(rendered)
|
||||
print(f" Rendered {tmpl_name} -> {dest}")
|
||||
"""Render one certificate and deterministic per-route resources."""
|
||||
jinja_env = Environment(loader=FileSystemLoader(str(Path(action_dir) / "templates")),
|
||||
keep_trailing_newline=True, undefined=StrictUndefined)
|
||||
manifests_dir.mkdir(parents=True, exist_ok=True)
|
||||
for child in manifests_dir.iterdir():
|
||||
if child.is_file() and child.suffix in {".yaml", ".yml"}:
|
||||
child.unlink()
|
||||
route_files = []
|
||||
for route in template_vars["routes"]:
|
||||
stem = "" if template_vars["compatibility"] else f"-{route['name']}"
|
||||
for kind in ("service", "ingress"):
|
||||
out_name = f"{kind}{stem}.yaml"
|
||||
destination = manifests_dir / out_name
|
||||
destination.write_text(jinja_env.get_template(f"{kind}.yaml.j2").render(
|
||||
**template_vars, route=route
|
||||
))
|
||||
route_files.append(out_name)
|
||||
print(f" Rendered {kind}.yaml.j2 -> {destination}")
|
||||
common_vars = {**template_vars, "route_files": route_files}
|
||||
for out_name, destination in {
|
||||
"certificate.yaml": manifests_dir / "certificate.yaml",
|
||||
"kustomization.yaml": manifests_dir / "kustomization.yaml",
|
||||
"app.yaml": app_dir / "app.yaml",
|
||||
}.items():
|
||||
destination.write_text(jinja_env.get_template(f"{out_name}.j2").render(**common_vars))
|
||||
print(f" Rendered {out_name}.j2 -> {destination}")
|
||||
|
||||
|
||||
def commit_and_push(apps_dir, message):
|
||||
run(f"git -C {apps_dir} add -A")
|
||||
result = subprocess.run(
|
||||
f"git -C {apps_dir} diff --cached --quiet",
|
||||
shell=True, check=False,
|
||||
)
|
||||
def git_auth_env(token):
|
||||
"""Return credential-safe Git authentication shared by clone and push."""
|
||||
auth_env = os.environ.copy()
|
||||
auth_env["CI_BOT_TOKEN"] = token
|
||||
auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
|
||||
auth_env["GIT_TERMINAL_PROMPT"] = "0"
|
||||
return auth_env
|
||||
|
||||
|
||||
def commit_and_push(apps_dir, message, token=None):
|
||||
run(["git", "-C", str(apps_dir), "add", "-A"])
|
||||
result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False)
|
||||
if result.returncode == 0:
|
||||
print("No manifest changes to commit")
|
||||
return False
|
||||
run(f"git -C {apps_dir} commit -m '{message}'")
|
||||
run(f"git -C {apps_dir} push")
|
||||
run(["git", "-C", str(apps_dir), "commit", "-m", message])
|
||||
push_env = git_auth_env(token) if token else None
|
||||
run(["git", "-C", str(apps_dir), "push"], env=push_env)
|
||||
print("Manifests pushed — ArgoCD will sync")
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user