This commit is contained in:
+57
-4
@@ -5,7 +5,7 @@ import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from utils import EXCLUDE_FILES, env, parse_site_yaml, run
|
||||
from utils import EXCLUDE_FILES, env, parse_site_yaml, run_args
|
||||
|
||||
|
||||
def build_static(site_dir, cfg):
|
||||
@@ -34,11 +34,11 @@ def build_static(site_dir, cfg):
|
||||
|
||||
elif cfg["type"] == "hugo":
|
||||
print(f"Building Hugo site from {src}")
|
||||
run(f"hugo --source {src} --destination {html_dir}")
|
||||
run_args(["hugo", "--source", str(src), "--destination", str(html_dir)])
|
||||
|
||||
elif cfg["type"] == "mkdocs":
|
||||
print(f"Building MkDocs site from {src}")
|
||||
run(f"cd {src} && mkdocs build -d {html_dir}")
|
||||
run_args(["mkdocs", "build", "-d", str(html_dir)], cwd=src)
|
||||
|
||||
if cfg.get("tidy", True):
|
||||
print("Running tidy on HTML files...")
|
||||
@@ -54,6 +54,56 @@ def build_static(site_dir, cfg):
|
||||
print(f"Build complete — content at {html_dir}")
|
||||
|
||||
|
||||
def stage_artifacts(site_dir, cfg):
|
||||
"""Snapshot caller-built outputs for the deploy phase."""
|
||||
staging_dir = site_dir / ".site-publish"
|
||||
if staging_dir.is_symlink():
|
||||
raise SystemExit("ERROR: .site-publish must not be a symbolic link")
|
||||
if staging_dir.exists():
|
||||
shutil.rmtree(staging_dir)
|
||||
staging_dir.mkdir()
|
||||
root = site_dir.resolve()
|
||||
|
||||
for name, artifact in cfg["artifacts"].items():
|
||||
source = (site_dir / artifact["source"]).resolve()
|
||||
try:
|
||||
source.relative_to(root)
|
||||
except ValueError:
|
||||
raise SystemExit(f"ERROR: artifact {name}.source escapes the repository")
|
||||
if not source.is_dir():
|
||||
raise SystemExit(f"ERROR: artifact {name}.source is not a directory: {artifact['source']}")
|
||||
files = [entry for entry in source.rglob("*") if entry.is_file()]
|
||||
if not files:
|
||||
raise SystemExit(f"ERROR: artifact {name}.source has no files: {artifact['source']}")
|
||||
for entry in source.rglob("*"):
|
||||
if entry.is_symlink():
|
||||
try:
|
||||
entry.resolve(strict=True).relative_to(root)
|
||||
except (FileNotFoundError, ValueError):
|
||||
raise SystemExit(
|
||||
f"ERROR: artifact {name}.source contains an unsafe symlink: {entry.relative_to(source)}"
|
||||
)
|
||||
destination = staging_dir / name
|
||||
destination.mkdir()
|
||||
routes = [route for route in cfg["routes"] if route["artifact"] == name]
|
||||
targets = set()
|
||||
print(f"Staging artifact {name} from {artifact['source']}")
|
||||
for route in routes:
|
||||
route_root = destination / route["path"].lstrip("/")
|
||||
for entry in files:
|
||||
relative = entry.relative_to(source)
|
||||
target = route_root / relative
|
||||
if target in targets:
|
||||
raise SystemExit(
|
||||
f"ERROR: artifact {name} routes map more than once to {target.relative_to(destination)}"
|
||||
)
|
||||
targets.add(target)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(entry, target, follow_symlinks=True)
|
||||
|
||||
print(f"Build complete — staged artifacts at {staging_dir}")
|
||||
|
||||
|
||||
def cmd_build():
|
||||
site_dir = Path(env("SITE_DIR"))
|
||||
cfg = parse_site_yaml(site_dir)
|
||||
@@ -62,4 +112,7 @@ def cmd_build():
|
||||
print("Site disabled — skipping build")
|
||||
return
|
||||
|
||||
build_static(site_dir, cfg)
|
||||
if cfg["mode"] == "multi":
|
||||
stage_artifacts(site_dir, cfg)
|
||||
else:
|
||||
build_static(site_dir, cfg)
|
||||
|
||||
+197
-40
@@ -2,16 +2,15 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from utils import (
|
||||
DEFAULT_S3_ENDPOINT,
|
||||
GITEA_HOST,
|
||||
NAMESPACE,
|
||||
clone_apps,
|
||||
commit_and_push,
|
||||
@@ -20,7 +19,7 @@ from utils import (
|
||||
k8s_name,
|
||||
parse_site_yaml,
|
||||
render_templates,
|
||||
run,
|
||||
run_args,
|
||||
)
|
||||
|
||||
GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
||||
@@ -31,21 +30,66 @@ GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
||||
CACHE_CONTROL = "public, max-age=0, must-revalidate"
|
||||
|
||||
|
||||
def s3_sync(site_name, site_dir, excludes=None):
|
||||
endpoint = os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
|
||||
html_dir = site_dir / "build" / "html"
|
||||
if not html_dir.exists():
|
||||
die(f"build/html not found — did the build step run? ({html_dir})")
|
||||
env("AWS_ACCESS_KEY_ID")
|
||||
env("AWS_SECRET_ACCESS_KEY")
|
||||
os.environ.setdefault("AWS_DEFAULT_REGION", "sjc001")
|
||||
def credential_environment(profile):
|
||||
if profile == "default":
|
||||
access_key = env("AWS_ACCESS_KEY_ID")
|
||||
secret_key = env("AWS_SECRET_ACCESS_KEY")
|
||||
else:
|
||||
prefix = f"SITE_PUBLISH_{profile.upper().replace('-', '_')}"
|
||||
access_key = env(f"{prefix}_S3_ACCESS_KEY_ID")
|
||||
secret_key = env(f"{prefix}_S3_SECRET_ACCESS_KEY")
|
||||
child_env = os.environ.copy()
|
||||
child_env["AWS_ACCESS_KEY_ID"] = access_key
|
||||
child_env["AWS_SECRET_ACCESS_KEY"] = secret_key
|
||||
child_env.setdefault("AWS_DEFAULT_REGION", "sjc001")
|
||||
return child_env
|
||||
|
||||
|
||||
def configure_cors(bucket, origins, endpoint, child_env):
|
||||
if origins is None:
|
||||
return
|
||||
if not origins:
|
||||
run_args(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
|
||||
"--bucket", bucket],
|
||||
env=child_env,
|
||||
)
|
||||
return
|
||||
config = {
|
||||
"CORSRules": [{
|
||||
"AllowedOrigins": origins,
|
||||
"AllowedMethods": ["GET", "HEAD"],
|
||||
"AllowedHeaders": ["*"],
|
||||
"ExposeHeaders": ["ETag"],
|
||||
"MaxAgeSeconds": 3600,
|
||||
}]
|
||||
}
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
|
||||
json.dump(config, handle)
|
||||
handle.flush()
|
||||
run_args(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
|
||||
"--bucket", bucket, "--cors-configuration", f"file://{handle.name}"],
|
||||
env=child_env,
|
||||
)
|
||||
|
||||
|
||||
def s3_sync(bucket, source, *, credential="default", endpoint=None,
|
||||
cache=None, cors_origins=None, excludes=None):
|
||||
endpoint = endpoint or os.environ.get("GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT)
|
||||
if not source.exists():
|
||||
die(f"staged artifact not found — did the build step run? ({source})")
|
||||
child_env = credential_environment(credential)
|
||||
cache = cache or {"default": CACHE_CONTROL, "rules": []}
|
||||
# `excludes` are patterns (site.yaml `excludes:` list) that should never
|
||||
# be uploaded *and* should never be deleted from the bucket — escape hatch
|
||||
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
|
||||
exclude_flags = " ".join(f"--exclude {shlex.quote(p)}" for p in (excludes or []))
|
||||
exclude_args = []
|
||||
for pattern in excludes or []:
|
||||
exclude_args.extend(["--exclude", pattern])
|
||||
if excludes:
|
||||
print(f"Excluding patterns: {excludes}")
|
||||
print(f"Syncing {html_dir} → s3://{site_name} via {endpoint}")
|
||||
print(f"Syncing {source} → s3://{bucket} via {endpoint}")
|
||||
# `sync --delete` handles new/changed/orphaned files. `cp --recursive`
|
||||
# then re-uploads everything to refresh metadata (cache-control,
|
||||
# content-type) on objects sync skipped because nothing changed.
|
||||
@@ -53,22 +97,63 @@ def s3_sync(site_name, site_dir, excludes=None):
|
||||
# small enough that that's free; correctness wins over throughput.
|
||||
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
|
||||
# so a fresh upload always carries the right MIME type.
|
||||
run(
|
||||
f"aws --endpoint-url {endpoint} s3 sync {html_dir}/ s3://{site_name}/ "
|
||||
f"--delete --only-show-errors "
|
||||
f"--cache-control '{CACHE_CONTROL}' "
|
||||
f"{exclude_flags}".rstrip()
|
||||
run_args(
|
||||
["aws", "--endpoint-url", endpoint, "s3", "sync", f"{source}/", f"s3://{bucket}/",
|
||||
"--delete", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
|
||||
env=child_env,
|
||||
)
|
||||
print("Re-stamping metadata on all objects...")
|
||||
run(
|
||||
f"aws --endpoint-url {endpoint} s3 cp {html_dir}/ s3://{site_name}/ "
|
||||
f"--recursive --only-show-errors "
|
||||
f"--cache-control '{CACHE_CONTROL}' "
|
||||
f"{exclude_flags}".rstrip()
|
||||
run_args(
|
||||
["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
|
||||
"--recursive", "--only-show-errors", "--cache-control", cache["default"], *exclude_args],
|
||||
env=child_env,
|
||||
)
|
||||
for rule in cache["rules"]:
|
||||
print(f"Applying cache metadata for {rule['match']}")
|
||||
run_args(
|
||||
["aws", "--endpoint-url", endpoint, "s3", "cp", f"{source}/", f"s3://{bucket}/",
|
||||
"--recursive", "--only-show-errors", "--exclude", "*",
|
||||
"--include", rule["match"], "--cache-control", rule["value"], *exclude_args],
|
||||
env=child_env,
|
||||
)
|
||||
configure_cors(bucket, cors_origins, endpoint, child_env)
|
||||
|
||||
|
||||
def preflight_artifacts(site_dir, cfg):
|
||||
"""Validate every source, credential, and bucket before the first write."""
|
||||
for artifact in cfg["artifacts"].values():
|
||||
source = site_dir / ".site-publish" / artifact["name"]
|
||||
if not source.is_dir() or not any(entry.is_file() for entry in source.rglob("*")):
|
||||
die(f"staged artifact is absent or empty: {source}")
|
||||
child_env = credential_environment(artifact["credential"])
|
||||
endpoint = artifact["endpoint"] or os.environ.get(
|
||||
"GARAGE_S3_ENDPOINT", DEFAULT_S3_ENDPOINT
|
||||
)
|
||||
run_args(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "head-bucket",
|
||||
"--bucket", artifact["bucket"]],
|
||||
env=child_env,
|
||||
)
|
||||
|
||||
|
||||
def routed_cache(cache, routes):
|
||||
"""Translate source-relative cache patterns to their public object keys."""
|
||||
rules = []
|
||||
for route in routes:
|
||||
prefix = route["path"].lstrip("/")
|
||||
for rule in cache["rules"]:
|
||||
pattern = f"{prefix}/{rule['match']}" if prefix else rule["match"]
|
||||
rules.append({"match": pattern, "value": rule["value"]})
|
||||
return {"default": cache["default"], "rules": rules}
|
||||
|
||||
|
||||
def garage_admin(method, path, token, body=None):
|
||||
parsed = urlsplit(GARAGE_ADMIN_ENDPOINT)
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"} or not parsed.netloc
|
||||
or parsed.username or parsed.password or parsed.query or parsed.fragment
|
||||
):
|
||||
die("GARAGE_ADMIN_ENDPOINT must be an HTTP URL without credentials")
|
||||
url = f"{GARAGE_ADMIN_ENDPOINT}{path}"
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
@@ -89,17 +174,17 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
|
||||
if not aliases:
|
||||
return
|
||||
if not admin_token:
|
||||
print(" (no GARAGE_ADMIN_TOKEN — skipping bucket alias reconcile)")
|
||||
return
|
||||
die("GARAGE_ADMIN_TOKEN is required when aliases are configured")
|
||||
|
||||
try:
|
||||
info = garage_admin("GET", f"/v2/GetBucketInfo?globalAlias={site_name}",
|
||||
admin_token)
|
||||
except (HTTPError, URLError) as e:
|
||||
print(f" WARNING: bucket lookup failed: {e}")
|
||||
return
|
||||
raise RuntimeError(f"bucket lookup failed for {site_name}: {e}") from e
|
||||
|
||||
bucket_id = info.get("id")
|
||||
if not isinstance(bucket_id, str) or not bucket_id:
|
||||
die(f"bucket lookup for {site_name} returned no bucket id")
|
||||
existing = set(info.get("globalAliases") or [])
|
||||
print(f" Bucket {site_name} ({bucket_id[:12]}…) currently aliases: {sorted(existing)}")
|
||||
|
||||
@@ -119,21 +204,92 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
|
||||
def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
|
||||
"""Always re-render manifests from current site.yaml. Templates own
|
||||
domain + aliases, so changes propagate without manual edits."""
|
||||
manifests_dir.mkdir(parents=True, exist_ok=True)
|
||||
if manifests_dir.exists():
|
||||
shutil.rmtree(manifests_dir)
|
||||
manifests_dir.mkdir(parents=True)
|
||||
site_k8s = k8s_name(site_name)
|
||||
|
||||
if cfg["mode"] == "legacy":
|
||||
artifacts = [{
|
||||
"name": "site",
|
||||
"bucket": site_name,
|
||||
"service_name": site_k8s,
|
||||
"external_name": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
|
||||
"virtual_host": False,
|
||||
}]
|
||||
routes = [{
|
||||
"name": "site",
|
||||
"path": "/",
|
||||
"artifact": "site",
|
||||
"service_name": site_k8s,
|
||||
"ingress_name": site_k8s,
|
||||
"middlewares": cfg["middlewares"],
|
||||
"middleware_refs": [f"{name}@file" for name in cfg["middlewares"]],
|
||||
}]
|
||||
else:
|
||||
artifacts = []
|
||||
artifact_by_name = {}
|
||||
for name, artifact in cfg["artifacts"].items():
|
||||
service_name = k8s_name(f"{site_name}-{name}")
|
||||
view = {
|
||||
**artifact,
|
||||
"service_name": service_name,
|
||||
"external_name": f"{artifact['bucket']}.web.sjc001.fritzlab.net",
|
||||
"virtual_host": True,
|
||||
}
|
||||
artifacts.append(view)
|
||||
artifact_by_name[name] = view
|
||||
routes = []
|
||||
for route in cfg["routes"]:
|
||||
artifact = artifact_by_name[route["artifact"]]
|
||||
ingress_name = k8s_name(f"{site_name}-{route['name']}")
|
||||
middleware_refs = [f"{name}@file" for name in route["middlewares"]]
|
||||
routes.append({
|
||||
**route,
|
||||
"service_name": artifact["service_name"],
|
||||
"ingress_name": ingress_name,
|
||||
"middleware_refs": middleware_refs,
|
||||
})
|
||||
|
||||
template_vars = {
|
||||
"site": site_name,
|
||||
"site_k8s": k8s_name(site_name),
|
||||
"site_k8s": site_k8s,
|
||||
"domain": cfg["domain"],
|
||||
"aliases": cfg["aliases"],
|
||||
"namespace": NAMESPACE,
|
||||
"middlewares": cfg["middlewares"],
|
||||
"artifacts": artifacts,
|
||||
"routes": routes,
|
||||
"modern": cfg["mode"] == "multi",
|
||||
}
|
||||
render_templates(action_dir, template_vars, app_dir, manifests_dir)
|
||||
|
||||
|
||||
def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
||||
s3_sync(site_name, site_dir, excludes=cfg.get("excludes"))
|
||||
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
|
||||
if cfg["mode"] == "legacy":
|
||||
s3_sync(
|
||||
site_name,
|
||||
site_dir / "build" / "html",
|
||||
excludes=cfg.get("excludes"),
|
||||
)
|
||||
ensure_bucket_aliases(
|
||||
site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")
|
||||
)
|
||||
else:
|
||||
preflight_artifacts(site_dir, cfg)
|
||||
for artifact in cfg["artifacts"].values():
|
||||
routes = [
|
||||
route for route in cfg["routes"]
|
||||
if route["artifact"] == artifact["name"]
|
||||
]
|
||||
s3_sync(
|
||||
artifact["bucket"],
|
||||
site_dir / ".site-publish" / artifact["name"],
|
||||
credential=artifact["credential"],
|
||||
endpoint=artifact["endpoint"],
|
||||
cache=routed_cache(artifact["cache"], routes),
|
||||
cors_origins=artifact["cors_origins"],
|
||||
excludes=artifact["excludes"],
|
||||
)
|
||||
|
||||
apps_dir = clone_apps(token)
|
||||
app_dir = apps_dir / "sjc001" / "websites" / site_name
|
||||
@@ -141,23 +297,24 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
||||
|
||||
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
|
||||
|
||||
commit_and_push(apps_dir, f"Deploy {site_name}")
|
||||
try:
|
||||
commit_and_push(apps_dir, f"Deploy {site_name}", token)
|
||||
finally:
|
||||
shutil.rmtree(apps_dir.parent, ignore_errors=True)
|
||||
|
||||
|
||||
def decommission(site_name, token):
|
||||
"""Remove manifests from apps repo."""
|
||||
user = env("CI_BOT_USER", "ci-bot")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
apps_dir = Path(tmp)
|
||||
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/fritzlab/apps.git {apps_dir}")
|
||||
apps_dir = clone_apps(token)
|
||||
try:
|
||||
site_path = apps_dir / "sjc001" / "websites" / site_name
|
||||
if not site_path.exists():
|
||||
print(f"No manifests for {site_name} — nothing to remove")
|
||||
return
|
||||
shutil.rmtree(site_path)
|
||||
run(f"git -C {apps_dir} config user.name {user}")
|
||||
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
|
||||
commit_and_push(apps_dir, f"Decommission {site_name}")
|
||||
commit_and_push(apps_dir, f"Decommission {site_name}", token)
|
||||
finally:
|
||||
shutil.rmtree(apps_dir.parent, ignore_errors=True)
|
||||
print(f"Bucket {site_name} and its objects are NOT purged automatically.")
|
||||
print(f" garage bucket delete {site_name} --yes")
|
||||
|
||||
|
||||
+3
-2
@@ -24,9 +24,10 @@ def ensure_aws():
|
||||
subprocess.run(["aws", "--version"], check=True)
|
||||
|
||||
|
||||
def ensure_jinja2():
|
||||
def ensure_python_modules():
|
||||
try:
|
||||
import jinja2
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("Installing jinja2 + pyyaml...")
|
||||
subprocess.run(
|
||||
@@ -36,6 +37,6 @@ def ensure_jinja2():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ensure_jinja2()
|
||||
ensure_python_modules()
|
||||
ensure_aws()
|
||||
print("Setup complete")
|
||||
|
||||
+369
-32
@@ -1,13 +1,18 @@
|
||||
"""Shared utilities for the site-publish action."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
||||
from yaml import YAMLError
|
||||
|
||||
APPS_REPO = "fritzlab/apps"
|
||||
GITEA_HOST = "code.fritzlab.net"
|
||||
@@ -16,11 +21,15 @@ DEFAULT_S3_ENDPOINT = "http://garage-s3.storage.svc:3900"
|
||||
|
||||
EXCLUDE_FILES = {
|
||||
".git", ".gitea", ".gitignore", "site.yaml",
|
||||
"build", "Makefile", "README.md", "CLAUDE.md",
|
||||
".site-publish", "build", "Makefile", "README.md", "CLAUDE.md",
|
||||
"Dockerfile", ".dockerignore", "go.mod", "go.sum",
|
||||
}
|
||||
|
||||
VALID_TYPES = {"static", "hugo", "mkdocs"}
|
||||
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
||||
MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
||||
PROFILE_RE = re.compile(r"^[a-z][a-z0-9-]*$")
|
||||
AUTH_MIDDLEWARE = "authentik-forwardauth"
|
||||
|
||||
DOCKER_DEPRECATION_MSG = """\
|
||||
type: docker is no longer supported by action/site-publish.
|
||||
@@ -43,13 +52,18 @@ example.\
|
||||
|
||||
|
||||
def k8s_name(name):
|
||||
"""Sanitize for DNS-1035 label (dots → dashes)."""
|
||||
return name.replace(".", "-")
|
||||
"""Return a stable DNS-1035 label, including for long host names."""
|
||||
normalized = re.sub(r"[^a-z0-9-]", "-", name.lower()).strip("-")
|
||||
normalized = re.sub(r"-+", "-", normalized)
|
||||
if len(normalized) <= 63:
|
||||
return normalized
|
||||
digest = hashlib.sha256(normalized.encode()).hexdigest()[:8]
|
||||
return f"{normalized[:54].rstrip('-')}-{digest}"
|
||||
|
||||
|
||||
def env(key, default=None):
|
||||
val = os.environ.get(key, default)
|
||||
if val is None:
|
||||
if val is None or val == "":
|
||||
die(f"Missing required env var: {key}")
|
||||
return val
|
||||
|
||||
@@ -59,9 +73,258 @@ def die(msg):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
print(f" $ {cmd}")
|
||||
return subprocess.run(cmd, shell=True, check=True, **kwargs)
|
||||
def run_args(args, *, display=None, **kwargs):
|
||||
"""Run an argv vector without shell interpolation or credential logging."""
|
||||
print(f" $ {display or ' '.join(args)}")
|
||||
return subprocess.run(args, check=True, **kwargs)
|
||||
|
||||
|
||||
def _string_list(cfg, key):
|
||||
value = cfg.get(key) or []
|
||||
if not isinstance(value, list) or any(not isinstance(item, str) for item in value):
|
||||
die(f"{key} must be a list of strings")
|
||||
return value
|
||||
|
||||
|
||||
def _patterns(cfg, key, field):
|
||||
values = _string_list(cfg, key)
|
||||
if any(not value or "\n" in value or "\r" in value for value in values):
|
||||
die(f"{field} must contain non-empty single-line patterns")
|
||||
return values
|
||||
|
||||
|
||||
def _cors_origins(raw, artifact_name):
|
||||
origins = _string_list(raw, "cors_origins")
|
||||
for origin in origins:
|
||||
if origin == "*":
|
||||
continue
|
||||
parsed = urlsplit(origin)
|
||||
if (
|
||||
parsed.scheme != "https" or not parsed.netloc or parsed.path not in {"", "/"}
|
||||
or parsed.query or parsed.fragment or parsed.username or parsed.password
|
||||
):
|
||||
die(f"artifact {artifact_name}.cors_origins must contain * or HTTPS origins")
|
||||
return origins
|
||||
|
||||
|
||||
def _validate_domain(value, field):
|
||||
if not isinstance(value, str) or not value or len(value) > 253:
|
||||
die(f"{field} must be a DNS name")
|
||||
labels = value.rstrip(".").split(".")
|
||||
if any(not NAME_RE.fullmatch(label) for label in labels):
|
||||
die(f"{field} must be a DNS name")
|
||||
return value.rstrip(".")
|
||||
|
||||
|
||||
def _validate_middlewares(value, field):
|
||||
if not isinstance(value, list) or any(
|
||||
not isinstance(item, str) or not MIDDLEWARE_RE.fullmatch(item)
|
||||
for item in value
|
||||
):
|
||||
die(f"{field} must contain file-provider middleware names")
|
||||
return value
|
||||
|
||||
|
||||
def _known_keys(value, allowed, field):
|
||||
unknown = sorted(set(value) - set(allowed))
|
||||
if unknown:
|
||||
die(f"{field} has unknown fields: {', '.join(unknown)}")
|
||||
|
||||
|
||||
def _cache_header(value, field):
|
||||
if not isinstance(value, str) or not value or "\n" in value or "\r" in value:
|
||||
die(f"{field} must be one Cache-Control header value")
|
||||
return value
|
||||
|
||||
|
||||
def _cache_config(raw, artifact_name):
|
||||
cache = raw.get("cache") or {}
|
||||
if not isinstance(cache, dict):
|
||||
die(f"artifact {artifact_name}.cache must be a mapping")
|
||||
_known_keys(cache, {"default", "rules"}, f"artifact {artifact_name}.cache")
|
||||
default = _cache_header(
|
||||
cache.get("default", "public, max-age=0, must-revalidate"),
|
||||
f"artifact {artifact_name}.cache.default",
|
||||
)
|
||||
rules_raw = cache.get("rules") or []
|
||||
if not isinstance(rules_raw, list):
|
||||
die(f"artifact {artifact_name}.cache.rules must be a list")
|
||||
rules = []
|
||||
patterns = set()
|
||||
for index, rule in enumerate(rules_raw):
|
||||
field = f"artifact {artifact_name}.cache.rules[{index}]"
|
||||
if not isinstance(rule, dict):
|
||||
die(f"{field} must be a mapping")
|
||||
_known_keys(rule, {"match", "value"}, field)
|
||||
pattern = rule.get("match")
|
||||
if (
|
||||
not isinstance(pattern, str) or not pattern or pattern.startswith("/")
|
||||
or "\n" in pattern or "\r" in pattern or ".." in Path(pattern).parts
|
||||
):
|
||||
die(f"{field}.match must be a relative aws-cli include pattern")
|
||||
if pattern in patterns:
|
||||
die(f"artifact {artifact_name} repeats cache match {pattern}")
|
||||
patterns.add(pattern)
|
||||
rules.append({
|
||||
"match": pattern,
|
||||
"value": _cache_header(rule.get("value"), f"{field}.value"),
|
||||
})
|
||||
return {"default": default, "rules": rules}
|
||||
|
||||
|
||||
def _cache_directives(value):
|
||||
return {part.strip().lower().split("=", 1)[0] for part in value.split(",")}
|
||||
|
||||
|
||||
def _modern_config(cfg, domain, aliases):
|
||||
artifacts_cfg = cfg.get("artifacts")
|
||||
routes_cfg = cfg.get("routes")
|
||||
if not isinstance(artifacts_cfg, dict) or not artifacts_cfg:
|
||||
die("artifacts must be a non-empty mapping")
|
||||
if not isinstance(routes_cfg, list) or not routes_cfg:
|
||||
die("routes must be a non-empty list")
|
||||
|
||||
artifacts = {}
|
||||
buckets = set()
|
||||
for name, raw in artifacts_cfg.items():
|
||||
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
|
||||
die(f"artifact name {name!r} must be a DNS label")
|
||||
if not isinstance(raw, dict):
|
||||
die(f"artifact {name} must be a mapping")
|
||||
_known_keys(
|
||||
raw,
|
||||
{"source", "bucket", "credential", "endpoint", "cache", "cors_origins", "excludes"},
|
||||
f"artifact {name}",
|
||||
)
|
||||
source = raw.get("source")
|
||||
bucket = raw.get("bucket")
|
||||
credential = raw.get("credential", "default")
|
||||
if not isinstance(source, str) or not source or Path(source).is_absolute():
|
||||
die(f"artifact {name}.source must be a relative path")
|
||||
if source in {".", "./"} or ".." in Path(source).parts:
|
||||
die(f"artifact {name}.source must name a build output inside the repository")
|
||||
bucket = _validate_domain(bucket, f"artifact {name}.bucket")
|
||||
if bucket in buckets:
|
||||
die(f"artifact bucket {bucket} is used more than once")
|
||||
buckets.add(bucket)
|
||||
if not isinstance(credential, str) or not PROFILE_RE.fullmatch(credential):
|
||||
die(f"artifact {name}.credential must be a lowercase profile name")
|
||||
endpoint = raw.get("endpoint")
|
||||
if endpoint is not None:
|
||||
parsed_endpoint = urlsplit(endpoint) if isinstance(endpoint, str) else None
|
||||
if (
|
||||
parsed_endpoint is None or parsed_endpoint.scheme not in {"http", "https"}
|
||||
or not parsed_endpoint.netloc or parsed_endpoint.username or parsed_endpoint.password
|
||||
or parsed_endpoint.query or parsed_endpoint.fragment
|
||||
):
|
||||
die(f"artifact {name}.endpoint must be an HTTP URL without credentials")
|
||||
cors_origins = _cors_origins(raw, name)
|
||||
artifacts[name] = {
|
||||
"name": name,
|
||||
"source": source,
|
||||
"bucket": bucket,
|
||||
"credential": credential,
|
||||
"cache": _cache_config(raw, name),
|
||||
"endpoint": endpoint,
|
||||
"cors_origins": cors_origins,
|
||||
"excludes": _patterns(raw, "excludes", f"artifact {name}.excludes"),
|
||||
}
|
||||
|
||||
routes = []
|
||||
route_names = set()
|
||||
route_paths = set()
|
||||
referenced = set()
|
||||
for raw in routes_cfg:
|
||||
if not isinstance(raw, dict):
|
||||
die("each route must be a mapping")
|
||||
_known_keys(
|
||||
raw,
|
||||
{"name", "path", "artifact", "access", "middlewares"},
|
||||
"route",
|
||||
)
|
||||
name = raw.get("name")
|
||||
path = raw.get("path")
|
||||
artifact = raw.get("artifact")
|
||||
access = raw.get("access")
|
||||
if not isinstance(name, str) or not NAME_RE.fullmatch(name):
|
||||
die("each route.name must be a DNS label")
|
||||
if name in route_names:
|
||||
die(f"route name {name} is used more than once")
|
||||
route_names.add(name)
|
||||
if not isinstance(path, str) or not re.fullmatch(
|
||||
r"/(?:[A-Za-z0-9._~-]+(?:/[A-Za-z0-9._~-]+)*)?", path
|
||||
):
|
||||
die(f"route {name}.path must be a canonical absolute URL path")
|
||||
if path != "/" and path.endswith("/"):
|
||||
die(f"route {name}.path must not end with /")
|
||||
if "//" in path or ".." in Path(path).parts or "?" in path or "#" in path:
|
||||
die(f"route {name}.path is not a canonical URL path")
|
||||
if path in route_paths:
|
||||
die(f"route path {path} is used more than once")
|
||||
route_paths.add(path)
|
||||
if artifact not in artifacts:
|
||||
die(f"route {name} references unknown artifact {artifact!r}")
|
||||
referenced.add(artifact)
|
||||
if access not in {"public", "protected"}:
|
||||
die(f"route {name}.access must be public or protected")
|
||||
route_middlewares = _validate_middlewares(
|
||||
raw.get("middlewares") or [], f"route {name}.middlewares"
|
||||
)
|
||||
if access == "protected":
|
||||
route_middlewares = [AUTH_MIDDLEWARE, *route_middlewares]
|
||||
if len(set(route_middlewares)) != len(route_middlewares):
|
||||
die(f"route {name}.middlewares contains a duplicate")
|
||||
routes.append({
|
||||
"name": name,
|
||||
"path": path,
|
||||
"artifact": artifact,
|
||||
"access": access,
|
||||
"middlewares": route_middlewares,
|
||||
})
|
||||
|
||||
unreferenced = sorted(set(artifacts) - referenced)
|
||||
if unreferenced:
|
||||
die(f"artifacts without routes: {', '.join(unreferenced)}")
|
||||
|
||||
for artifact_name, artifact in artifacts.items():
|
||||
access_modes = {
|
||||
route["access"] for route in routes if route["artifact"] == artifact_name
|
||||
}
|
||||
if len(access_modes) != 1:
|
||||
die(f"artifact {artifact_name} cannot cross public and protected routes")
|
||||
access = next(iter(access_modes))
|
||||
policies = [artifact["cache"]["default"], *(
|
||||
rule["value"] for rule in artifact["cache"]["rules"]
|
||||
)]
|
||||
for policy in policies:
|
||||
directives = _cache_directives(policy)
|
||||
if access == "protected" and not ({"private", "no-store"} & directives):
|
||||
die(f"protected artifact {artifact_name} cache policy must be private or no-store")
|
||||
if access == "protected" and ({"public", "s-maxage"} & directives):
|
||||
die(f"protected artifact {artifact_name} cannot use shared-cache directives")
|
||||
if access == "public" and "private" in directives:
|
||||
die(f"public artifact {artifact_name} cannot use private cache metadata")
|
||||
if access == "protected" and "*" in artifact["cors_origins"]:
|
||||
die(f"protected artifact {artifact_name} cannot allow wildcard CORS")
|
||||
|
||||
profile_access = {}
|
||||
for artifact_name, artifact in artifacts.items():
|
||||
access = next(
|
||||
route["access"] for route in routes if route["artifact"] == artifact_name
|
||||
)
|
||||
existing = profile_access.setdefault(artifact["credential"], access)
|
||||
if existing != access:
|
||||
die(
|
||||
f"credential profile {artifact['credential']} cannot cross public and protected artifacts"
|
||||
)
|
||||
|
||||
return {
|
||||
"mode": "multi",
|
||||
"domain": domain,
|
||||
"aliases": aliases,
|
||||
"artifacts": artifacts,
|
||||
"routes": routes,
|
||||
}
|
||||
|
||||
|
||||
def parse_site_yaml(site_dir):
|
||||
@@ -69,12 +332,50 @@ def parse_site_yaml(site_dir):
|
||||
if not path.exists():
|
||||
die("site.yaml not found in repo root")
|
||||
|
||||
with open(path) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
except YAMLError as error:
|
||||
die(f"site.yaml is not valid YAML: {error}")
|
||||
|
||||
if not isinstance(cfg, dict):
|
||||
die("site.yaml must contain a mapping")
|
||||
if not cfg.get("domain"):
|
||||
die("domain is required in site.yaml")
|
||||
|
||||
domain = _validate_domain(cfg["domain"], "domain")
|
||||
aliases = [
|
||||
_validate_domain(value, "aliases entry")
|
||||
for value in _string_list(cfg, "aliases")
|
||||
]
|
||||
if len(set([domain, *aliases])) != 1 + len(aliases):
|
||||
die("domain and aliases must be unique")
|
||||
|
||||
has_artifacts = "artifacts" in cfg
|
||||
has_routes = "routes" in cfg
|
||||
if has_artifacts != has_routes:
|
||||
die("artifacts and routes must be declared together")
|
||||
if has_artifacts:
|
||||
_known_keys(
|
||||
cfg,
|
||||
{"domain", "aliases", "enabled", "artifacts", "routes"},
|
||||
"site.yaml",
|
||||
)
|
||||
site = _modern_config(cfg, domain, aliases)
|
||||
site["type"] = "artifacts"
|
||||
site["enabled"] = cfg.get("enabled", True)
|
||||
if not isinstance(site["enabled"], bool):
|
||||
die("enabled must be true or false")
|
||||
site["tidy"] = False
|
||||
site["content_dir"] = ""
|
||||
site["excludes"] = []
|
||||
site["middlewares"] = []
|
||||
print("Site config:")
|
||||
print(f" domain: {site['domain']}")
|
||||
print(f" artifacts: {', '.join(site['artifacts'])}")
|
||||
print(f" routes: {', '.join(route['path'] for route in site['routes'])}")
|
||||
return site
|
||||
|
||||
site_type = cfg.get("type", "static")
|
||||
|
||||
if site_type == "docker":
|
||||
@@ -83,20 +384,26 @@ def parse_site_yaml(site_dir):
|
||||
if site_type not in VALID_TYPES:
|
||||
die(f"Unknown site type: {site_type} (valid: {', '.join(sorted(VALID_TYPES))})")
|
||||
|
||||
excludes = cfg.get("excludes") or []
|
||||
if not isinstance(excludes, list) or any(not isinstance(p, str) for p in excludes):
|
||||
die("excludes must be a list of string patterns")
|
||||
excludes = _string_list(cfg, "excludes")
|
||||
middlewares = _validate_middlewares(cfg.get("middlewares") or [], "middlewares")
|
||||
|
||||
middlewares = cfg.get("middlewares") or []
|
||||
if not isinstance(middlewares, list) or any(not isinstance(m, str) for m in middlewares):
|
||||
die("middlewares must be a list of Traefik file-provider middleware names")
|
||||
content_dir = cfg.get("content_dir", "")
|
||||
if not isinstance(content_dir, str) or Path(content_dir).is_absolute():
|
||||
die("content_dir must be a relative path")
|
||||
if ".." in Path(content_dir).parts:
|
||||
die("content_dir cannot escape the repository")
|
||||
|
||||
enabled = cfg.get("enabled", True)
|
||||
if not isinstance(enabled, bool):
|
||||
die("enabled must be true or false")
|
||||
|
||||
site = {
|
||||
"domain": cfg["domain"],
|
||||
"mode": "legacy",
|
||||
"domain": domain,
|
||||
"type": site_type,
|
||||
"enabled": cfg.get("enabled", True),
|
||||
"aliases": cfg.get("aliases") or [],
|
||||
"content_dir": cfg.get("content_dir", ""),
|
||||
"enabled": enabled,
|
||||
"aliases": aliases,
|
||||
"content_dir": content_dir,
|
||||
"tidy": cfg.get("tidy", True),
|
||||
"excludes": excludes,
|
||||
"middlewares": middlewares,
|
||||
@@ -110,21 +417,46 @@ def parse_site_yaml(site_dir):
|
||||
|
||||
def clone_apps(token):
|
||||
user = env("CI_BOT_USER", "ci-bot")
|
||||
apps_dir = Path("/tmp/apps-deploy")
|
||||
if apps_dir.exists():
|
||||
shutil.rmtree(apps_dir)
|
||||
run(f"git clone --depth 1 https://{user}:{token}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}")
|
||||
run(f"git -C {apps_dir} config user.name {user}")
|
||||
run(f"git -C {apps_dir} config user.email {user}@fritzlab.net")
|
||||
clone_root = Path(tempfile.mkdtemp(prefix="apps-deploy-"))
|
||||
apps_dir = clone_root / "repo"
|
||||
askpass = clone_root / "askpass"
|
||||
askpass.write_text(
|
||||
"#!/bin/sh\ncase \"$1\" in *Username*) printf '%s\\n' \"$GIT_AUTH_USER\" ;; "
|
||||
"*) printf '%s\\n' \"$GIT_AUTH_TOKEN\" ;; esac\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
askpass.chmod(0o700)
|
||||
git_env = git_auth_environment(token, user, askpass)
|
||||
try:
|
||||
run_args(
|
||||
["git", "clone", "--depth", "1", f"https://{GITEA_HOST}/{APPS_REPO}.git", str(apps_dir)],
|
||||
display=f"git clone --depth 1 https://{GITEA_HOST}/{APPS_REPO}.git {apps_dir}",
|
||||
env=git_env,
|
||||
)
|
||||
except Exception:
|
||||
shutil.rmtree(clone_root, ignore_errors=True)
|
||||
raise
|
||||
run_args(["git", "-C", str(apps_dir), "config", "user.name", user])
|
||||
run_args(["git", "-C", str(apps_dir), "config", "user.email", f"{user}@fritzlab.net"])
|
||||
return apps_dir
|
||||
|
||||
|
||||
def git_auth_environment(token, user, askpass):
|
||||
child_env = os.environ.copy()
|
||||
child_env["GIT_ASKPASS"] = str(askpass)
|
||||
child_env["GIT_TERMINAL_PROMPT"] = "0"
|
||||
child_env["GIT_AUTH_USER"] = user
|
||||
child_env["GIT_AUTH_TOKEN"] = token
|
||||
return child_env
|
||||
|
||||
|
||||
def render_templates(action_dir, template_vars, app_dir, manifests_dir):
|
||||
"""Render Jinja2 templates for a static-content site."""
|
||||
templates_dir = Path(action_dir) / "templates"
|
||||
jinja_env = Environment(
|
||||
loader=FileSystemLoader(str(templates_dir)),
|
||||
keep_trailing_newline=True,
|
||||
undefined=StrictUndefined,
|
||||
)
|
||||
|
||||
tmpl_names = ["app.yaml.j2", "certificate.yaml.j2", "ingress.yaml.j2",
|
||||
@@ -139,16 +471,21 @@ def render_templates(action_dir, template_vars, app_dir, manifests_dir):
|
||||
print(f" Rendered {tmpl_name} -> {dest}")
|
||||
|
||||
|
||||
def commit_and_push(apps_dir, message):
|
||||
run(f"git -C {apps_dir} add -A")
|
||||
def commit_and_push(apps_dir, message, token):
|
||||
run_args(["git", "-C", str(apps_dir), "add", "-A"])
|
||||
result = subprocess.run(
|
||||
f"git -C {apps_dir} diff --cached --quiet",
|
||||
shell=True, check=False,
|
||||
["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"],
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print("No manifest changes to commit")
|
||||
return False
|
||||
run(f"git -C {apps_dir} commit -m '{message}'")
|
||||
run(f"git -C {apps_dir} push")
|
||||
run_args(["git", "-C", str(apps_dir), "commit", "-m", message])
|
||||
git_env = git_auth_environment(
|
||||
token,
|
||||
env("CI_BOT_USER", "ci-bot"),
|
||||
apps_dir.parent / "askpass",
|
||||
)
|
||||
run_args(["git", "-C", str(apps_dir), "push"], env=git_env)
|
||||
print("Manifests pushed — ArgoCD will sync")
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user