feat(site-publish): add split-surface publishing

Authored-By: OpenAI (GPT-5) <noreply@openai.com>

README.md

@@ -120,6 +120,12 @@ revalidation, or `no-store` plus a positive max-age) are rejected. Protected
 artifacts require `private` or `no-store` and cannot emit `public`.
 Metadata restamping transfers each artifact once even on a no-op publication;
 that is the cost of making policy changes effective on unchanged Garage objects.
+An immutable cache path is excluded from sync and deletion. Every object key in
+that path must contain exactly one full publication SHA-256, calculated over its
+cache policy, content type, and bytes. That content address makes concurrent
+writes identical even though Garage v2.2.0 has no conditional destination
+write. An identical retry converges; a changed object, missing digest metadata,
+wrong address, or nested policy under that immutable prefix fails publication.
 
 Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net`
 ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route

scripts/deploy.py

@@ -1,8 +1,13 @@
 """Deploy phase — S3 sync, manifest rendering, alias reconcile."""
 
+import fnmatch
+import hashlib
 import json
+import mimetypes
 import os
+import re
 import shutil
+import subprocess
 import tempfile
 from pathlib import Path
 from urllib.error import HTTPError, URLError
@@ -47,6 +52,103 @@ def validate_publication_environment(cfg):
         die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
 
 
+def _is_immutable(rule):
+    return "immutable" in {
+        part.strip().lower().split("=", 1)[0]
+        for part in rule["cache_control"].split(",")
+    }
+
+
+def _aws_capture(args, aws_env):
+    """Run a non-streaming AWS request without exposing environment credentials."""
+    print(f"  $ {' '.join(str(part) for part in args)}")
+    return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
+
+
+def _immutable_head(endpoint, bucket, key, aws_env):
+    args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
+            "--bucket", bucket, "--key", key, "--output", "json"]
+    result = _aws_capture(args, aws_env)
+    if result.returncode == 0:
+        return json.loads(result.stdout)
+    error = f"{result.stdout}\n{result.stderr}"
+    if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
+        return None
+    raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
+
+
+def _immutable_digests(source, cache_control, content_type):
+    with source.open("rb") as stream:
+        content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
+    publication = hashlib.sha256()
+    publication.update(cache_control.encode())
+    publication.update(b"\0")
+    publication.update(content_type.encode())
+    publication.update(b"\0")
+    with source.open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            publication.update(block)
+    return content_digest, publication.hexdigest()
+
+
+def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
+    metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
+    return (
+        metadata.get("sha256") == content_digest
+        and metadata.get("publication-sha256") == publication_digest
+        and info.get("CacheControl") == cache_control
+        and info.get("ContentType") == content_type
+    )
+
+
+def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
+    """Publish a content-addressed key; identical retries converge."""
+    content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
+    content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
+    address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
+    if address_digests != [publication_digest]:
+        raise RuntimeError(
+            f"immutable key must contain its one publication SHA-256 {publication_digest}: "
+            f"s3://{bucket}/{key}"
+        )
+    existing = _immutable_head(endpoint, bucket, key, aws_env)
+    if existing is not None:
+        if _same_immutable_object(
+            existing, content_digest, publication_digest, cache_control, content_type,
+        ):
+            print(f"  Immutable object already matches: s3://{bucket}/{key}")
+            return False
+        raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
+    args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
+            "--bucket", bucket, "--key", key, "--body", str(source),
+            "--content-type", content_type, "--cache-control", cache_control,
+            "--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
+    result = _aws_capture(args, aws_env)
+    if result.returncode == 0:
+        return True
+    error = f"{result.stdout}\n{result.stderr}"
+    raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
+
+
+def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
+    """Publish one immutable cache partition without overwrite or deletion."""
+    rule_root = html_dir / rule["path"]
+    child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
+                   if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
+    object_prefix = route["path"].strip("/")
+    for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
+        relative = source.relative_to(html_dir).as_posix()
+        if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
+            continue
+        if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
+            continue
+        key = "/".join(part for part in (object_prefix, relative) if part)
+        publish_immutable_file(
+            artifact["s3_endpoint"], artifact["bucket"], key, source,
+            rule["cache_control"], aws_env,
+        )
+
+
 def s3_sync(artifact, route, site_dir, credential_env_names=None):
     endpoint = artifact["s3_endpoint"]
     html_dir = site_dir / artifact["build_dir"]
@@ -68,6 +170,7 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     bucket_destination = f"s3://{bucket}/"
     destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}"
     default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
+    immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
     # `excludes` are patterns (site.yaml `excludes:` list) that should never
     # be uploaded *and* should never be deleted from the bucket — escape hatch
     # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
@@ -83,15 +186,20 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     # so a fresh upload always carries the right MIME type.
     stage = None
     sync_source = html_dir
-    sync_excludes = exclude_args
+    sync_excludes = [*exclude_args,
+                     *(arg for path in immutable_paths for arg in ("--exclude", f"{path}/*"))]
     if object_prefix:
         stage = tempfile.TemporaryDirectory()
         sync_source = Path(stage.name)
         staged_artifact = sync_source / object_prefix
         staged_artifact.parent.mkdir(parents=True, exist_ok=True)
         staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True)
-        sync_excludes = [arg for pattern in artifact["excludes"]
-                         for arg in ("--exclude", f"{object_prefix}/{pattern}")]
+        sync_excludes = [
+            *(arg for pattern in artifact["excludes"]
+              for arg in ("--exclude", f"{object_prefix}/{pattern}")),
+            *(arg for path in immutable_paths
+              for arg in ("--exclude", f"{object_prefix}/{path}/*")),
+        ]
     try:
         # Sync the complete bucket authority so moving a route prefix also
         # deletes objects under its old prefix instead of leaving them public.
@@ -110,6 +218,9 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     for rule in artifact["cache_rules"]:
         if not rule["path"]:
             continue
+        if _is_immutable(rule):
+            publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
+            continue
         include = f"{rule['path'].rstrip('/')}/*"
         child_filters = [arg for path in specific_paths
                          if path.startswith(f"{rule['path'].rstrip('/')}/")
@@ -208,7 +319,7 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
 
     render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
 
-    commit_and_push(apps_dir, f"Deploy {site_name}")
+    commit_and_push(apps_dir, f"Deploy {site_name}", token)
 
 
 def decommission(site_name, token, buckets=None):
@@ -219,7 +330,7 @@ def decommission(site_name, token, buckets=None):
         print(f"No manifests for {site_name} — nothing to remove")
         return
     shutil.rmtree(site_path)
-    commit_and_push(apps_dir, f"Decommission {site_name}")
+    commit_and_push(apps_dir, f"Decommission {site_name}", token)
     for bucket in buckets or [site_name]:
         print(f"Bucket {bucket} and its objects are NOT purged automatically.")
         print(f"  garage bucket delete {bucket} --yes")

scripts/utils.py

@@ -270,6 +270,14 @@ def _artifact(item, index):
     if "" not in paths:
         raise ConfigError(f"{label}.cache.rules must declare a '/' default")
     cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"]))
+    immutable_paths = [rule["path"] for rule in cache_rules
+                       if "immutable" in {part.strip().lower().split("=", 1)[0]
+                                          for part in rule["cache_control"].split(",")}]
+    if "" in immutable_paths:
+        raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'")
+    for path in immutable_paths:
+        if any(other.startswith(f"{path}/") for other in paths):
+            raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}")
     authority = _hostname(
         publish.get("website_authority", f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}"),
         f"{label}.publish.website_authority",
@@ -436,10 +444,7 @@ def clone_apps(token):
     apps_dir = Path("/tmp/apps-deploy")
     if apps_dir.exists():
         shutil.rmtree(apps_dir)
-    clone_env = os.environ.copy()
-    clone_env["CI_BOT_TOKEN"] = token
-    clone_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
-    clone_env["GIT_TERMINAL_PROMPT"] = "0"
+    clone_env = git_auth_env(token)
     url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git"
     run(["git", "clone", "--depth", "1", url, str(apps_dir)],
         display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env)
@@ -477,13 +482,23 @@ def render_templates(action_dir, template_vars, app_dir, manifests_dir):
         print(f"  Rendered {out_name}.j2 -> {destination}")
 
 
-def commit_and_push(apps_dir, message):
+def git_auth_env(token):
+    """Return credential-safe Git authentication shared by clone and push."""
+    auth_env = os.environ.copy()
+    auth_env["CI_BOT_TOKEN"] = token
+    auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
+    auth_env["GIT_TERMINAL_PROMPT"] = "0"
+    return auth_env
+
+
+def commit_and_push(apps_dir, message, token=None):
     run(["git", "-C", str(apps_dir), "add", "-A"])
     result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False)
     if result.returncode == 0:
         print("No manifest changes to commit")
         return False
     run(["git", "-C", str(apps_dir), "commit", "-m", message])
-    run(["git", "-C", str(apps_dir), "push"])
+    push_env = git_auth_env(token) if token else None
+    run(["git", "-C", str(apps_dir), "push"], env=push_env)
     print("Manifests pushed — ArgoCD will sync")
     return True

tests/test_contract.py

@@ -1,11 +1,19 @@
+import base64
 import copy
+import hashlib
 import io
+import json
 import os
+import socket
+import subprocess
 import sys
 import tempfile
+import threading
 import unittest
 from contextlib import redirect_stderr, redirect_stdout
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
+from urllib.parse import urlsplit
 from unittest.mock import patch
 
 import yaml
@@ -23,6 +31,65 @@ def fixture(name):
     return yaml.safe_load((ROOT / "tests" / "fixtures" / name).read_text())
 
 
+class IPv6GitHTTPServer(ThreadingHTTPServer):
+    address_family = socket.AF_INET6
+
+
+class AuthenticatedGitHandler(BaseHTTPRequestHandler):
+    project_root = None
+    expected_authorization = None
+
+    def log_message(self, _format, *_args):
+        pass
+
+    def do_GET(self):
+        self._git_backend()
+
+    def do_POST(self):
+        self._git_backend()
+
+    def _git_backend(self):
+        if self.headers.get("Authorization") != self.expected_authorization:
+            self.send_response(401)
+            self.send_header("WWW-Authenticate", 'Basic realm="test"')
+            self.end_headers()
+            return
+        parsed = urlsplit(self.path)
+        length = int(self.headers.get("Content-Length", "0"))
+        request_body = self.rfile.read(length) if length else b""
+        backend_env = os.environ.copy()
+        backend_env.update({
+            "GIT_PROJECT_ROOT": str(self.project_root),
+            "GIT_HTTP_EXPORT_ALL": "1",
+            "PATH_INFO": parsed.path,
+            "QUERY_STRING": parsed.query,
+            "REQUEST_METHOD": self.command,
+            "CONTENT_TYPE": self.headers.get("Content-Type", ""),
+            "CONTENT_LENGTH": str(length),
+            "REMOTE_USER": "ci-bot",
+            "REMOTE_ADDR": "::1",
+            "GATEWAY_INTERFACE": "CGI/1.1",
+            "SERVER_PROTOCOL": "HTTP/1.1",
+        })
+        result = subprocess.run(
+            ["git", "http-backend"], input=request_body, env=backend_env,
+            capture_output=True, check=True,
+        )
+        raw_headers, response_body = result.stdout.split(b"\r\n\r\n", 1)
+        headers, status = [], 200
+        for line in raw_headers.decode().split("\r\n"):
+            name, value = line.split(":", 1)
+            if name.lower() == "status":
+                status = int(value.strip().split(" ", 1)[0])
+            else:
+                headers.append((name, value.strip()))
+        self.send_response(status)
+        for name, value in headers:
+            self.send_header(name, value)
+        self.end_headers()
+        self.wfile.write(response_body)
+
+
 class ConfigContractTests(unittest.TestCase):
     def setUp(self):
         self.raw = fixture("split-site.yaml")
@@ -99,6 +166,14 @@ class ConfigContractTests(unittest.TestCase):
             "immutable requires",
         )
 
+    def test_cache_policy_cannot_nest_below_immutable_path(self):
+        def mutate(raw):
+            raw["artifacts"][1]["cache"]["rules"].append({
+                "path": "releases/candidates",
+                "cache_control": "public, max-age=0, must-revalidate",
+            })
+        self.assert_invalid(mutate, "cannot nest another policy under immutable /releases")
+
     def test_public_cache_is_rejected_on_protected_route(self):
         self.assert_invalid(
             lambda raw: raw["artifacts"][0]["cache"]["rules"][0].__setitem__(
@@ -215,6 +290,56 @@ class PublishingTests(unittest.TestCase):
             self.assertNotIn(secret, " ".join(command))
             self.assertNotIn(secret, kwargs.get("display", ""))
 
+    def test_askpass_authenticates_clone_and_push_round_trip(self):
+        token = "round-trip-token"
+        expected = "Basic " + base64.b64encode(f"ci-bot:{token}".encode()).decode()
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            bare = root / "repo.git"
+            seed = root / "seed"
+            checkout = root / "checkout"
+            subprocess.run(["git", "init", "--bare", "--initial-branch=main", str(bare)], check=True,
+                           stdout=subprocess.DEVNULL)
+            subprocess.run(["git", "-C", str(bare), "config", "http.receivepack", "true"], check=True)
+            subprocess.run(["git", "init", "--initial-branch=main", str(seed)], check=True,
+                           stdout=subprocess.DEVNULL)
+            subprocess.run(["git", "-C", str(seed), "config", "user.name", "Test"], check=True)
+            subprocess.run(["git", "-C", str(seed), "config", "user.email", "test@example.invalid"], check=True)
+            (seed / "README.md").write_text("seed\n")
+            subprocess.run(["git", "-C", str(seed), "add", "README.md"], check=True)
+            subprocess.run(["git", "-C", str(seed), "commit", "-m", "seed"], check=True,
+                           stdout=subprocess.DEVNULL)
+            subprocess.run(["git", "-C", str(seed), "push", str(bare), "main"], check=True,
+                           stdout=subprocess.DEVNULL)
+
+            handler = type("GitHandler", (AuthenticatedGitHandler,), {
+                "project_root": root, "expected_authorization": expected,
+            })
+            server = IPv6GitHTTPServer(("::1", 0), handler)
+            thread = threading.Thread(target=server.serve_forever, daemon=True)
+            thread.start()
+            try:
+                url = f"http://ci-bot@[::1]:{server.server_port}/repo.git"
+                with patch.dict(os.environ, {"NO_PROXY": "::1,[::1]", "no_proxy": "::1,[::1]"}, clear=False):
+                    subprocess.run(
+                        ["git", "clone", url, str(checkout)], env=utils.git_auth_env(token),
+                        check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                    )
+                    subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Test"], check=True)
+                    subprocess.run(["git", "-C", str(checkout), "config", "user.email", "test@example.invalid"],
+                                   check=True)
+                    (checkout / "roundtrip.txt").write_text("authenticated\n")
+                    utils.commit_and_push(checkout, "authenticated round trip", token)
+            finally:
+                server.shutdown()
+                server.server_close()
+                thread.join(timeout=5)
+            result = subprocess.run(
+                ["git", "--git-dir", str(bare), "show", "main:roundtrip.txt"],
+                check=True, text=True, capture_output=True,
+            )
+            self.assertEqual("authenticated\n", result.stdout)
+
     def test_cache_headers_credentials_and_route_prefix_are_separate(self):
         cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
         artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
@@ -235,7 +360,9 @@ class PublishingTests(unittest.TestCase):
             output = io.StringIO()
             with patch.dict(os.environ, {
                 "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret
-            }, clear=False), patch.object(deploy, "run", side_effect=capture), redirect_stdout(output):
+            }, clear=False), patch.object(deploy, "run", side_effect=capture), \
+                    patch.object(deploy, "publish_immutable_rule") as immutable_publish, \
+                    redirect_stdout(output):
                 deploy.s3_sync(artifact, route, root)
 
             self.assertTrue(all(secret not in " ".join(command) for command, _ in commands))
@@ -244,13 +371,16 @@ class PublishingTests(unittest.TestCase):
             self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands))
             rendered = [" ".join(command) for command, _ in commands]
             self.assertIn("s3://baseline-dist/", rendered[0])
+            self.assertIn("dist/releases/*", rendered[0])
             self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:]))
-            self.assertTrue(any("releases/" in command and
-                                "public, max-age=31536000, immutable" in command
-                                for command in rendered))
             self.assertTrue(any("channels/" in command and
                                 "public, max-age=0, must-revalidate" in command
                                 for command in rendered))
+            immutable_publish.assert_called_once()
+            self.assertEqual(
+                "public, max-age=31536000, immutable",
+                immutable_publish.call_args.args[2]["cache_control"],
+            )
 
     def test_absent_artifact_is_detected_before_publish(self):
         cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
@@ -269,5 +399,87 @@ class PublishingTests(unittest.TestCase):
                 deploy.validate_artifact_output(Path(tmp), artifact)
 
 
+class ImmutablePublicationTests(unittest.TestCase):
+    CACHE = "public, max-age=31536000, immutable"
+
+    def result(self, returncode, stdout="", stderr=""):
+        return subprocess.CompletedProcess([], returncode, stdout, stderr)
+
+    def key_and_digests(self, source):
+        content_type = "text/javascript"
+        content_digest, publication_digest = deploy._immutable_digests(
+            source, self.CACHE, content_type,
+        )
+        return f"dist/releases/release-{publication_digest}.js", content_digest, publication_digest
+
+    def test_new_key_uses_content_address_and_digest_metadata(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            source = Path(tmp) / "release.js"
+            source.write_text("fixed release")
+            key, content_digest, publication_digest = self.key_and_digests(source)
+            calls = []
+
+            def capture(args, _env):
+                calls.append(args)
+                return self.result(1, stderr="404 Not Found") if len(calls) == 1 else self.result(0, "{}")
+
+            with patch.object(deploy, "_aws_capture", side_effect=capture):
+                created = deploy.publish_immutable_file(
+                    "http://garage-s3.storage.svc:3900", "dist-bucket",
+                    key, source, self.CACHE, {},
+                )
+            self.assertTrue(created)
+            put = calls[1]
+            self.assertEqual("put-object", put[4])
+            self.assertNotIn("--if-none-match", put)
+            self.assertEqual(
+                f"sha256={content_digest},publication-sha256={publication_digest}",
+                put[put.index("--metadata") + 1],
+            )
+
+    def test_identical_retry_converges_without_put(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            source = Path(tmp) / "release.js"
+            source.write_text("fixed release")
+            key, content_digest, publication_digest = self.key_and_digests(source)
+            head = json.dumps({
+                "Metadata": {"sha256": content_digest, "publication-sha256": publication_digest},
+                "CacheControl": self.CACHE,
+                "ContentType": "text/javascript",
+            })
+            with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)) as request:
+                created = deploy.publish_immutable_file(
+                    "http://garage-s3.storage.svc:3900", "dist-bucket",
+                    key, source, self.CACHE, {},
+                )
+            self.assertFalse(created)
+            self.assertEqual(1, request.call_count)
+
+    def test_changed_immutable_key_is_refused(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            source = Path(tmp) / "release.js"
+            source.write_text("changed release")
+            key, _, _ = self.key_and_digests(source)
+            head = json.dumps({"Metadata": {"sha256": "different"}, "CacheControl": self.CACHE})
+            with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)), \
+                    self.assertRaisesRegex(RuntimeError, "immutable object differs"):
+                deploy.publish_immutable_file(
+                    "http://garage-s3.storage.svc:3900", "dist-bucket",
+                    key, source, self.CACHE, {},
+                )
+
+    def test_immutable_key_without_publication_digest_is_refused_before_s3(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            source = Path(tmp) / "release.js"
+            source.write_text("fixed release")
+            with patch.object(deploy, "_aws_capture") as request, \
+                    self.assertRaisesRegex(RuntimeError, "must contain its one publication SHA-256"):
+                deploy.publish_immutable_file(
+                    "http://garage-s3.storage.svc:3900", "dist-bucket",
+                    "dist/releases/release.js", source, self.CACHE, {},
+                )
+            request.assert_not_called()
+
+
 if __name__ == "__main__":
     unittest.main()