feat(site-publish): add split-surface publishing

Authored-By: OpenAI (GPT-5) <noreply@openai.com>

README.md

@@ -120,12 +120,6 @@ revalidation, or `no-store` plus a positive max-age) are rejected. Protected
 artifacts require `private` or `no-store` and cannot emit `public`.
 Metadata restamping transfers each artifact once even on a no-op publication;
 that is the cost of making policy changes effective on unchanged Garage objects.
-An immutable cache path is excluded from sync and deletion. Every object key in
-that path must contain exactly one full publication SHA-256, calculated over its
-cache policy, content type, and bytes. That content address makes concurrent
-writes identical even though Garage v2.2.0 has no conditional destination
-write. An identical retry converges; a changed object, missing digest metadata,
-wrong address, or nested policy under that immutable prefix fails publication.
 
 Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net`
 ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route

scripts/deploy.py

@@ -1,13 +1,8 @@
 """Deploy phase — S3 sync, manifest rendering, alias reconcile."""
 
-import fnmatch
-import hashlib
 import json
-import mimetypes
 import os
-import re
 import shutil
-import subprocess
 import tempfile
 from pathlib import Path
 from urllib.error import HTTPError, URLError
@@ -52,103 +47,6 @@ def validate_publication_environment(cfg):
         die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
 
 
-def _is_immutable(rule):
-    return "immutable" in {
-        part.strip().lower().split("=", 1)[0]
-        for part in rule["cache_control"].split(",")
-    }
-
-
-def _aws_capture(args, aws_env):
-    """Run a non-streaming AWS request without exposing environment credentials."""
-    print(f"  $ {' '.join(str(part) for part in args)}")
-    return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
-
-
-def _immutable_head(endpoint, bucket, key, aws_env):
-    args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
-            "--bucket", bucket, "--key", key, "--output", "json"]
-    result = _aws_capture(args, aws_env)
-    if result.returncode == 0:
-        return json.loads(result.stdout)
-    error = f"{result.stdout}\n{result.stderr}"
-    if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
-        return None
-    raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
-
-
-def _immutable_digests(source, cache_control, content_type):
-    with source.open("rb") as stream:
-        content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
-    publication = hashlib.sha256()
-    publication.update(cache_control.encode())
-    publication.update(b"\0")
-    publication.update(content_type.encode())
-    publication.update(b"\0")
-    with source.open("rb") as stream:
-        for block in iter(lambda: stream.read(1024 * 1024), b""):
-            publication.update(block)
-    return content_digest, publication.hexdigest()
-
-
-def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
-    metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
-    return (
-        metadata.get("sha256") == content_digest
-        and metadata.get("publication-sha256") == publication_digest
-        and info.get("CacheControl") == cache_control
-        and info.get("ContentType") == content_type
-    )
-
-
-def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
-    """Publish a content-addressed key; identical retries converge."""
-    content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
-    content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
-    address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
-    if address_digests != [publication_digest]:
-        raise RuntimeError(
-            f"immutable key must contain its one publication SHA-256 {publication_digest}: "
-            f"s3://{bucket}/{key}"
-        )
-    existing = _immutable_head(endpoint, bucket, key, aws_env)
-    if existing is not None:
-        if _same_immutable_object(
-            existing, content_digest, publication_digest, cache_control, content_type,
-        ):
-            print(f"  Immutable object already matches: s3://{bucket}/{key}")
-            return False
-        raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
-    args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
-            "--bucket", bucket, "--key", key, "--body", str(source),
-            "--content-type", content_type, "--cache-control", cache_control,
-            "--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
-    result = _aws_capture(args, aws_env)
-    if result.returncode == 0:
-        return True
-    error = f"{result.stdout}\n{result.stderr}"
-    raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
-
-
-def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
-    """Publish one immutable cache partition without overwrite or deletion."""
-    rule_root = html_dir / rule["path"]
-    child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
-                   if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
-    object_prefix = route["path"].strip("/")
-    for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
-        relative = source.relative_to(html_dir).as_posix()
-        if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
-            continue
-        if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
-            continue
-        key = "/".join(part for part in (object_prefix, relative) if part)
-        publish_immutable_file(
-            artifact["s3_endpoint"], artifact["bucket"], key, source,
-            rule["cache_control"], aws_env,
-        )
-
-
 def s3_sync(artifact, route, site_dir, credential_env_names=None):
     endpoint = artifact["s3_endpoint"]
     html_dir = site_dir / artifact["build_dir"]
@@ -170,7 +68,6 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     bucket_destination = f"s3://{bucket}/"
     destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}"
     default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
-    immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
     # `excludes` are patterns (site.yaml `excludes:` list) that should never
     # be uploaded *and* should never be deleted from the bucket — escape hatch
     # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
@@ -186,20 +83,15 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     # so a fresh upload always carries the right MIME type.
     stage = None
     sync_source = html_dir
-    sync_excludes = [*exclude_args,
+    sync_excludes = exclude_args
-                     *(arg for path in immutable_paths for arg in ("--exclude", f"{path}/*"))]
     if object_prefix:
         stage = tempfile.TemporaryDirectory()
         sync_source = Path(stage.name)
         staged_artifact = sync_source / object_prefix
         staged_artifact.parent.mkdir(parents=True, exist_ok=True)
         staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True)
-        sync_excludes = [
+        sync_excludes = [arg for pattern in artifact["excludes"]
-            *(arg for pattern in artifact["excludes"]
+                         for arg in ("--exclude", f"{object_prefix}/{pattern}")]
-              for arg in ("--exclude", f"{object_prefix}/{pattern}")),
-            *(arg for path in immutable_paths
-              for arg in ("--exclude", f"{object_prefix}/{path}/*")),
-        ]
     try:
         # Sync the complete bucket authority so moving a route prefix also
         # deletes objects under its old prefix instead of leaving them public.
@@ -218,9 +110,6 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     for rule in artifact["cache_rules"]:
         if not rule["path"]:
             continue
-        if _is_immutable(rule):
-            publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
-            continue
         include = f"{rule['path'].rstrip('/')}/*"
         child_filters = [arg for path in specific_paths
                          if path.startswith(f"{rule['path'].rstrip('/')}/")
@@ -319,7 +208,7 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
 
     render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
 
-    commit_and_push(apps_dir, f"Deploy {site_name}", token)
+    commit_and_push(apps_dir, f"Deploy {site_name}")
 
 
 def decommission(site_name, token, buckets=None):
@@ -330,7 +219,7 @@ def decommission(site_name, token, buckets=None):
         print(f"No manifests for {site_name} — nothing to remove")
         return
     shutil.rmtree(site_path)
-    commit_and_push(apps_dir, f"Decommission {site_name}", token)
+    commit_and_push(apps_dir, f"Decommission {site_name}")
     for bucket in buckets or [site_name]:
         print(f"Bucket {bucket} and its objects are NOT purged automatically.")
         print(f"  garage bucket delete {bucket} --yes")

scripts/utils.py

@@ -270,14 +270,6 @@ def _artifact(item, index):
     if "" not in paths:
         raise ConfigError(f"{label}.cache.rules must declare a '/' default")
     cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"]))
-    immutable_paths = [rule["path"] for rule in cache_rules
-                       if "immutable" in {part.strip().lower().split("=", 1)[0]
-                                          for part in rule["cache_control"].split(",")}]
-    if "" in immutable_paths:
-        raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'")
-    for path in immutable_paths:
-        if any(other.startswith(f"{path}/") for other in paths):
-            raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}")
     authority = _hostname(
         publish.get("website_authority", f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}"),
         f"{label}.publish.website_authority",
@@ -444,7 +436,10 @@ def clone_apps(token):
     apps_dir = Path("/tmp/apps-deploy")
     if apps_dir.exists():
         shutil.rmtree(apps_dir)
-    clone_env = git_auth_env(token)
+    clone_env = os.environ.copy()
+    clone_env["CI_BOT_TOKEN"] = token
+    clone_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
+    clone_env["GIT_TERMINAL_PROMPT"] = "0"
     url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git"
     run(["git", "clone", "--depth", "1", url, str(apps_dir)],
         display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env)
@@ -482,23 +477,13 @@ def render_templates(action_dir, template_vars, app_dir, manifests_dir):
         print(f"  Rendered {out_name}.j2 -> {destination}")
 
 
-def git_auth_env(token):
+def commit_and_push(apps_dir, message):
-    """Return credential-safe Git authentication shared by clone and push."""
-    auth_env = os.environ.copy()
-    auth_env["CI_BOT_TOKEN"] = token
-    auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
-    auth_env["GIT_TERMINAL_PROMPT"] = "0"
-    return auth_env
-
-
-def commit_and_push(apps_dir, message, token=None):
     run(["git", "-C", str(apps_dir), "add", "-A"])
     result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False)
     if result.returncode == 0:
         print("No manifest changes to commit")
         return False
     run(["git", "-C", str(apps_dir), "commit", "-m", message])
-    push_env = git_auth_env(token) if token else None
+    run(["git", "-C", str(apps_dir), "push"])
-    run(["git", "-C", str(apps_dir), "push"], env=push_env)
     print("Manifests pushed — ArgoCD will sync")
     return True

tests/test_contract.py

@@ -1,19 +1,11 @@
-import base64
 import copy
-import hashlib
 import io
-import json
 import os
-import socket
-import subprocess
 import sys
 import tempfile
-import threading
 import unittest
 from contextlib import redirect_stderr, redirect_stdout
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
-from urllib.parse import urlsplit
 from unittest.mock import patch
 
 import yaml
@@ -31,65 +23,6 @@ def fixture(name):
     return yaml.safe_load((ROOT / "tests" / "fixtures" / name).read_text())
 
 
-class IPv6GitHTTPServer(ThreadingHTTPServer):
-    address_family = socket.AF_INET6
-
-
-class AuthenticatedGitHandler(BaseHTTPRequestHandler):
-    project_root = None
-    expected_authorization = None
-
-    def log_message(self, _format, *_args):
-        pass
-
-    def do_GET(self):
-        self._git_backend()
-
-    def do_POST(self):
-        self._git_backend()
-
-    def _git_backend(self):
-        if self.headers.get("Authorization") != self.expected_authorization:
-            self.send_response(401)
-            self.send_header("WWW-Authenticate", 'Basic realm="test"')
-            self.end_headers()
-            return
-        parsed = urlsplit(self.path)
-        length = int(self.headers.get("Content-Length", "0"))
-        request_body = self.rfile.read(length) if length else b""
-        backend_env = os.environ.copy()
-        backend_env.update({
-            "GIT_PROJECT_ROOT": str(self.project_root),
-            "GIT_HTTP_EXPORT_ALL": "1",
-            "PATH_INFO": parsed.path,
-            "QUERY_STRING": parsed.query,
-            "REQUEST_METHOD": self.command,
-            "CONTENT_TYPE": self.headers.get("Content-Type", ""),
-            "CONTENT_LENGTH": str(length),
-            "REMOTE_USER": "ci-bot",
-            "REMOTE_ADDR": "::1",
-            "GATEWAY_INTERFACE": "CGI/1.1",
-            "SERVER_PROTOCOL": "HTTP/1.1",
-        })
-        result = subprocess.run(
-            ["git", "http-backend"], input=request_body, env=backend_env,
-            capture_output=True, check=True,
-        )
-        raw_headers, response_body = result.stdout.split(b"\r\n\r\n", 1)
-        headers, status = [], 200
-        for line in raw_headers.decode().split("\r\n"):
-            name, value = line.split(":", 1)
-            if name.lower() == "status":
-                status = int(value.strip().split(" ", 1)[0])
-            else:
-                headers.append((name, value.strip()))
-        self.send_response(status)
-        for name, value in headers:
-            self.send_header(name, value)
-        self.end_headers()
-        self.wfile.write(response_body)
-
-
 class ConfigContractTests(unittest.TestCase):
     def setUp(self):
         self.raw = fixture("split-site.yaml")
@@ -166,14 +99,6 @@ class ConfigContractTests(unittest.TestCase):
             "immutable requires",
         )
 
-    def test_cache_policy_cannot_nest_below_immutable_path(self):
-        def mutate(raw):
-            raw["artifacts"][1]["cache"]["rules"].append({
-                "path": "releases/candidates",
-                "cache_control": "public, max-age=0, must-revalidate",
-            })
-        self.assert_invalid(mutate, "cannot nest another policy under immutable /releases")
-
     def test_public_cache_is_rejected_on_protected_route(self):
         self.assert_invalid(
             lambda raw: raw["artifacts"][0]["cache"]["rules"][0].__setitem__(
@@ -290,56 +215,6 @@ class PublishingTests(unittest.TestCase):
             self.assertNotIn(secret, " ".join(command))
             self.assertNotIn(secret, kwargs.get("display", ""))
 
-    def test_askpass_authenticates_clone_and_push_round_trip(self):
-        token = "round-trip-token"
-        expected = "Basic " + base64.b64encode(f"ci-bot:{token}".encode()).decode()
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            bare = root / "repo.git"
-            seed = root / "seed"
-            checkout = root / "checkout"
-            subprocess.run(["git", "init", "--bare", "--initial-branch=main", str(bare)], check=True,
-                           stdout=subprocess.DEVNULL)
-            subprocess.run(["git", "-C", str(bare), "config", "http.receivepack", "true"], check=True)
-            subprocess.run(["git", "init", "--initial-branch=main", str(seed)], check=True,
-                           stdout=subprocess.DEVNULL)
-            subprocess.run(["git", "-C", str(seed), "config", "user.name", "Test"], check=True)
-            subprocess.run(["git", "-C", str(seed), "config", "user.email", "test@example.invalid"], check=True)
-            (seed / "README.md").write_text("seed\n")
-            subprocess.run(["git", "-C", str(seed), "add", "README.md"], check=True)
-            subprocess.run(["git", "-C", str(seed), "commit", "-m", "seed"], check=True,
-                           stdout=subprocess.DEVNULL)
-            subprocess.run(["git", "-C", str(seed), "push", str(bare), "main"], check=True,
-                           stdout=subprocess.DEVNULL)
-
-            handler = type("GitHandler", (AuthenticatedGitHandler,), {
-                "project_root": root, "expected_authorization": expected,
-            })
-            server = IPv6GitHTTPServer(("::1", 0), handler)
-            thread = threading.Thread(target=server.serve_forever, daemon=True)
-            thread.start()
-            try:
-                url = f"http://ci-bot@[::1]:{server.server_port}/repo.git"
-                with patch.dict(os.environ, {"NO_PROXY": "::1,[::1]", "no_proxy": "::1,[::1]"}, clear=False):
-                    subprocess.run(
-                        ["git", "clone", url, str(checkout)], env=utils.git_auth_env(token),
-                        check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
-                    )
-                    subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Test"], check=True)
-                    subprocess.run(["git", "-C", str(checkout), "config", "user.email", "test@example.invalid"],
-                                   check=True)
-                    (checkout / "roundtrip.txt").write_text("authenticated\n")
-                    utils.commit_and_push(checkout, "authenticated round trip", token)
-            finally:
-                server.shutdown()
-                server.server_close()
-                thread.join(timeout=5)
-            result = subprocess.run(
-                ["git", "--git-dir", str(bare), "show", "main:roundtrip.txt"],
-                check=True, text=True, capture_output=True,
-            )
-            self.assertEqual("authenticated\n", result.stdout)
-
     def test_cache_headers_credentials_and_route_prefix_are_separate(self):
         cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
         artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
@@ -360,9 +235,7 @@ class PublishingTests(unittest.TestCase):
             output = io.StringIO()
             with patch.dict(os.environ, {
                 "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret
-            }, clear=False), patch.object(deploy, "run", side_effect=capture), \
+            }, clear=False), patch.object(deploy, "run", side_effect=capture), redirect_stdout(output):
-                    patch.object(deploy, "publish_immutable_rule") as immutable_publish, \
-                    redirect_stdout(output):
                 deploy.s3_sync(artifact, route, root)
 
             self.assertTrue(all(secret not in " ".join(command) for command, _ in commands))
@@ -371,16 +244,13 @@ class PublishingTests(unittest.TestCase):
             self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands))
             rendered = [" ".join(command) for command, _ in commands]
             self.assertIn("s3://baseline-dist/", rendered[0])
-            self.assertIn("dist/releases/*", rendered[0])
             self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:]))
+            self.assertTrue(any("releases/" in command and
+                                "public, max-age=31536000, immutable" in command
+                                for command in rendered))
             self.assertTrue(any("channels/" in command and
                                 "public, max-age=0, must-revalidate" in command
                                 for command in rendered))
-            immutable_publish.assert_called_once()
-            self.assertEqual(
-                "public, max-age=31536000, immutable",
-                immutable_publish.call_args.args[2]["cache_control"],
-            )
 
     def test_absent_artifact_is_detected_before_publish(self):
         cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
@@ -399,87 +269,5 @@ class PublishingTests(unittest.TestCase):
                 deploy.validate_artifact_output(Path(tmp), artifact)
 
 
-class ImmutablePublicationTests(unittest.TestCase):
-    CACHE = "public, max-age=31536000, immutable"
-
-    def result(self, returncode, stdout="", stderr=""):
-        return subprocess.CompletedProcess([], returncode, stdout, stderr)
-
-    def key_and_digests(self, source):
-        content_type = "text/javascript"
-        content_digest, publication_digest = deploy._immutable_digests(
-            source, self.CACHE, content_type,
-        )
-        return f"dist/releases/release-{publication_digest}.js", content_digest, publication_digest
-
-    def test_new_key_uses_content_address_and_digest_metadata(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            source = Path(tmp) / "release.js"
-            source.write_text("fixed release")
-            key, content_digest, publication_digest = self.key_and_digests(source)
-            calls = []
-
-            def capture(args, _env):
-                calls.append(args)
-                return self.result(1, stderr="404 Not Found") if len(calls) == 1 else self.result(0, "{}")
-
-            with patch.object(deploy, "_aws_capture", side_effect=capture):
-                created = deploy.publish_immutable_file(
-                    "http://garage-s3.storage.svc:3900", "dist-bucket",
-                    key, source, self.CACHE, {},
-                )
-            self.assertTrue(created)
-            put = calls[1]
-            self.assertEqual("put-object", put[4])
-            self.assertNotIn("--if-none-match", put)
-            self.assertEqual(
-                f"sha256={content_digest},publication-sha256={publication_digest}",
-                put[put.index("--metadata") + 1],
-            )
-
-    def test_identical_retry_converges_without_put(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            source = Path(tmp) / "release.js"
-            source.write_text("fixed release")
-            key, content_digest, publication_digest = self.key_and_digests(source)
-            head = json.dumps({
-                "Metadata": {"sha256": content_digest, "publication-sha256": publication_digest},
-                "CacheControl": self.CACHE,
-                "ContentType": "text/javascript",
-            })
-            with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)) as request:
-                created = deploy.publish_immutable_file(
-                    "http://garage-s3.storage.svc:3900", "dist-bucket",
-                    key, source, self.CACHE, {},
-                )
-            self.assertFalse(created)
-            self.assertEqual(1, request.call_count)
-
-    def test_changed_immutable_key_is_refused(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            source = Path(tmp) / "release.js"
-            source.write_text("changed release")
-            key, _, _ = self.key_and_digests(source)
-            head = json.dumps({"Metadata": {"sha256": "different"}, "CacheControl": self.CACHE})
-            with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)), \
-                    self.assertRaisesRegex(RuntimeError, "immutable object differs"):
-                deploy.publish_immutable_file(
-                    "http://garage-s3.storage.svc:3900", "dist-bucket",
-                    key, source, self.CACHE, {},
-                )
-
-    def test_immutable_key_without_publication_digest_is_refused_before_s3(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            source = Path(tmp) / "release.js"
-            source.write_text("fixed release")
-            with patch.object(deploy, "_aws_capture") as request, \
-                    self.assertRaisesRegex(RuntimeError, "must contain its one publication SHA-256"):
-                deploy.publish_immutable_file(
-                    "http://garage-s3.storage.svc:3900", "dist-bucket",
-                    "dist/releases/release.js", source, self.CACHE, {},
-                )
-            request.assert_not_called()
-
-
 if __name__ == "__main__":
     unittest.main()