feat(site-publish): add split-surface publishing

Authored-By: OpenAI (GPT-5) <noreply@openai.com>

scripts/deploy.py

@@ -8,6 +8,7 @@ import os
 import re
 import shutil
 import subprocess
+import tempfile
 from pathlib import Path
 from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
@@ -77,6 +78,53 @@ def _immutable_head(endpoint, bucket, key, aws_env):
     raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
 
 
+def _list_object_keys(endpoint, bucket, aws_env):
+    """List a bucket through the AWS CLI paginator in one client process."""
+    args = ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
+            "--bucket", bucket, "--query", "Contents[].Key", "--output", "json"]
+    result = _aws_capture(args, aws_env)
+    if result.returncode != 0:
+        error = f"{result.stdout}\n{result.stderr}".strip()
+        raise RuntimeError(f"list-objects-v2 failed for s3://{bucket}: {error}")
+    return sorted(json.loads(result.stdout) or [])
+
+
+def _has_publication_address(key):
+    return bool(re.search(r"(?<![0-9a-f])[0-9a-f]{64}(?![0-9a-f])", key.lower()))
+
+
+def delete_stale_mutable(endpoint, bucket, object_prefix, html_dir, artifact, aws_env):
+    """Delete stale keys only from the currently served mutable partition."""
+    prefix = f"{object_prefix}/" if object_prefix else ""
+    expected = {
+        f"{prefix}{path.relative_to(html_dir).as_posix()}"
+        for path in html_dir.rglob("*") if path.is_file()
+    }
+    immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
+    stale = []
+    for key in _list_object_keys(endpoint, bucket, aws_env):
+        if prefix and not key.startswith(prefix):
+            continue
+        relative = key[len(prefix):]
+        if key in expected or _has_publication_address(key):
+            continue
+        if any(relative == path or relative.startswith(f"{path.rstrip('/')}/")
+               for path in immutable_paths):
+            continue
+        if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
+            continue
+        stale.append(key)
+    for offset in range(0, len(stale), 1000):
+        batch = stale[offset:offset + 1000]
+        with tempfile.TemporaryDirectory() as tmp:
+            request = Path(tmp) / "delete.json"
+            request.write_text(json.dumps({
+                "Objects": [{"Key": key} for key in batch], "Quiet": True,
+            }))
+            run(["aws", "--endpoint-url", endpoint, "s3api", "delete-objects",
+                 "--bucket", bucket, "--delete", f"file://{request}"], env=aws_env)
+
+
 def _immutable_digests(source, cache_control, content_type):
     with source.open("rb") as stream:
         content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
@@ -177,10 +225,10 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     if artifact["excludes"]:
         print(f"Excluding patterns: {artifact['excludes']}")
     print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
-    # Sync and deletion are scoped to the current route prefix. A route move
-    # leaves its old bucket partition intact but unreachable after the old
-    # Ingress disappears, while stale mutable keys on the serving prefix are
-    # deleted. Immutable subtrees are structurally excluded. Partitioned
+    # Sync handles new and changed mutable files. Stale mutable keys are then
+    # deleted explicitly only inside the current route prefix; old route
+    # prefixes become unreachable with their old Ingress and no immutable key
+    # can match the deletion set. Partitioned
     # `cp --recursive` calls then re-upload each file once to refresh metadata
     # (cache-control, content-type) on objects sync skipped as unchanged.
     # A no-op deploy therefore transfers the artifact bytes once.
@@ -189,8 +237,11 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
     sync_excludes = [*exclude_args,
                      *(arg for path in immutable_paths for arg in ("--exclude", f"{path}/*"))]
     run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
-         "--delete", "--only-show-errors", "--cache-control", default_cache,
+         "--only-show-errors", "--cache-control", default_cache,
          *sync_excludes], env=aws_env)
+    delete_stale_mutable(
+        endpoint, bucket, object_prefix, html_dir, artifact, aws_env,
+    )
     print("Re-stamping metadata on all objects...")
     specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
     default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]

tests/test_contract.py

@@ -389,6 +389,7 @@ class PublishingTests(unittest.TestCase):
             with patch.dict(os.environ, {
                 "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret
             }, clear=False), patch.object(deploy, "run", side_effect=capture), \
+                    patch.object(deploy, "_list_object_keys", return_value=[]), \
                     patch.object(deploy, "publish_immutable_rule") as immutable_publish, \
                     redirect_stdout(output):
                 deploy.s3_sync(artifact, route, root)
@@ -400,7 +401,7 @@ class PublishingTests(unittest.TestCase):
             rendered = [" ".join(command) for command, _ in commands]
             self.assertIn("s3://baseline-dist/dist/", rendered[0])
             self.assertIn("releases/*", rendered[0])
-            self.assertIn("--delete", rendered[0])
+            self.assertNotIn("--delete", rendered[0])
             self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:]))
             self.assertTrue(any("channels/" in command and
                                 "public, max-age=0, must-revalidate" in command
@@ -411,6 +412,28 @@ class PublishingTests(unittest.TestCase):
                 immutable_publish.call_args.args[2]["cache_control"],
             )
 
+    def test_stale_cleanup_cannot_target_old_prefix_or_content_address(self):
+        cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
+        artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
+        digest_key = "dist/old-" + ("a" * 64) + ".js"
+        keys = ["old-dist/channels/stable.json", digest_key,
+                "dist/releases/legacy.js", "dist/channels/stale.json"]
+        with tempfile.TemporaryDirectory() as tmp:
+            html = Path(tmp)
+            (html / "channels").mkdir()
+            (html / "channels" / "current.json").write_text("current")
+            commands, requests = [], []
+            def capture(command, **_kwargs):
+                commands.append(command)
+                requests.append(json.loads(Path(command[-1].removeprefix("file://")).read_text()))
+            with patch.object(deploy, "_list_object_keys", return_value=keys), \
+                    patch.object(deploy, "run", side_effect=capture):
+                deploy.delete_stale_mutable(
+                    "http://garage-s3.storage.svc:3900", "baseline-dist", "dist",
+                    html, artifact, {},
+                )
+            self.assertEqual(1, len(commands))
+            self.assertEqual([{"Key": "dist/channels/stale.json"}], requests[0]["Objects"])
 
     def test_absent_artifact_is_detected_before_publish(self):
         cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")