Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Evelyn Chen
|
ec2d32c958 |
+42
-48
@@ -78,51 +78,27 @@ def _immutable_head(endpoint, bucket, key, aws_env):
|
||||
raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
|
||||
|
||||
|
||||
def _list_object_keys(endpoint, bucket, aws_env):
|
||||
"""List a bucket through the AWS CLI paginator in one client process."""
|
||||
def _existing_immutable_keys(endpoint, bucket, aws_env):
|
||||
"""Enumerate publisher-owned immutable keys so route moves cannot delete them."""
|
||||
args = ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
|
||||
"--bucket", bucket, "--query", "Contents[].Key", "--output", "json"]
|
||||
result = _aws_capture(args, aws_env)
|
||||
if result.returncode != 0:
|
||||
error = f"{result.stdout}\n{result.stderr}".strip()
|
||||
raise RuntimeError(f"list-objects-v2 failed for s3://{bucket}: {error}")
|
||||
return sorted(json.loads(result.stdout) or [])
|
||||
|
||||
|
||||
def _has_publication_address(key):
|
||||
return bool(re.search(r"(?<![0-9a-f])[0-9a-f]{64}(?![0-9a-f])", key.lower()))
|
||||
|
||||
|
||||
def delete_stale_mutable(endpoint, bucket, object_prefix, html_dir, artifact, aws_env):
|
||||
"""Delete stale keys only from the currently served mutable partition."""
|
||||
prefix = f"{object_prefix}/" if object_prefix else ""
|
||||
expected = {
|
||||
f"{prefix}{path.relative_to(html_dir).as_posix()}"
|
||||
for path in html_dir.rglob("*") if path.is_file()
|
||||
keys = json.loads(result.stdout) or []
|
||||
immutable = []
|
||||
for key in keys:
|
||||
info = _immutable_head(endpoint, bucket, key, aws_env)
|
||||
if info is None:
|
||||
continue
|
||||
directives = {
|
||||
part.strip().lower().split("=", 1)[0]
|
||||
for part in (info.get("CacheControl") or "").split(",")
|
||||
}
|
||||
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
|
||||
stale = []
|
||||
for key in _list_object_keys(endpoint, bucket, aws_env):
|
||||
if prefix and not key.startswith(prefix):
|
||||
continue
|
||||
relative = key[len(prefix):]
|
||||
if key in expected or _has_publication_address(key):
|
||||
continue
|
||||
if any(relative == path or relative.startswith(f"{path.rstrip('/')}/")
|
||||
for path in immutable_paths):
|
||||
continue
|
||||
if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
|
||||
continue
|
||||
stale.append(key)
|
||||
for offset in range(0, len(stale), 1000):
|
||||
batch = stale[offset:offset + 1000]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
request = Path(tmp) / "delete.json"
|
||||
request.write_text(json.dumps({
|
||||
"Objects": [{"Key": key} for key in batch], "Quiet": True,
|
||||
}))
|
||||
run(["aws", "--endpoint-url", endpoint, "s3api", "delete-objects",
|
||||
"--bucket", bucket, "--delete", f"file://{request}"], env=aws_env)
|
||||
if "immutable" in directives:
|
||||
immutable.append(key)
|
||||
return sorted(immutable)
|
||||
|
||||
|
||||
def _immutable_digests(source, cache_control, content_type):
|
||||
@@ -215,9 +191,11 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
|
||||
})
|
||||
bucket = artifact["bucket"]
|
||||
object_prefix = route["path"].strip("/")
|
||||
destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}"
|
||||
bucket_destination = f"s3://{bucket}/"
|
||||
destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}"
|
||||
default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
|
||||
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
|
||||
existing_immutable = _existing_immutable_keys(endpoint, bucket, aws_env)
|
||||
# `excludes` are patterns (site.yaml `excludes:` list) that should never
|
||||
# be uploaded *and* should never be deleted from the bucket — escape hatch
|
||||
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
|
||||
@@ -225,23 +203,39 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None):
|
||||
if artifact["excludes"]:
|
||||
print(f"Excluding patterns: {artifact['excludes']}")
|
||||
print(f"Syncing artifact {artifact['name']} → {destination} via {endpoint}")
|
||||
# Sync handles new and changed mutable files. Stale mutable keys are then
|
||||
# deleted explicitly only inside the current route prefix; old route
|
||||
# prefixes become unreachable with their old Ingress and no immutable key
|
||||
# can match the deletion set. Partitioned
|
||||
# `sync --delete` handles new/changed/orphaned files. Partitioned
|
||||
# `cp --recursive` calls then re-upload each file once to refresh metadata
|
||||
# (cache-control, content-type) on objects sync skipped as unchanged.
|
||||
# A no-op deploy therefore transfers the artifact bytes once.
|
||||
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
|
||||
# so a fresh upload always carries the right MIME type.
|
||||
stage = None
|
||||
sync_source = html_dir
|
||||
sync_excludes = [*exclude_args,
|
||||
*(arg for key in existing_immutable for arg in ("--exclude", key)),
|
||||
*(arg for path in immutable_paths for arg in ("--exclude", f"{path}/*"))]
|
||||
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
|
||||
"--only-show-errors", "--cache-control", default_cache,
|
||||
if object_prefix:
|
||||
stage = tempfile.TemporaryDirectory()
|
||||
sync_source = Path(stage.name)
|
||||
staged_artifact = sync_source / object_prefix
|
||||
staged_artifact.parent.mkdir(parents=True, exist_ok=True)
|
||||
staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True)
|
||||
sync_excludes = [
|
||||
*(arg for pattern in artifact["excludes"]
|
||||
for arg in ("--exclude", f"{object_prefix}/{pattern}")),
|
||||
*(arg for key in existing_immutable for arg in ("--exclude", key)),
|
||||
*(arg for path in immutable_paths
|
||||
for arg in ("--exclude", f"{object_prefix}/{path}/*")),
|
||||
]
|
||||
try:
|
||||
# Sync the complete bucket authority so moving a route prefix also
|
||||
# deletes objects under its old prefix instead of leaving them public.
|
||||
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{sync_source}/", bucket_destination,
|
||||
"--delete", "--only-show-errors", "--cache-control", default_cache,
|
||||
*sync_excludes], env=aws_env)
|
||||
delete_stale_mutable(
|
||||
endpoint, bucket, object_prefix, html_dir, artifact, aws_env,
|
||||
)
|
||||
finally:
|
||||
if stage:
|
||||
stage.cleanup()
|
||||
print("Re-stamping metadata on all objects...")
|
||||
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
|
||||
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
|
||||
|
||||
+6
-27
@@ -389,7 +389,9 @@ class PublishingTests(unittest.TestCase):
|
||||
with patch.dict(os.environ, {
|
||||
"DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret
|
||||
}, clear=False), patch.object(deploy, "run", side_effect=capture), \
|
||||
patch.object(deploy, "_list_object_keys", return_value=[]), \
|
||||
patch.object(deploy, "_existing_immutable_keys", return_value=[
|
||||
"old-dist/releases/preserved.js"
|
||||
]), \
|
||||
patch.object(deploy, "publish_immutable_rule") as immutable_publish, \
|
||||
redirect_stdout(output):
|
||||
deploy.s3_sync(artifact, route, root)
|
||||
@@ -399,9 +401,9 @@ class PublishingTests(unittest.TestCase):
|
||||
self.assertTrue(all(call_env["AWS_ACCESS_KEY_ID"] == "dist-key" for _, call_env in commands))
|
||||
self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands))
|
||||
rendered = [" ".join(command) for command, _ in commands]
|
||||
self.assertIn("s3://baseline-dist/dist/", rendered[0])
|
||||
self.assertIn("releases/*", rendered[0])
|
||||
self.assertNotIn("--delete", rendered[0])
|
||||
self.assertIn("s3://baseline-dist/", rendered[0])
|
||||
self.assertIn("dist/releases/*", rendered[0])
|
||||
self.assertIn("old-dist/releases/preserved.js", rendered[0])
|
||||
self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:]))
|
||||
self.assertTrue(any("channels/" in command and
|
||||
"public, max-age=0, must-revalidate" in command
|
||||
@@ -412,29 +414,6 @@ class PublishingTests(unittest.TestCase):
|
||||
immutable_publish.call_args.args[2]["cache_control"],
|
||||
)
|
||||
|
||||
def test_stale_cleanup_cannot_target_old_prefix_or_content_address(self):
|
||||
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
|
||||
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
|
||||
digest_key = "dist/old-" + ("a" * 64) + ".js"
|
||||
keys = ["old-dist/channels/stable.json", digest_key,
|
||||
"dist/releases/legacy.js", "dist/channels/stale.json"]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
html = Path(tmp)
|
||||
(html / "channels").mkdir()
|
||||
(html / "channels" / "current.json").write_text("current")
|
||||
commands, requests = [], []
|
||||
def capture(command, **_kwargs):
|
||||
commands.append(command)
|
||||
requests.append(json.loads(Path(command[-1].removeprefix("file://")).read_text()))
|
||||
with patch.object(deploy, "_list_object_keys", return_value=keys), \
|
||||
patch.object(deploy, "run", side_effect=capture):
|
||||
deploy.delete_stale_mutable(
|
||||
"http://garage-s3.storage.svc:3900", "baseline-dist", "dist",
|
||||
html, artifact, {},
|
||||
)
|
||||
self.assertEqual(1, len(commands))
|
||||
self.assertEqual([{"Key": "dist/channels/stale.json"}], requests[0]["Objects"])
|
||||
|
||||
def test_absent_artifact_is_detected_before_publish(self):
|
||||
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
|
||||
with tempfile.TemporaryDirectory() as tmp, redirect_stderr(io.StringIO()), \
|
||||
|
||||
Reference in New Issue
Block a user