Author SHA1 Message Date
Evelyn Chen ede4de910a feat(site-publish): add split-surface publishing
Test / contract (pull_request) Successful in 7s
Authored-By: OpenAI (GPT-5) <noreply@openai.com>
2026-08-29 21:51:41 +00:00
4 changed files with 99 additions and 397 deletions
+2 -14
View File
@@ -128,23 +128,11 @@ cache policy, content type, and bytes. That content address makes concurrent
writes identical even though Garage v2.2.0 has no conditional destination
write. An identical retry converges; a changed object, missing digest metadata,
wrong address, or nested policy under that immutable prefix fails publication.
Every immutable target across every artifact is validated and published before
any route's mutable objects change.
Mutable default and override partitions receive their final cache policy before
the matching prefix-scoped stale deletion, so publication never exposes a
provisional cache policy or a pointer to a missing immutable target.
An append-only `site-publish-history.json` beside each generated site retains every bucket's access
class and absolute immutable prefixes, including removed routes and rules. When a route move places
a retired prefix inside the new sync scope, that subtree is excluded; a current-file collision
fails publication. Protected access remains sticky across artifact renames and legacy mode, so
publishing the same artifact publicly requires a new bucket. Decommissioning removes the live
application and manifests while retaining this history because its Garage bucket is not purged.
Artifact input directories must be pairwise disjoint after filesystem
resolution. Publication stops before build or upload if one contains another or
escapes the repository. Symlinked roots, components, and descendants are also rejected, preventing
protected input from entering a public artifact through dereference. Split
storage endpoints are pinned to Garage, and each website
escapes the repository, preventing protected input from entering a public
artifact. Split storage endpoints are pinned to Garage, and each website
authority is derived from its bucket; a site cannot expose an arbitrary backend.
Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net`
+74 -183
View File
@@ -8,7 +8,8 @@ import os
import re
import shutil
import subprocess
from pathlib import Path, PurePosixPath
import tempfile
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
@@ -28,7 +29,6 @@ from utils import (
GARAGE_ADMIN_ENDPOINT = os.environ.get(
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
)
HISTORY_FILE = "site-publish-history.json"
def validate_artifact_output(site_dir, artifact):
@@ -78,6 +78,53 @@ def _immutable_head(endpoint, bucket, key, aws_env):
raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
def _list_object_keys(endpoint, bucket, aws_env):
"""List a bucket through the AWS CLI paginator in one client process."""
args = ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
"--bucket", bucket, "--query", "Contents[].Key", "--output", "json"]
result = _aws_capture(args, aws_env)
if result.returncode != 0:
error = f"{result.stdout}\n{result.stderr}".strip()
raise RuntimeError(f"list-objects-v2 failed for s3://{bucket}: {error}")
return sorted(json.loads(result.stdout) or [])
def _has_publication_address(key):
return bool(re.search(r"(?<![0-9a-f])[0-9a-f]{64}(?![0-9a-f])", key.lower()))
def delete_stale_mutable(endpoint, bucket, object_prefix, html_dir, artifact, aws_env):
"""Delete stale keys only from the currently served mutable partition."""
prefix = f"{object_prefix}/" if object_prefix else ""
expected = {
f"{prefix}{path.relative_to(html_dir).as_posix()}"
for path in html_dir.rglob("*") if path.is_file()
}
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
stale = []
for key in _list_object_keys(endpoint, bucket, aws_env):
if prefix and not key.startswith(prefix):
continue
relative = key[len(prefix):]
if key in expected or _has_publication_address(key):
continue
if any(relative == path or relative.startswith(f"{path.rstrip('/')}/")
for path in immutable_paths):
continue
if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
continue
stale.append(key)
for offset in range(0, len(stale), 1000):
batch = stale[offset:offset + 1000]
with tempfile.TemporaryDirectory() as tmp:
request = Path(tmp) / "delete.json"
request.write_text(json.dumps({
"Objects": [{"Key": key} for key in batch], "Quiet": True,
}))
run(["aws", "--endpoint-url", endpoint, "s3api", "delete-objects",
"--bucket", bucket, "--delete", f"file://{request}"], env=aws_env)
def _immutable_digests(source, cache_control, content_type):
with source.open("rb") as stream:
content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
@@ -150,8 +197,9 @@ def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
)
def publication_aws_env(artifact, credential_env_names=None):
"""Build the route-scoped AWS environment without leaking other credentials."""
def s3_sync(artifact, route, site_dir, credential_env_names=None):
endpoint = artifact["s3_endpoint"]
html_dir = site_dir / artifact["build_dir"]
access_key = env(artifact["credentials"]["access_key_env"])
secret_key = env(artifact["credentials"]["secret_key_env"])
aws_env = os.environ.copy()
@@ -165,55 +213,6 @@ def publication_aws_env(artifact, credential_env_names=None):
"AWS_SECRET_ACCESS_KEY": secret_key,
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
})
return aws_env
def publish_route_immutables(artifact, route, site_dir, credential_env_names=None):
"""Publish one route's immutable partitions during the global preflight."""
html_dir = site_dir / artifact["build_dir"]
aws_env = publication_aws_env(artifact, credential_env_names)
for rule in artifact["cache_rules"]:
if _is_immutable(rule):
publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
def retired_immutable_filters(artifact, route, html_dir, previous_contract):
"""Protect every historical immutable prefix inside the current sync scope."""
if not previous_contract:
return []
current_prefix = route["path"].strip("/")
filters = []
current_immutable = set(immutable_prefixes(artifact, route))
for immutable_prefix in previous_contract["immutable_prefixes"]:
if current_prefix:
marker = f"{current_prefix}/"
if not immutable_prefix.startswith(marker):
continue
relative_path = immutable_prefix[len(marker):]
else:
relative_path = immutable_prefix
collision = html_dir / relative_path
if (immutable_prefix not in current_immutable and collision.exists()
and any(path.is_file() for path in collision.rglob("*"))):
raise RuntimeError(
f"current artifact collides with retired immutable partition: {relative_path}"
)
filters.extend(("--exclude", f"{relative_path}/*"))
return filters
def immutable_prefixes(artifact, route):
route_prefix = route["path"].strip("/")
return sorted({
"/".join(part for part in (route_prefix, rule["path"]) if part)
for rule in artifact["cache_rules"] if _is_immutable(rule)
})
def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contract=None):
endpoint = artifact["s3_endpoint"]
html_dir = site_dir / artifact["build_dir"]
aws_env = publication_aws_env(artifact, credential_env_names)
bucket = artifact["bucket"]
object_prefix = route["path"].strip("/")
destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}"
@@ -226,29 +225,34 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contr
if artifact["excludes"]:
print(f"Excluding patterns: {artifact['excludes']}")
print(f"Syncing artifact {artifact['name']}{destination} via {endpoint}")
# Upload with the final cache policy before cleanup. Sync and deletion are
# scoped to the same current route prefix and cache partition. A route move
# leaves its old bucket partition intact but unreachable after the old
# Ingress disappears, while stale mutable keys on the serving prefix are
# deleted. Immutable subtrees are structurally excluded. `cp --recursive`
# refreshes metadata atomically per object before `sync --delete` removes
# stale keys without ever exposing new bytes under a provisional policy.
# Sync handles new and changed mutable files. Stale mutable keys are then
# deleted explicitly only inside the current route prefix; old route
# prefixes become unreachable with their old Ingress and no immutable key
# can match the deletion set. Partitioned
# `cp --recursive` calls then re-upload each file once to refresh metadata
# (cache-control, content-type) on objects sync skipped as unchanged.
# A no-op deploy therefore transfers the artifact bytes once.
# AWS CLI guesses Content-Type from file extension on local→S3 uploads,
# so a fresh upload always carries the right MIME type.
sync_excludes = [*exclude_args,
*(arg for path in immutable_paths for arg in ("--exclude", f"{path}/*"))]
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--only-show-errors", "--cache-control", default_cache,
*sync_excludes], env=aws_env)
delete_stale_mutable(
endpoint, bucket, object_prefix, html_dir, artifact, aws_env,
)
print("Re-stamping metadata on all objects...")
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
retired_filters = retired_immutable_filters(artifact, route, html_dir, previous_contract)
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *retired_filters, *exclude_args], env=aws_env)
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--delete", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *retired_filters, *exclude_args], env=aws_env)
*default_filters, *exclude_args], env=aws_env)
for rule in artifact["cache_rules"]:
if not rule["path"]:
continue
if _is_immutable(rule):
publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
continue
include = f"{rule['path'].rstrip('/')}/*"
child_filters = [arg for path in specific_paths
@@ -259,9 +263,6 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contr
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--delete", "--only-show-errors", "--cache-control", rule["cache_control"],
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
def garage_admin(method, path, token, body=None):
@@ -332,99 +333,6 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
render_templates(action_dir, template_vars, app_dir, manifests_dir)
def previous_route_contracts(app_dir):
"""Read the append-only bucket history kept beside generated manifests."""
path = app_dir / HISTORY_FILE
if not path.exists():
return {}
try:
document = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError(f"invalid site-publish route history in {path}") from exc
if (not isinstance(document, dict) or set(document) != {"schemaVersion", "buckets"}
or document["schemaVersion"] != 1 or not isinstance(document["buckets"], dict)):
raise RuntimeError(f"invalid site-publish route history in {path}")
for bucket, contract in document["buckets"].items():
if (not isinstance(bucket, str) or not isinstance(contract, dict)
or set(contract) != {"access", "artifact", "immutablePrefixes", "routePath"}
or contract["access"] not in {"legacy", "protected", "public"}
or not isinstance(contract["artifact"], str)
or not isinstance(contract["routePath"], str)
or not contract["routePath"].startswith("/")
or not isinstance(contract["immutablePrefixes"], list)
or any(not _valid_immutable_prefix(value)
for value in contract["immutablePrefixes"])
or len(contract["immutablePrefixes"]) != len(set(contract["immutablePrefixes"]))):
raise RuntimeError(f"invalid site-publish route history in {path}")
return {
bucket: {
"access": contract["access"],
"artifact": contract["artifact"],
"immutable_prefixes": sorted(contract["immutablePrefixes"]),
"path": contract["routePath"],
}
for bucket, contract in document["buckets"].items()
}
def next_route_contracts(cfg, previous_contracts):
"""Carry protected access and immutable prefixes forward for every known bucket."""
contracts = json.loads(json.dumps(previous_contracts))
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
for route in cfg["routes"]:
artifact = artifacts[route["artifact"]]
bucket = artifact["bucket"]
previous = previous_contracts.get(bucket)
access = "protected" if (
route["access"] == "protected" or previous and previous["access"] == "protected"
) else route["access"]
contracts[bucket] = {
"access": access,
"artifact": route["artifact"],
"immutable_prefixes": sorted(set(
(previous or {}).get("immutable_prefixes", []) + immutable_prefixes(artifact, route)
)),
"path": route["path"],
}
return contracts
def write_route_contracts(app_dir, contracts):
app_dir.mkdir(parents=True, exist_ok=True)
document = {
"schemaVersion": 1,
"buckets": {
bucket: {
"access": contract["access"],
"artifact": contract["artifact"],
"immutablePrefixes": contract["immutable_prefixes"],
"routePath": contract["path"],
}
for bucket, contract in sorted(contracts.items())
},
}
(app_dir / HISTORY_FILE).write_text(f"{json.dumps(document, indent=2, sort_keys=True)}\n")
def _valid_immutable_prefix(value):
return (isinstance(value, str) and value and not value.startswith("/")
and not value.endswith("/") and ".." not in PurePosixPath(value).parts)
def validate_route_migrations(cfg, previous_contracts):
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
for route in cfg["routes"]:
artifact = artifacts[route["artifact"]]
previous = previous_contracts.get(artifact["bucket"])
if (previous and previous["access"] == "protected"
and route["access"] in {"public", "legacy"}
):
raise RuntimeError(
f"artifact {route['artifact']} cannot become public while reusing protected "
f"bucket {artifact['bucket']}"
)
def deploy_static(site_name, site_dir, action_dir, token, cfg):
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
credential_env_names = {
@@ -433,27 +341,15 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
validate_publication_environment(cfg)
for artifact in cfg["artifacts"]:
validate_artifact_output(site_dir, artifact)
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
manifests_dir = app_dir / "manifests"
previous_contracts = previous_route_contracts(app_dir)
validate_route_migrations(cfg, previous_contracts)
# Complete immutable work across the whole publication before any route's
# mutable pointers can change. Partial immutable success is safe; mixing a
# new route with an old route after a later immutable failure is not.
for route in cfg["routes"]:
publish_route_immutables(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
)
for route in cfg["routes"]:
s3_sync(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]),
)
s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
if cfg["compatibility"]:
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
write_route_contracts(app_dir, next_route_contracts(cfg, previous_contracts))
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
manifests_dir = app_dir / "manifests"
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
commit_and_push(apps_dir, f"Deploy {site_name}", token)
@@ -466,12 +362,7 @@ def decommission(site_name, token, buckets=None):
if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove")
return
history_path = site_path / HISTORY_FILE
history = history_path.read_bytes() if history_path.exists() else None
shutil.rmtree(site_path)
if history is not None:
site_path.mkdir(parents=True)
(site_path / HISTORY_FILE).write_bytes(history)
commit_and_push(apps_dir, f"Decommission {site_name}", token)
for bucket in buckets or [site_name]:
print(f"Bucket {bucket} and its objects are NOT purged automatically.")
+1 -18
View File
@@ -427,31 +427,14 @@ def normalize_site_config(raw, site_name):
def validate_artifact_inputs(site_dir, cfg):
"""Reject source containment before a public or protected build starts."""
if cfg["compatibility"]:
return
root = Path(site_dir).resolve()
sources = []
for artifact in cfg["artifacts"]:
declared = root
for component in Path(artifact["content_dir"]).parts:
declared /= component
if declared.is_symlink():
raise ConfigError(
f"artifact {artifact['name']} content_dir contains symlink component: "
f"{declared.relative_to(root)}"
)
source = declared.resolve()
source = (root / artifact["content_dir"]).resolve()
if source != root and root not in source.parents:
raise ConfigError(
f"artifact {artifact['name']} content_dir resolves outside the repository"
)
if source.exists():
symlink = next((path for path in source.rglob("*") if path.is_symlink()), None)
if symlink is not None:
raise ConfigError(
f"artifact {artifact['name']} build input contains symlink: "
f"{symlink.relative_to(root)}"
)
sources.append((artifact["name"], source))
for index, (name, source) in enumerate(sources):
for other_name, other_source in sources[index + 1:]:
+21 -181
View File
@@ -219,27 +219,6 @@ class ConfigContractTests(unittest.TestCase):
with self.assertRaisesRegex(ConfigError, "build inputs overlap after resolution"):
validate_artifact_inputs(root, cfg)
def test_descendant_symlink_cannot_cross_artifact_boundary(self):
cfg = normalize_site_config(self.raw, "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "dist").mkdir()
(root / "portal" / "build").mkdir(parents=True)
(root / "portal" / "build" / "private.txt").write_text("private")
(root / "dist" / "portal-link").symlink_to(root / "portal" / "build")
with self.assertRaisesRegex(ConfigError, "build input contains symlink"):
validate_artifact_inputs(root, cfg)
def test_artifact_root_symlink_is_rejected_before_resolution(self):
cfg = normalize_site_config(self.raw, "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "dist-real").mkdir()
(root / "dist").symlink_to(root / "dist-real")
(root / "portal" / "build").mkdir(parents=True)
with self.assertRaisesRegex(ConfigError, "content_dir contains symlink component"):
validate_artifact_inputs(root, cfg)
class GenerationTests(unittest.TestCase):
def render(self, raw):
@@ -273,18 +252,6 @@ class GenerationTests(unittest.TestCase):
self.assertNotIn("passhostheader", files["manifests/ingress-portal.yaml"])
self.assertIn("baseline-dist.web.sjc001.fritzlab.net", files["manifests/service-distributions.yaml"])
def test_history_keeps_yaml_ambiguous_artifact_names_as_strings(self):
raw = fixture("split-site.yaml")
raw["artifacts"][0]["name"] = "yes"
raw["routes"][0]["artifact"] = "yes"
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
contracts = deploy.next_route_contracts(cfg, {})
with tempfile.TemporaryDirectory() as tmp:
app_dir = Path(tmp)
deploy.write_route_contracts(app_dir, contracts)
restored = deploy.previous_route_contracts(app_dir)
self.assertEqual("yes", restored["baseline-portal"]["artifact"])
def test_generation_is_deterministic_when_input_lists_are_reversed(self):
raw = fixture("split-site.yaml")
first_tmp, _, first = self.render(raw)
@@ -308,14 +275,12 @@ class GenerationTests(unittest.TestCase):
self.assertFalse(stale.exists())
def test_legacy_names_and_garage_s3_target_are_preserved(self):
tmp, app_dir, files = self.render(fixture("legacy-site.yaml"))
tmp, _, files = self.render(fixture("legacy-site.yaml"))
self.addCleanup(tmp.cleanup)
self.assertIn("manifests/service.yaml", files)
self.assertIn("manifests/ingress.yaml", files)
self.assertIn("garage-s3.storage.svc.k8s.sjc001.fritzlab.net", files["manifests/service.yaml"])
self.assertNotIn("passhostheader", files["manifests/ingress.yaml"])
self.assertNotIn("site-publish.fritzlab.net", files["manifests/ingress.yaml"])
self.assertEqual({}, deploy.previous_route_contracts(app_dir))
class BuildTests(unittest.TestCase):
@@ -414,27 +379,22 @@ class PublishingTests(unittest.TestCase):
(html / "channels").mkdir()
(html / "releases" / "1.0.js").write_text("release")
(html / "channels" / "stable.json").write_text("channel")
commands, events = [], []
commands = []
def capture(command, **kwargs):
commands.append((command, kwargs["env"]))
events.append("mutable")
def publish_immutable(*_args):
events.append("immutable")
secret = "secret-must-not-appear"
output = io.StringIO()
with patch.dict(os.environ, {
"DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret
}, clear=False), patch.object(deploy, "run", side_effect=capture), \
patch.object(deploy, "publish_immutable_rule", side_effect=publish_immutable) as immutable_publish, \
patch.object(deploy, "_list_object_keys", return_value=[]), \
patch.object(deploy, "publish_immutable_rule") as immutable_publish, \
redirect_stdout(output):
deploy.publish_route_immutables(artifact, route, root)
deploy.s3_sync(artifact, route, root)
self.assertTrue(all(secret not in " ".join(command) for command, _ in commands))
self.assertEqual("immutable", events[0])
self.assertNotIn(secret, output.getvalue())
self.assertTrue(all(call_env["AWS_ACCESS_KEY_ID"] == "dist-key" for _, call_env in commands))
self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands))
@@ -442,7 +402,6 @@ class PublishingTests(unittest.TestCase):
self.assertIn("s3://baseline-dist/dist/", rendered[0])
self.assertIn("releases/*", rendered[0])
self.assertNotIn("--delete", rendered[0])
self.assertIn("--delete", rendered[1])
self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:]))
self.assertTrue(any("channels/" in command and
"public, max-age=0, must-revalidate" in command
@@ -453,147 +412,28 @@ class PublishingTests(unittest.TestCase):
immutable_publish.call_args.args[2]["cache_control"],
)
def test_root_move_preserves_only_actual_retired_immutable_prefix(self):
def test_stale_cleanup_cannot_target_old_prefix_or_content_address(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
route = {**next(item for item in cfg["routes"] if item["artifact"] == "distributions"),
"path": "/"}
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
html = root / artifact["build_dir"]
(html / "releases").mkdir(parents=True)
(html / "channels").mkdir()
(html / "docs" / "releases").mkdir(parents=True)
(html / "channels" / "stable.json").write_text("channel")
(html / "docs" / "releases" / "index.html").write_text("mutable")
commands = []
with patch.dict(os.environ, {
"DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": "dist-secret"
}, clear=False), patch.object(
deploy, "run", side_effect=lambda command, **_: commands.append(command)
):
deploy.s3_sync(artifact, route, root, previous_contract={
"path": "/foo", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["foo/releases"],
})
rendered = [" ".join(command) for command in commands]
self.assertTrue(all("foo/releases/*" in command for command in rendered[:2]))
self.assertTrue(all("*/releases/*" not in command for command in rendered))
def test_protected_bucket_cannot_become_public_across_deployments(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
previous = {
"baseline-dist": {
"path": "/dist", "access": "protected", "artifact": "old-name",
"immutable_prefixes": ["dist/releases"],
}
}
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(cfg, previous)
renamed = copy.deepcopy(cfg)
artifact = next(item for item in renamed["artifacts"] if item["name"] == "distributions")
artifact["name"] = "downloads"
route = next(item for item in renamed["routes"] if item["artifact"] == "distributions")
route["artifact"] = "downloads"
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(renamed, previous)
previous = {"retired-protected-bucket": previous["baseline-dist"]}
deploy.validate_route_migrations(cfg, previous)
def test_protected_split_bucket_cannot_become_legacy_public(self):
cfg = normalize_site_config(fixture("legacy-site.yaml"), "baseline.fritzlab.net")
previous = {
"baseline.fritzlab.net": {
"path": "/portal", "access": "protected", "artifact": "portal",
"immutable_prefixes": [],
}
}
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(cfg, previous)
def test_removed_immutable_rule_preserves_prior_keys(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
artifact["cache_rules"] = [
rule for rule in artifact["cache_rules"] if rule["path"] != "releases"
]
route = next(item for item in cfg["routes"] if item["artifact"] == "distributions")
digest_key = "dist/old-" + ("a" * 64) + ".js"
keys = ["old-dist/channels/stable.json", digest_key,
"dist/releases/legacy.js", "dist/channels/stale.json"]
with tempfile.TemporaryDirectory() as tmp:
html = Path(tmp)
filters = deploy.retired_immutable_filters(artifact, route, html, {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["dist/releases"],
})
self.assertEqual(["--exclude", "releases/*"], filters)
previous = {
"baseline-dist": {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["dist/releases"],
}
}
next_contracts = deploy.next_route_contracts(cfg, previous)
self.assertEqual(
["dist/releases"], next_contracts["baseline-dist"]["immutable_prefixes"],
(html / "channels").mkdir()
(html / "channels" / "current.json").write_text("current")
commands, requests = [], []
def capture(command, **_kwargs):
commands.append(command)
requests.append(json.loads(Path(command[-1].removeprefix("file://")).read_text()))
with patch.object(deploy, "_list_object_keys", return_value=keys), \
patch.object(deploy, "run", side_effect=capture):
deploy.delete_stale_mutable(
"http://garage-s3.storage.svc:3900", "baseline-dist", "dist",
html, artifact, {},
)
self.assertEqual(
["--exclude", "releases/*"],
deploy.retired_immutable_filters(
artifact, route, html, next_contracts["baseline-dist"],
),
)
(html / "releases").mkdir()
(html / "releases" / "replacement.js").write_text("mutable")
with self.assertRaisesRegex(RuntimeError, "collides with retired immutable"):
deploy.retired_immutable_filters(artifact, route, html, {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["dist/releases"],
})
def test_removed_route_history_remains_append_only(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
previous = {
"retired-bucket": {
"path": "/retired", "access": "protected", "artifact": "retired",
"immutable_prefixes": ["retired/releases"],
}
}
contracts = deploy.next_route_contracts(cfg, previous)
self.assertEqual(previous["retired-bucket"], contracts["retired-bucket"])
def test_decommission_preserves_history_for_an_unpurged_bucket(self):
with tempfile.TemporaryDirectory() as tmp:
apps = Path(tmp)
site = apps / "sjc001/websites/baseline"
site.mkdir(parents=True)
history = b'{"schemaVersion":1,"buckets":{}}\n'
(site / deploy.HISTORY_FILE).write_bytes(history)
(site / "app.yaml").write_text("live\n")
with patch.object(deploy, "clone_apps", return_value=apps), patch.object(
deploy, "commit_and_push"
) as commit:
deploy.decommission("baseline", "token", ["baseline-dist"])
self.assertEqual(history, (site / deploy.HISTORY_FILE).read_bytes())
self.assertFalse((site / "app.yaml").exists())
commit.assert_called_once_with(apps, "Decommission baseline", "token")
def test_later_route_immutable_failure_stops_all_mutable_publication(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
with patch.object(deploy, "validate_publication_environment"), \
patch.object(deploy, "validate_artifact_output"), \
patch.object(deploy, "clone_apps", return_value=root / "apps"), patch.object(
deploy, "publish_route_immutables",
side_effect=[None, RuntimeError("immutable failed")],
) as immutable_publish, patch.object(deploy, "s3_sync") as mutable_sync, \
self.assertRaisesRegex(
RuntimeError, "immutable failed"
):
deploy.deploy_static("baseline", root, root, "token", cfg)
self.assertEqual(2, immutable_publish.call_count)
mutable_sync.assert_not_called()
self.assertEqual(1, len(commands))
self.assertEqual([{"Key": "dist/channels/stale.json"}], requests[0]["Objects"])
def test_absent_artifact_is_detected_before_publish(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")