Author SHA1 Message Date
Evelyn Chen a224109868 fix(site-publish): close split migration boundaries
Test / contract (pull_request) Successful in 6s
Authored-By: OpenAI (GPT-5) <noreply@openai.com>
2026-08-29 23:01:18 +00:00
6 changed files with 26 additions and 580 deletions
+2 -14
View File
@@ -52,7 +52,6 @@ artifacts:
- name: distributions - name: distributions
type: static type: static
content_dir: dist content_dir: dist
cors_origins: ['*']
publish: publish:
bucket: baseline-dist bucket: baseline-dist
credentials: credentials:
@@ -134,16 +133,11 @@ any route's mutable objects change.
Mutable default and override partitions receive their final cache policy before Mutable default and override partitions receive their final cache policy before
the matching prefix-scoped stale deletion, so publication never exposes a the matching prefix-scoped stale deletion, so publication never exposes a
provisional cache policy or a pointer to a missing immutable target. provisional cache policy or a pointer to a missing immutable target.
Generated Ingress annotations and `site-publish-history.yaml` retain every Generated Ingress annotations retain each bucket's prior route. When a move
seen bucket's access, prior route, and cumulative bucket-relative immutable key places that retired prefix inside the new sync scope, only its declared
prefixes, including while an artifact is absent. Removed or renamed rules stay
recorded. When a move places a retired prefix inside the new sync scope, its
immutable subtrees are excluded; a current-file collision fails publication. immutable subtrees are excluded; a current-file collision fails publication.
The bucket-keyed history rejects a protected-to-public transition even when the The bucket-keyed history rejects a protected-to-public transition even when the
artifact is renamed; publishing that artifact publicly requires a new bucket. artifact is renamed; publishing that artifact publicly requires a new bucket.
Legacy single-surface is public for this downgrade check. Removing or renaming
an immutable rule preserves its prior URLs; current mutable content at one of
those paths is rejected instead of replacing it.
Artifact input directories must be pairwise disjoint after filesystem Artifact input directories must be pairwise disjoint after filesystem
resolution. Publication stops before build or upload if one contains another or resolution. Publication stops before build or upload if one contains another or
@@ -152,12 +146,6 @@ protected input from entering a public artifact through dereference. Split
storage endpoints are pinned to Garage, and each website storage endpoints are pinned to Garage, and each website
authority is derived from its bucket; a site cannot expose an arbitrary backend. authority is derived from its bucket; a site cannot expose an arbitrary backend.
`cors_origins` is reconciled as a bucket policy on every split publication. Values are either `*`
or HTTPS origins; browser access is limited to `GET` and `HEAD`. Omitting the field removes stale
CORS from that bucket. Protected artifacts cannot allow wildcard CORS. All immutable objects and
all bucket CORS policies complete before mutable channels change; if any policy write fails, the
policies already attempted are restored to their pre-publication values.
Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net` Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net`
ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route
Ingresses share the hostname's certificate Secret. The access middleware and Ingresses share the hostname's certificate Secret. The access middleware and
+20 -196
View File
@@ -8,7 +8,6 @@ import os
import re import re
import shutil import shutil
import subprocess import subprocess
import tempfile
from pathlib import Path from pathlib import Path
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
@@ -170,98 +169,6 @@ def publication_aws_env(artifact, credential_env_names=None):
return aws_env return aws_env
def configure_cors(bucket, origins, endpoint, aws_env):
"""Reconcile read-only browser access without exposing publication credentials."""
if origins is None:
return
config = None
if origins:
config = {
"CORSRules": [{
"AllowedOrigins": origins,
"AllowedMethods": ["GET", "HEAD"],
"AllowedHeaders": ["*"],
"ExposeHeaders": ["ETag"],
"MaxAgeSeconds": 3600,
}],
}
set_cors_configuration(bucket, config, endpoint, aws_env)
def set_cors_configuration(bucket, config, endpoint, aws_env):
"""Apply an exact bucket CORS configuration, or remove it when absent."""
if config is None:
run([
"aws", "--endpoint-url", endpoint, "s3api", "delete-bucket-cors",
"--bucket", bucket,
], env=aws_env)
return
with tempfile.NamedTemporaryFile("w", suffix=".json", encoding="utf-8") as handle:
json.dump(config, handle)
handle.flush()
run([
"aws", "--endpoint-url", endpoint, "s3api", "put-bucket-cors",
"--bucket", bucket, "--cors-configuration", f"file://{handle.name}",
], env=aws_env)
def get_cors_configuration(bucket, endpoint, aws_env):
"""Read the exact bucket CORS configuration for rollback."""
result = _aws_capture([
"aws", "--endpoint-url", endpoint, "s3api", "get-bucket-cors",
"--bucket", bucket, "--output", "json",
], aws_env)
if result.returncode == 0:
try:
config = json.loads(result.stdout)
except json.JSONDecodeError as error:
raise RuntimeError(f"get-bucket-cors returned invalid JSON for {bucket}") from error
if not isinstance(config, dict) or not isinstance(config.get("CORSRules"), list):
raise RuntimeError(f"get-bucket-cors returned an invalid policy for {bucket}")
return config
error = f"{result.stdout}\n{result.stderr}"
if "NoSuchCORSConfiguration" in error:
return None
raise RuntimeError(f"get-bucket-cors failed for {bucket}: {error.strip()}")
def reconcile_artifact_cors(artifacts, credential_env_names=None):
"""Reconcile all policies, restoring the prior set if any write fails."""
snapshots = []
for artifact in artifacts:
if artifact["cors_origins"] is None:
continue
aws_env = publication_aws_env(artifact, credential_env_names)
snapshots.append((
artifact,
aws_env,
get_cors_configuration(artifact["bucket"], artifact["s3_endpoint"], aws_env),
))
attempted = []
try:
for artifact, aws_env, previous in snapshots:
attempted.append((artifact, aws_env, previous))
configure_cors(
artifact["bucket"], artifact["cors_origins"], artifact["s3_endpoint"], aws_env,
)
except Exception as error:
rollback_errors = []
for artifact, aws_env, previous in reversed(attempted):
try:
set_cors_configuration(
artifact["bucket"], previous, artifact["s3_endpoint"], aws_env,
)
except Exception as rollback_error:
rollback_errors.append(f"{artifact['bucket']}: {rollback_error}")
if rollback_errors:
raise RuntimeError(
f"CORS reconciliation failed ({error}); rollback also failed for "
f"{'; '.join(rollback_errors)}"
) from error
raise
def publish_route_immutables(artifact, route, site_dir, credential_env_names=None): def publish_route_immutables(artifact, route, site_dir, credential_env_names=None):
"""Publish one route's immutable partitions during the global preflight.""" """Publish one route's immutable partitions during the global preflight."""
html_dir = site_dir / artifact["build_dir"] html_dir = site_dir / artifact["build_dir"]
@@ -271,38 +178,22 @@ def publish_route_immutables(artifact, route, site_dir, credential_env_names=Non
publish_immutable_rule(artifact, route, rule, html_dir, aws_env) publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
def immutable_key_prefixes(artifact, route): def retired_immutable_filters(artifact, route, html_dir, previous_path):
"""Return immutable partitions as bucket-relative key prefixes.""" """Protect immutable keys only when an old route falls inside the new scope."""
route_prefix = route["path"].strip("/") if not previous_path or previous_path == route["path"]:
return [
"/".join(part for part in (route_prefix, rule["path"]) if part)
for rule in artifact["cache_rules"] if _is_immutable(rule)
]
def retained_immutable_paths(artifact, route, previous_contract):
"""Carry all bucket history forward so later route moves cannot delete it."""
previous_paths = previous_contract["immutable_paths"] if previous_contract else []
return sorted(set(previous_paths) | set(immutable_key_prefixes(artifact, route)))
def retired_immutable_filters(artifact, route, html_dir, previous_contract):
"""Protect historical immutable keys that fall inside the current sync scope."""
if not previous_contract:
return [] return []
current_prefix = route["path"].strip("/") current_prefix = route["path"].strip("/")
filters = [] previous_prefix = previous_path.strip("/")
current_immutable = set(immutable_key_prefixes(artifact, route))
for immutable_path in previous_contract["immutable_paths"]:
if immutable_path in current_immutable:
continue
if current_prefix: if current_prefix:
marker = f"{current_prefix}/" marker = f"{current_prefix}/"
if not immutable_path.startswith(marker): if not previous_prefix.startswith(marker):
return []
previous_prefix = previous_prefix[len(marker):]
filters = []
for rule in artifact["cache_rules"]:
if not _is_immutable(rule):
continue continue
retired_path = immutable_path[len(marker):] retired_path = "/".join(part for part in (previous_prefix, rule["path"]) if part)
else:
retired_path = immutable_path
collision = html_dir / retired_path collision = html_dir / retired_path
if collision.exists() and any(path.is_file() for path in collision.rglob("*")): if collision.exists() and any(path.is_file() for path in collision.rglob("*")):
raise RuntimeError( raise RuntimeError(
@@ -312,7 +203,7 @@ def retired_immutable_filters(artifact, route, html_dir, previous_contract):
return filters return filters
def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contract=None): def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_path=None):
endpoint = artifact["s3_endpoint"] endpoint = artifact["s3_endpoint"]
html_dir = site_dir / artifact["build_dir"] html_dir = site_dir / artifact["build_dir"]
aws_env = publication_aws_env(artifact, credential_env_names) aws_env = publication_aws_env(artifact, credential_env_names)
@@ -340,7 +231,7 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contr
# so a fresh upload always carries the right MIME type. # so a fresh upload always carries the right MIME type.
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]] specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")] default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
retired_filters = retired_immutable_filters(artifact, route, html_dir, previous_contract) retired_filters = retired_immutable_filters(artifact, route, html_dir, previous_path)
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination, run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", default_cache, "--recursive", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *retired_filters, *exclude_args], env=aws_env) *default_filters, *retired_filters, *exclude_args], env=aws_env)
@@ -412,32 +303,16 @@ def ensure_bucket_aliases(site_name, aliases, admin_token):
raise raise
def render_site_manifests( def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
site_name, action_dir, app_dir, manifests_dir, cfg, previous_contracts=None,
):
"""Always re-render manifests from current site.yaml. Templates own """Always re-render manifests from current site.yaml. Templates own
domain + aliases, so changes propagate without manual edits.""" domain + aliases, so changes propagate without manual edits."""
manifests_dir.mkdir(parents=True, exist_ok=True) manifests_dir.mkdir(parents=True, exist_ok=True)
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
routes = [] routes = []
previous_contracts = previous_contracts or {}
next_contracts = {bucket: dict(contract)
for bucket, contract in previous_contracts.items()}
for route in cfg["routes"]: for route in cfg["routes"]:
artifact = artifact_by_name[route["artifact"]] artifact = artifact_by_name[route["artifact"]]
resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}" resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
previous = previous_contracts.get(artifact["bucket"]) routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
immutable_paths = retained_immutable_paths(artifact, route, previous)
next_contracts[artifact["bucket"]] = {
"path": route["path"],
"access": "public" if route["access"] == "legacy" else route["access"],
"artifact": route["artifact"],
"immutable_paths": immutable_paths,
}
routes.append({
**route, "resource_name": resource_name, "artifact_config": artifact,
"immutable_paths_json": json.dumps(immutable_paths, separators=(",", ":")),
})
template_vars = { template_vars = {
"site": site_name, "site": site_name,
"site_k8s": k8s_name(site_name), "site_k8s": k8s_name(site_name),
@@ -448,44 +323,10 @@ def render_site_manifests(
"routes": routes, "routes": routes,
} }
render_templates(action_dir, template_vars, app_dir, manifests_dir) render_templates(action_dir, template_vars, app_dir, manifests_dir)
if not cfg["compatibility"] or previous_contracts:
(app_dir / "site-publish-history.yaml").write_text(yaml.safe_dump(
{"version": 1, "buckets": next_contracts}, sort_keys=True,
))
def _validate_route_contract(bucket, contract, path):
expected = {"path", "access", "artifact", "immutable_paths"}
if (not isinstance(bucket, str) or not bucket or not isinstance(contract, dict)
or set(contract) != expected
or contract.get("access") not in {"public", "protected"}
or not isinstance(contract.get("path"), str)
or not contract["path"].startswith("/")
or not isinstance(contract.get("artifact"), str) or not contract["artifact"]
or not isinstance(contract.get("immutable_paths"), list)
or any(not isinstance(item, str) or not item or item.startswith("/")
or any(part in {"", ".", ".."} for part in item.split("/"))
for item in contract["immutable_paths"])):
raise RuntimeError(f"invalid site-publish route history in {path}")
return {
"path": contract["path"], "access": contract["access"],
"artifact": contract["artifact"],
"immutable_paths": sorted(set(contract["immutable_paths"])),
}
def previous_route_contracts(app_dir): def previous_route_contracts(app_dir):
"""Read bucket-keyed route history from generated Ingresses.""" """Read bucket-keyed route history from generated Ingresses."""
history_path = app_dir / "site-publish-history.yaml"
if history_path.exists():
document = yaml.safe_load(history_path.read_text())
if (not isinstance(document, dict) or set(document) != {"version", "buckets"}
or document["version"] != 1 or not isinstance(document["buckets"], dict)):
raise RuntimeError(f"invalid site-publish route history in {history_path}")
return {
bucket: _validate_route_contract(bucket, contract, history_path)
for bucket, contract in document["buckets"].items()
}
contracts = {} contracts = {}
manifests = app_dir / "manifests" manifests = app_dir / "manifests"
if not manifests.exists(): if not manifests.exists():
@@ -496,26 +337,16 @@ def previous_route_contracts(app_dir):
artifact = annotations.get("site-publish.fritzlab.net/artifact") artifact = annotations.get("site-publish.fritzlab.net/artifact")
access = annotations.get("site-publish.fritzlab.net/access") access = annotations.get("site-publish.fritzlab.net/access")
bucket = annotations.get("site-publish.fritzlab.net/bucket") bucket = annotations.get("site-publish.fritzlab.net/bucket")
immutable_paths_json = annotations.get("site-publish.fritzlab.net/immutable-paths")
route_path = annotations.get("site-publish.fritzlab.net/route-path") route_path = annotations.get("site-publish.fritzlab.net/route-path")
values = (artifact, access, bucket, immutable_paths_json, route_path) values = (artifact, access, bucket, route_path)
if all(value is None for value in values): if all(value is None for value in values):
continue continue
if (not all(isinstance(value, str) for value in values) if (not all(isinstance(value, str) for value in values)
or access not in {"public", "protected"} or not route_path.startswith("/")): or access not in {"public", "protected"} or not route_path.startswith("/")):
raise RuntimeError(f"invalid site-publish route history in {path}") raise RuntimeError(f"invalid site-publish route history in {path}")
try:
immutable_paths = json.loads(immutable_paths_json)
except json.JSONDecodeError as exc:
raise RuntimeError(f"invalid site-publish route history in {path}") from exc
if not isinstance(immutable_paths, list) or any(not isinstance(item, str) for item in immutable_paths):
raise RuntimeError(f"invalid site-publish route history in {path}")
if bucket in contracts: if bucket in contracts:
raise RuntimeError(f"duplicate site-publish route history for bucket {bucket}") raise RuntimeError(f"duplicate site-publish route history for bucket {bucket}")
contracts[bucket] = _validate_route_contract(bucket, { contracts[bucket] = {"path": route_path, "access": access, "artifact": artifact}
"path": route_path, "access": access, "artifact": artifact,
"immutable_paths": immutable_paths,
}, path)
return contracts return contracts
@@ -524,8 +355,7 @@ def validate_route_migrations(cfg, previous_contracts):
for route in cfg["routes"]: for route in cfg["routes"]:
artifact = artifacts[route["artifact"]] artifact = artifacts[route["artifact"]]
previous = previous_contracts.get(artifact["bucket"]) previous = previous_contracts.get(artifact["bucket"])
if (previous and previous["access"] == "protected" if (previous and previous["access"] == "protected" and route["access"] == "public"
and route["access"] in {"public", "legacy"}
): ):
raise RuntimeError( raise RuntimeError(
f"artifact {route['artifact']} cannot become public while reusing protected " f"artifact {route['artifact']} cannot become public while reusing protected "
@@ -553,21 +383,15 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
publish_route_immutables( publish_route_immutables(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
) )
# Reconcile every browser-read policy before publishing mutable content.
# A CORS failure therefore cannot leave a new channel pointing at a release
# whose cross-origin assets browsers cannot consume.
reconcile_artifact_cors(cfg["artifacts"], credential_env_names)
for route in cfg["routes"]: for route in cfg["routes"]:
s3_sync( s3_sync(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]), (previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]) or {}).get("path"),
) )
if cfg["compatibility"]: if cfg["compatibility"]:
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")) ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
render_site_manifests( render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
site_name, action_dir, app_dir, manifests_dir, cfg, previous_contracts,
)
commit_and_push(apps_dir, f"Deploy {site_name}", token) commit_and_push(apps_dir, f"Deploy {site_name}", token)
+1 -64
View File
@@ -1,6 +1,5 @@
"""Shared utilities for the site-publish action.""" """Shared utilities for the site-publish action."""
import ipaddress
import os import os
import re import re
import shutil import shutil
@@ -105,55 +104,6 @@ def _strings(value, label):
return values return values
def _cors_origins(value, label):
origins = _list(value, label)
if any(not isinstance(item, str) or not item for item in origins):
raise ConfigError(f"{label} must be a list of non-empty strings")
canonical = []
for origin in origins:
if origin == "*":
canonical.append(origin)
continue
parsed = urlparse(origin)
try:
port = parsed.port
except ValueError:
raise ConfigError(f"{label} must contain '*' or canonical HTTPS origins") from None
if parsed.scheme != "https" or not parsed.hostname or parsed.path or parsed.params or (
parsed.query or parsed.fragment or parsed.username or parsed.password
):
raise ConfigError(f"{label} must contain '*' or canonical HTTPS origins")
try:
address = ipaddress.ip_address(parsed.hostname)
except ValueError:
try:
hostname = parsed.hostname.encode("idna").decode("ascii")
except UnicodeError:
raise ConfigError(
f"{label} must contain '*' or canonical HTTPS origins"
) from None
try:
_hostname(hostname, f"{label} hostname")
except ConfigError:
raise ConfigError(
f"{label} must contain '*' or canonical HTTPS origins"
) from None
else:
if getattr(address, "scope_id", None) is not None:
raise ConfigError(
f"{label} must contain '*' or canonical HTTPS origins"
)
hostname = f"[{address.compressed}]" if address.version == 6 else address.compressed
canonical.append(f"https://{hostname}{f':{port}' if port not in (None, 443) else ''}")
if len(canonical) != len(set(canonical)):
raise ConfigError(f"{label} must not contain duplicate canonical origins")
if "*" in canonical and len(canonical) != 1:
raise ConfigError(f"{label} wildcard must be the only origin")
if origins != canonical:
raise ConfigError(f"{label} must contain '*' or canonical HTTPS origins")
return origins
def _hostname(value, label): def _hostname(value, label):
if not isinstance(value, str) or len(value) > 253 or value.endswith("."): if not isinstance(value, str) or len(value) > 253 or value.endswith("."):
raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot") raise ConfigError(f"{label} must be a lowercase DNS hostname without a trailing dot")
@@ -265,7 +215,6 @@ def _legacy_config(raw, site_name):
"website_authority": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net", "website_authority": "garage-s3.storage.svc.k8s.sjc001.fritzlab.net",
"credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", "secret_key_env": "AWS_SECRET_ACCESS_KEY"}, "credentials": {"access_key_env": "AWS_ACCESS_KEY_ID", "secret_key_env": "AWS_SECRET_ACCESS_KEY"},
"cache_rules": [{"path": "", "cache_control": DEFAULT_CACHE_CONTROL}], "cache_rules": [{"path": "", "cache_control": DEFAULT_CACHE_CONTROL}],
"cors_origins": None,
} }
return { return {
"version": 1, "version": 1,
@@ -285,11 +234,7 @@ def _legacy_config(raw, site_name):
def _artifact(item, index): def _artifact(item, index):
label = f"artifacts[{index}]" label = f"artifacts[{index}]"
item = _mapping(item, label) item = _mapping(item, label)
_known_keys( _known_keys(item, {"name", "type", "content_dir", "tidy", "excludes", "publish", "cache"}, label)
item,
{"name", "type", "content_dir", "tidy", "excludes", "publish", "cache", "cors_origins"},
label,
)
name = item.get("name") name = item.get("name")
if not isinstance(name, str) or not NAME_RE.fullmatch(name): if not isinstance(name, str) or not NAME_RE.fullmatch(name):
raise ConfigError(f"{label}.name must be a DNS label") raise ConfigError(f"{label}.name must be a DNS label")
@@ -358,10 +303,6 @@ def _artifact(item, index):
"website_authority": authority, "website_authority": authority,
"credentials": normalized_credentials, "credentials": normalized_credentials,
"cache_rules": cache_rules, "cache_rules": cache_rules,
"cors_origins": (
_cors_origins(item["cors_origins"], f"{label}.cors_origins")
if "cors_origins" in item else []
),
} }
@@ -448,10 +389,6 @@ def _validate_multi(cfg):
raise ConfigError(f"protected route {route['name']} cannot use shared-cache max-age") raise ConfigError(f"protected route {route['name']} cannot use shared-cache max-age")
if route["access"] == "public" and "private" in directives: if route["access"] == "public" and "private" in directives:
raise ConfigError(f"public route {route['name']} cannot use private cache policy") raise ConfigError(f"public route {route['name']} cannot use private cache policy")
if route["access"] == "protected" and "*" in artifact["cors_origins"]:
raise ConfigError(
f"protected route {route['name']} cannot allow wildcard CORS"
)
root = next(route for route in routes if route["path"] == "/") root = next(route for route in routes if route["path"] == "/")
if any(route["access"] == "protected" for route in routes) and root["access"] == "public": if any(route["access"] == "protected" for route in routes) and root["access"] == "public":
raise ConfigError("a public '/' catch-all would expose unmatched protected content") raise ConfigError("a public '/' catch-all would expose unmatched protected content")
-1
View File
@@ -8,7 +8,6 @@ metadata:
site-publish.fritzlab.net/artifact: {{ route.artifact | tojson }} site-publish.fritzlab.net/artifact: {{ route.artifact | tojson }}
site-publish.fritzlab.net/access: {{ route.access | tojson }} site-publish.fritzlab.net/access: {{ route.access | tojson }}
site-publish.fritzlab.net/bucket: {{ route.artifact_config.bucket | tojson }} site-publish.fritzlab.net/bucket: {{ route.artifact_config.bucket | tojson }}
site-publish.fritzlab.net/immutable-paths: {{ route.immutable_paths_json | tojson }}
site-publish.fritzlab.net/route-path: {{ route.path | tojson }} site-publish.fritzlab.net/route-path: {{ route.path | tojson }}
{%- endif %} {%- endif %}
traefik.ingress.kubernetes.io/router.middlewares: https-redirect@file,retry-upstream@file{% if route.access_middleware %},{{ route.access_middleware }}@file{% endif %}{% for m in route.middlewares %},{{ m }}@file{% endfor %} traefik.ingress.kubernetes.io/router.middlewares: https-redirect@file,retry-upstream@file{% if route.access_middleware %},{{ route.access_middleware }}@file{% endif %}{% for m in route.middlewares %},{{ m }}@file{% endfor %}
-1
View File
@@ -15,7 +15,6 @@ artifacts:
- name: distributions - name: distributions
type: static type: static
content_dir: dist content_dir: dist
cors_origins: ['*']
publish: publish:
bucket: baseline-dist bucket: baseline-dist
credentials: credentials:
+2 -303
View File
@@ -113,69 +113,6 @@ class ConfigContractTests(unittest.TestCase):
self.assertEqual(["/dist", "/"], [route["path"] for route in cfg["routes"]]) self.assertEqual(["/dist", "/"], [route["path"] for route in cfg["routes"]])
credentials = {artifact["name"]: artifact["credentials"] for artifact in cfg["artifacts"]} credentials = {artifact["name"]: artifact["credentials"] for artifact in cfg["artifacts"]}
self.assertNotEqual(credentials["distributions"], credentials["portal"]) self.assertNotEqual(credentials["distributions"], credentials["portal"])
distributions = next(
artifact for artifact in cfg["artifacts"] if artifact["name"] == "distributions"
)
self.assertEqual(["*"], distributions["cors_origins"])
def test_cors_origins_are_https_origins_or_wildcard(self):
self.assert_invalid(
lambda raw: raw["artifacts"][1].__setitem__(
"cors_origins", ["http://consumer.example"]
),
"must contain '\\*' or canonical HTTPS origins",
)
for origin in (
"https://consumer example",
"https://Consumer.example",
"https://consumer.example:443",
"https://consumer.example/",
"https://consumer.example:invalid",
"https://[fe80::1%eth0]",
"https://[fe80::1%25eth0]",
):
with self.subTest(origin=origin):
self.assert_invalid(
lambda raw, origin=origin: raw["artifacts"][1].__setitem__(
"cors_origins", [origin]
),
"must contain '\\*' or canonical HTTPS origins",
)
self.assert_invalid(
lambda raw: raw["artifacts"][1].__setitem__(
"cors_origins", ["https://consumer.example", "https://consumer.example:443"]
),
"must not contain duplicate canonical origins",
)
self.assert_invalid(
lambda raw: raw["artifacts"][1].__setitem__(
"cors_origins", ["*", "https://consumer.example"]
),
"wildcard must be the only origin",
)
raw = copy.deepcopy(self.raw)
raw["artifacts"][1]["cors_origins"] = ["https://[2602:817:3000::1]:8443"]
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
self.assertEqual(
["https://[2602:817:3000::1]:8443"],
next(a for a in cfg["artifacts"] if a["name"] == "distributions")["cors_origins"],
)
def test_declared_cors_origins_must_be_a_list(self):
for value in (None, False, 0, "", {}):
with self.subTest(value=value):
self.assert_invalid(
lambda raw, value=value: raw["artifacts"][1].__setitem__(
"cors_origins", value
),
"cors_origins must be a list",
)
def test_protected_route_rejects_wildcard_cors(self):
self.assert_invalid(
lambda raw: raw["artifacts"][0].__setitem__("cors_origins", ["*"]),
"protected route portal cannot allow wildcard CORS",
)
def test_equivalent_route_paths_are_ambiguous(self): def test_equivalent_route_paths_are_ambiguous(self):
self.assert_invalid(lambda raw: raw["routes"].append({ self.assert_invalid(lambda raw: raw["routes"].append({
@@ -194,7 +131,6 @@ class ConfigContractTests(unittest.TestCase):
raw["routes"][1]["access"] = { raw["routes"][1]["access"] = {
"mode": "protected", "middleware": "authentik-forwardauth" "mode": "protected", "middleware": "authentik-forwardauth"
} }
raw["artifacts"][1]["cors_origins"] = []
raw["artifacts"][0]["cache"]["rules"][0]["cache_control"] = ( raw["artifacts"][0]["cache"]["rules"][0]["cache_control"] = (
"public, max-age=0, must-revalidate" "public, max-age=0, must-revalidate"
) )
@@ -327,7 +263,6 @@ class GenerationTests(unittest.TestCase):
"app.yaml", "manifests/certificate.yaml", "manifests/ingress-distributions.yaml", "app.yaml", "manifests/certificate.yaml", "manifests/ingress-distributions.yaml",
"manifests/ingress-portal.yaml", "manifests/kustomization.yaml", "manifests/ingress-portal.yaml", "manifests/kustomization.yaml",
"manifests/service-distributions.yaml", "manifests/service-portal.yaml", "manifests/service-distributions.yaml", "manifests/service-portal.yaml",
"site-publish-history.yaml",
}, set(files)) }, set(files))
for content in files.values(): for content in files.values():
self.assertIsNotNone(yaml.safe_load(content)) self.assertIsNotNone(yaml.safe_load(content))
@@ -405,75 +340,6 @@ class BuildTests(unittest.TestCase):
class PublishingTests(unittest.TestCase): class PublishingTests(unittest.TestCase):
def test_cors_policy_is_reconciled_as_read_only_browser_access(self):
captured = {}
def capture(command, **_kwargs):
config_path = command[command.index("--cors-configuration") + 1]
captured["command"] = command
captured["config"] = json.loads(Path(config_path.removeprefix("file://")).read_text())
with patch.object(deploy, "run", side_effect=capture):
deploy.configure_cors(
"baseline-dist", ["*"], "http://garage-s3.storage.svc:3900", {}
)
self.assertIn("put-bucket-cors", captured["command"])
self.assertEqual(["GET", "HEAD"], captured["config"]["CORSRules"][0]["AllowedMethods"])
self.assertEqual(["*"], captured["config"]["CORSRules"][0]["AllowedOrigins"])
def test_empty_cors_policy_removes_stale_bucket_cors(self):
with patch.object(deploy, "run") as request:
deploy.configure_cors(
"baseline-catalogue", [], "http://garage-s3.storage.svc:3900", {}
)
self.assertIn("delete-bucket-cors", request.call_args.args[0])
def test_cors_reconciliation_restores_prior_policies_on_failure(self):
artifacts = [
{
"bucket": "first", "cors_origins": ["https://new.example"],
"s3_endpoint": "http://garage-s3.storage.svc:3900", "credentials": {},
},
{
"bucket": "second", "cors_origins": [],
"s3_endpoint": "http://garage-s3.storage.svc:3900", "credentials": {},
},
]
prior = [
{"CORSRules": [{"AllowedOrigins": ["https://old.example"]}]},
None,
]
writes = []
def write(bucket, config, *_args):
writes.append((bucket, config))
if bucket == "second" and len(writes) == 2:
raise RuntimeError("write failed")
with patch.object(deploy, "publication_aws_env", return_value={}), patch.object(
deploy, "get_cors_configuration", side_effect=prior,
) as read, patch.object(deploy, "set_cors_configuration", side_effect=write), \
self.assertRaisesRegex(RuntimeError, "write failed"):
deploy.reconcile_artifact_cors(artifacts)
self.assertEqual(2, read.call_count)
self.assertEqual(
[
("first", {"CORSRules": [{
"AllowedOrigins": ["https://new.example"],
"AllowedMethods": ["GET", "HEAD"],
"AllowedHeaders": ["*"],
"ExposeHeaders": ["ETag"],
"MaxAgeSeconds": 3600,
}]}),
("second", None),
("second", None),
("first", prior[0]),
],
writes,
)
def test_apps_clone_never_places_token_in_argv_or_log(self): def test_apps_clone_never_places_token_in_argv_or_log(self):
calls = [] calls = []
secret = "clone-secret-must-not-appear" secret = "clone-secret-must-not-appear"
@@ -605,10 +471,7 @@ class PublishingTests(unittest.TestCase):
}, clear=False), patch.object( }, clear=False), patch.object(
deploy, "run", side_effect=lambda command, **_: commands.append(command) deploy, "run", side_effect=lambda command, **_: commands.append(command)
): ):
deploy.s3_sync(artifact, route, root, previous_contract={ deploy.s3_sync(artifact, route, root, previous_path="/foo")
"path": "/foo", "access": "public", "artifact": "distributions",
"immutable_paths": ["foo/releases"],
})
rendered = [" ".join(command) for command in commands] rendered = [" ".join(command) for command in commands]
self.assertTrue(all("foo/releases/*" in command for command in rendered[:2])) self.assertTrue(all("foo/releases/*" in command for command in rendered[:2]))
self.assertTrue(all("*/releases/*" not in command for command in rendered)) self.assertTrue(all("*/releases/*" not in command for command in rendered))
@@ -618,7 +481,6 @@ class PublishingTests(unittest.TestCase):
previous = { previous = {
"baseline-dist": { "baseline-dist": {
"path": "/dist", "access": "protected", "artifact": "old-name", "path": "/dist", "access": "protected", "artifact": "old-name",
"immutable_paths": ["dist/releases"],
} }
} }
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"): with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
@@ -633,142 +495,6 @@ class PublishingTests(unittest.TestCase):
previous = {"retired-protected-bucket": previous["baseline-dist"]} previous = {"retired-protected-bucket": previous["baseline-dist"]}
deploy.validate_route_migrations(cfg, previous) deploy.validate_route_migrations(cfg, previous)
def test_protected_split_bucket_cannot_become_legacy_public(self):
cfg = normalize_site_config(fixture("legacy-site.yaml"), "baseline.fritzlab.net")
previous = {
"baseline.fritzlab.net": {
"path": "/portal", "access": "protected", "artifact": "portal",
"immutable_paths": [],
}
}
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(cfg, previous)
def test_removed_immutable_rule_preserves_prior_keys(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
artifact["cache_rules"] = [
rule for rule in artifact["cache_rules"] if rule["path"] != "releases"
]
route = next(item for item in cfg["routes"] if item["artifact"] == "distributions")
with tempfile.TemporaryDirectory() as tmp:
html = Path(tmp)
filters = deploy.retired_immutable_filters(artifact, route, html, {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_paths": ["dist/releases"],
})
self.assertEqual(["--exclude", "releases/*"], filters)
(html / "releases").mkdir()
(html / "releases" / "replacement.js").write_text("mutable")
with self.assertRaisesRegex(RuntimeError, "collides with retired immutable"):
deploy.retired_immutable_filters(artifact, route, html, {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_paths": ["dist/releases"],
})
def test_removed_immutable_rule_remains_in_next_manifest_history(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
artifact["cache_rules"] = [
rule for rule in artifact["cache_rules"] if rule["path"] != "releases"
]
previous = {
"baseline-dist": {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_paths": ["dist/releases"],
}
}
with tempfile.TemporaryDirectory() as tmp:
app_dir = Path(tmp) / "app"
manifests = app_dir / "manifests"
deploy.render_site_manifests(
"baseline.fritzlab.net", ROOT, app_dir, manifests, cfg, previous,
)
first = deploy.previous_route_contracts(app_dir)
self.assertEqual(["dist/releases"], first["baseline-dist"]["immutable_paths"])
deploy.render_site_manifests(
"baseline.fritzlab.net", ROOT, app_dir, manifests, cfg, first,
)
second = deploy.previous_route_contracts(app_dir)
self.assertEqual(first, second)
def test_protected_split_bucket_survives_absence_and_blocks_legacy(self):
raw = fixture("split-site.yaml")
raw["artifacts"] = [
item for item in raw["artifacts"] if item["name"] == "distributions"
]
raw["routes"] = [
item for item in raw["routes"] if item["artifact"] == "distributions"
]
raw["routes"][0]["path"] = "/"
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
previous = {
"baseline.fritzlab.net": {
"path": "/", "access": "protected", "artifact": "portal",
"immutable_paths": [],
}
}
with tempfile.TemporaryDirectory() as tmp:
app_dir = Path(tmp) / "app"
deploy.render_site_manifests(
"baseline.fritzlab.net", ROOT, app_dir, app_dir / "manifests", cfg, previous,
)
retained = deploy.previous_route_contracts(app_dir)
self.assertEqual(
previous["baseline.fritzlab.net"], retained["baseline.fritzlab.net"],
)
legacy = normalize_site_config(
fixture("legacy-site.yaml"), "baseline.fritzlab.net",
)
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(legacy, retained)
def test_malformed_persistent_history_fails_closed(self):
with tempfile.TemporaryDirectory() as tmp:
app_dir = Path(tmp)
(app_dir / "site-publish-history.yaml").write_text(
"version: 1\nbuckets:\n bucket:\n path: /\n"
" access: public\n artifact: site\n"
" immutable_paths: [../releases]\n"
)
with self.assertRaisesRegex(RuntimeError, "invalid site-publish route history"):
deploy.previous_route_contracts(app_dir)
def test_retired_immutable_history_survives_a_route_move(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
artifact["cache_rules"] = [
rule for rule in artifact["cache_rules"] if rule["path"] != "releases"
]
route = next(item for item in cfg["routes"] if item["artifact"] == "distributions")
route["path"] = "/"
previous = {
"baseline-dist": {
"path": "/foo", "access": "public", "artifact": "distributions",
"immutable_paths": ["foo/releases"],
}
}
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
app_dir = root / "app"
manifests = app_dir / "manifests"
app_dir.mkdir()
deploy.render_site_manifests(
"baseline.fritzlab.net", ROOT, app_dir, manifests, cfg, previous,
)
following = deploy.previous_route_contracts(app_dir)
self.assertEqual(["foo/releases"], following["baseline-dist"]["immutable_paths"])
html = root / "html"
html.mkdir()
first_filters = deploy.retired_immutable_filters(
artifact, route, html, previous["baseline-dist"],
)
following_filters = deploy.retired_immutable_filters(
artifact, route, html, following["baseline-dist"],
)
self.assertEqual(["--exclude", "foo/releases/*"], first_filters)
self.assertEqual(first_filters, following_filters)
def test_later_route_immutable_failure_stops_all_mutable_publication(self): def test_later_route_immutable_failure_stops_all_mutable_publication(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
@@ -778,42 +504,15 @@ class PublishingTests(unittest.TestCase):
patch.object(deploy, "clone_apps", return_value=root / "apps"), patch.object( patch.object(deploy, "clone_apps", return_value=root / "apps"), patch.object(
deploy, "publish_route_immutables", deploy, "publish_route_immutables",
side_effect=[None, RuntimeError("immutable failed")], side_effect=[None, RuntimeError("immutable failed")],
) as immutable_publish, patch.object( ) as immutable_publish, patch.object(deploy, "s3_sync") as mutable_sync, \
deploy, "reconcile_artifact_cors"
) as cors_reconcile, patch.object(deploy, "s3_sync") as mutable_sync, \
self.assertRaisesRegex( self.assertRaisesRegex(
RuntimeError, "immutable failed" RuntimeError, "immutable failed"
): ):
deploy.deploy_static("baseline", root, root, "token", cfg) deploy.deploy_static("baseline", root, root, "token", cfg)
self.assertEqual(2, immutable_publish.call_count) self.assertEqual(2, immutable_publish.call_count)
cors_reconcile.assert_not_called()
mutable_sync.assert_not_called() mutable_sync.assert_not_called()
def test_all_cors_policies_complete_before_mutable_publication(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
events = []
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
apps = root / "apps"
apps.mkdir()
with patch.object(deploy, "validate_publication_environment"), patch.object(
deploy, "validate_artifact_output"
), patch.object(
deploy, "publish_route_immutables", side_effect=lambda *_args: events.append("immutable")
), patch.object(
deploy, "reconcile_artifact_cors", side_effect=lambda *_args: events.append("cors")
), patch.object(
deploy, "s3_sync", side_effect=lambda *_args: events.append("mutable")
), patch.object(deploy, "clone_apps", return_value=apps), patch.object(
deploy, "render_site_manifests"
), patch.object(deploy, "commit_and_push"):
deploy.deploy_static("baseline", root, root, "token", cfg)
self.assertEqual(
["immutable", "immutable", "cors", "mutable", "mutable"], events
)
def test_absent_artifact_is_detected_before_publish(self): def test_absent_artifact_is_detected_before_publish(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")