Author SHA1 Message Date
Evelyn Chen 7667ae540e feat: add split-surface site publishing
Test / contract (pull_request) Successful in 5s
2026-08-29 21:22:04 +00:00
5 changed files with 68 additions and 833 deletions
+2 -28
View File
@@ -92,10 +92,8 @@ routes:
``` ```
The caller supplies each declared credential name as an environment variable The caller supplies each declared credential name as an environment variable
on the action step. Names must be matched `<NAME>_S3_ACCESS_KEY` and on the action step. Credential values are passed to `aws` only through its
`<NAME>_S3_SECRET_KEY` pairs; arbitrary environment variables cannot become environment and never appear in a logged command or process argument.
publication credentials. Values pass to `aws` only through its environment and
never appear in a logged command or process argument.
```yaml ```yaml
- uses: https://code.fritzlab.net/action/site-publish@v1 - uses: https://code.fritzlab.net/action/site-publish@v1
@@ -122,30 +120,6 @@ revalidation, or `no-store` plus a positive max-age) are rejected. Protected
artifacts require `private` or `no-store` and cannot emit `public`. artifacts require `private` or `no-store` and cannot emit `public`.
Metadata restamping transfers each artifact once even on a no-op publication; Metadata restamping transfers each artifact once even on a no-op publication;
that is the cost of making policy changes effective on unchanged Garage objects. that is the cost of making policy changes effective on unchanged Garage objects.
An immutable cache path is excluded from sync and deletion. Every object key in
that path must contain exactly one full publication SHA-256, calculated over its
cache policy, content type, and bytes. That content address makes concurrent
writes identical even though Garage v2.2.0 has no conditional destination
write. An identical retry converges; a changed object, missing digest metadata,
wrong address, or nested policy under that immutable prefix fails publication.
Every immutable target across every artifact is validated and published before
any route's mutable objects change.
Mutable default and override partitions receive their final cache policy before
the matching prefix-scoped stale deletion, so publication never exposes a
provisional cache policy or a pointer to a missing immutable target.
An append-only `site-publish-history.json` beside each generated site retains every bucket's access
class and absolute immutable prefixes, including removed routes and rules. When a route move places
a retired prefix inside the new sync scope, that subtree is excluded; a current-file collision
fails publication. Protected access remains sticky across artifact renames and legacy mode, so
publishing the same artifact publicly requires a new bucket. Decommissioning removes the live
application and manifests while retaining this history because its Garage bucket is not purged.
Artifact input directories must be pairwise disjoint after filesystem
resolution. Publication stops before build or upload if one contains another or
escapes the repository. Symlinked roots, components, and descendants are also rejected, preventing
protected input from entering a public artifact through dereference. Split
storage endpoints are pinned to Garage, and each website
authority is derived from its bucket; a site cannot expose an arbitrary backend.
Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net` Each split route gets a bucket-specific `<bucket>.web.sjc001.fritzlab.net`
ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route ExternalName Service annotated to disable pass-host-header and a separate Ingress. Route
+1 -3
View File
@@ -5,7 +5,7 @@ import subprocess
import tempfile import tempfile
from pathlib import Path from pathlib import Path
from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs from utils import EXCLUDE_FILES, env, parse_site_yaml, run
def build_artifact(site_dir, artifact): def build_artifact(site_dir, artifact):
@@ -64,7 +64,5 @@ def cmd_build():
print("Site disabled — skipping build") print("Site disabled — skipping build")
return return
validate_artifact_inputs(site_dir, cfg)
for artifact in cfg["artifacts"]: for artifact in cfg["artifacts"]:
build_artifact(site_dir, artifact) build_artifact(site_dir, artifact)
+39 -293
View File
@@ -1,14 +1,10 @@
"""Deploy phase — S3 sync, manifest rendering, alias reconcile.""" """Deploy phase — S3 sync, manifest rendering, alias reconcile."""
import fnmatch
import hashlib
import json import json
import mimetypes
import os import os
import re
import shutil import shutil
import subprocess import tempfile
from pathlib import Path, PurePosixPath from pathlib import Path
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
@@ -22,13 +18,11 @@ from utils import (
parse_site_yaml, parse_site_yaml,
render_templates, render_templates,
run, run,
validate_artifact_inputs,
) )
GARAGE_ADMIN_ENDPOINT = os.environ.get( GARAGE_ADMIN_ENDPOINT = os.environ.get(
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903" "GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
) )
HISTORY_FILE = "site-publish-history.json"
def validate_artifact_output(site_dir, artifact): def validate_artifact_output(site_dir, artifact):
@@ -53,105 +47,9 @@ def validate_publication_environment(cfg):
die("GARAGE_ADMIN_TOKEN is required when aliases are declared") die("GARAGE_ADMIN_TOKEN is required when aliases are declared")
def _is_immutable(rule): def s3_sync(artifact, route, site_dir, credential_env_names=None):
return "immutable" in { endpoint = artifact["s3_endpoint"]
part.strip().lower().split("=", 1)[0] html_dir = site_dir / artifact["build_dir"]
for part in rule["cache_control"].split(",")
}
def _aws_capture(args, aws_env):
"""Run a non-streaming AWS request without exposing environment credentials."""
print(f" $ {' '.join(str(part) for part in args)}")
return subprocess.run(args, env=aws_env, text=True, capture_output=True, check=False)
def _immutable_head(endpoint, bucket, key, aws_env):
args = ["aws", "--endpoint-url", endpoint, "s3api", "head-object",
"--bucket", bucket, "--key", key, "--output", "json"]
result = _aws_capture(args, aws_env)
if result.returncode == 0:
return json.loads(result.stdout)
error = f"{result.stdout}\n{result.stderr}"
if any(marker in error for marker in ("404", "Not Found", "NoSuchKey")):
return None
raise RuntimeError(f"head-object failed for s3://{bucket}/{key}: {error.strip()}")
def _immutable_digests(source, cache_control, content_type):
with source.open("rb") as stream:
content_digest = hashlib.file_digest(stream, "sha256").hexdigest()
publication = hashlib.sha256()
publication.update(cache_control.encode())
publication.update(b"\0")
publication.update(content_type.encode())
publication.update(b"\0")
with source.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
publication.update(block)
return content_digest, publication.hexdigest()
def _same_immutable_object(info, content_digest, publication_digest, cache_control, content_type):
metadata = {key.lower(): value for key, value in (info.get("Metadata") or {}).items()}
return (
metadata.get("sha256") == content_digest
and metadata.get("publication-sha256") == publication_digest
and info.get("CacheControl") == cache_control
and info.get("ContentType") == content_type
)
def publish_immutable_file(endpoint, bucket, key, source, cache_control, aws_env):
"""Publish a content-addressed key; identical retries converge."""
content_type = mimetypes.guess_type(source.name)[0] or "application/octet-stream"
content_digest, publication_digest = _immutable_digests(source, cache_control, content_type)
address_digests = re.findall(r"(?<![0-9a-f])([0-9a-f]{64})(?![0-9a-f])", key.lower())
if address_digests != [publication_digest]:
raise RuntimeError(
f"immutable key must contain its one publication SHA-256 {publication_digest}: "
f"s3://{bucket}/{key}"
)
existing = _immutable_head(endpoint, bucket, key, aws_env)
if existing is not None:
if _same_immutable_object(
existing, content_digest, publication_digest, cache_control, content_type,
):
print(f" Immutable object already matches: s3://{bucket}/{key}")
return False
raise RuntimeError(f"immutable object differs or lacks publisher digest: s3://{bucket}/{key}")
args = ["aws", "--endpoint-url", endpoint, "s3api", "put-object",
"--bucket", bucket, "--key", key, "--body", str(source),
"--content-type", content_type, "--cache-control", cache_control,
"--metadata", f"sha256={content_digest},publication-sha256={publication_digest}"]
result = _aws_capture(args, aws_env)
if result.returncode == 0:
return True
error = f"{result.stdout}\n{result.stderr}"
raise RuntimeError(f"put-object failed for s3://{bucket}/{key}: {error.strip()}")
def publish_immutable_rule(artifact, route, rule, html_dir, aws_env):
"""Publish one immutable cache partition without overwrite or deletion."""
rule_root = html_dir / rule["path"]
child_paths = [candidate["path"] for candidate in artifact["cache_rules"]
if candidate["path"].startswith(f"{rule['path'].rstrip('/')}/")]
object_prefix = route["path"].strip("/")
for source in sorted(path for path in rule_root.rglob("*") if path.is_file()):
relative = source.relative_to(html_dir).as_posix()
if any(relative == child or relative.startswith(f"{child}/") for child in child_paths):
continue
if any(fnmatch.fnmatch(relative, pattern) for pattern in artifact["excludes"]):
continue
key = "/".join(part for part in (object_prefix, relative) if part)
publish_immutable_file(
artifact["s3_endpoint"], artifact["bucket"], key, source,
rule["cache_control"], aws_env,
)
def publication_aws_env(artifact, credential_env_names=None):
"""Build the route-scoped AWS environment without leaking other credentials."""
access_key = env(artifact["credentials"]["access_key_env"]) access_key = env(artifact["credentials"]["access_key_env"])
secret_key = env(artifact["credentials"]["secret_key_env"]) secret_key = env(artifact["credentials"]["secret_key_env"])
aws_env = os.environ.copy() aws_env = os.environ.copy()
@@ -165,60 +63,11 @@ def publication_aws_env(artifact, credential_env_names=None):
"AWS_SECRET_ACCESS_KEY": secret_key, "AWS_SECRET_ACCESS_KEY": secret_key,
"AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"), "AWS_DEFAULT_REGION": os.environ.get("AWS_DEFAULT_REGION", "sjc001"),
}) })
return aws_env
def publish_route_immutables(artifact, route, site_dir, credential_env_names=None):
"""Publish one route's immutable partitions during the global preflight."""
html_dir = site_dir / artifact["build_dir"]
aws_env = publication_aws_env(artifact, credential_env_names)
for rule in artifact["cache_rules"]:
if _is_immutable(rule):
publish_immutable_rule(artifact, route, rule, html_dir, aws_env)
def retired_immutable_filters(artifact, route, html_dir, previous_contract):
"""Protect every historical immutable prefix inside the current sync scope."""
if not previous_contract:
return []
current_prefix = route["path"].strip("/")
filters = []
current_immutable = set(immutable_prefixes(artifact, route))
for immutable_prefix in previous_contract["immutable_prefixes"]:
if current_prefix:
marker = f"{current_prefix}/"
if not immutable_prefix.startswith(marker):
continue
relative_path = immutable_prefix[len(marker):]
else:
relative_path = immutable_prefix
collision = html_dir / relative_path
if (immutable_prefix not in current_immutable and collision.exists()
and any(path.is_file() for path in collision.rglob("*"))):
raise RuntimeError(
f"current artifact collides with retired immutable partition: {relative_path}"
)
filters.extend(("--exclude", f"{relative_path}/*"))
return filters
def immutable_prefixes(artifact, route):
route_prefix = route["path"].strip("/")
return sorted({
"/".join(part for part in (route_prefix, rule["path"]) if part)
for rule in artifact["cache_rules"] if _is_immutable(rule)
})
def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contract=None):
endpoint = artifact["s3_endpoint"]
html_dir = site_dir / artifact["build_dir"]
aws_env = publication_aws_env(artifact, credential_env_names)
bucket = artifact["bucket"] bucket = artifact["bucket"]
object_prefix = route["path"].strip("/") object_prefix = route["path"].strip("/")
destination = f"s3://{bucket}/{object_prefix + '/' if object_prefix else ''}" bucket_destination = f"s3://{bucket}/"
destination = f"{bucket_destination}{object_prefix + '/' if object_prefix else ''}"
default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"]) default_cache = next(rule["cache_control"] for rule in artifact["cache_rules"] if not rule["path"])
immutable_paths = [rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)]
# `excludes` are patterns (site.yaml `excludes:` list) that should never # `excludes` are patterns (site.yaml `excludes:` list) that should never
# be uploaded *and* should never be deleted from the bucket — escape hatch # be uploaded *and* should never be deleted from the bucket — escape hatch
# for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli). # for assets managed out-of-band (e.g. large PDFs uploaded via aws-cli).
@@ -226,30 +75,41 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contr
if artifact["excludes"]: if artifact["excludes"]:
print(f"Excluding patterns: {artifact['excludes']}") print(f"Excluding patterns: {artifact['excludes']}")
print(f"Syncing artifact {artifact['name']}{destination} via {endpoint}") print(f"Syncing artifact {artifact['name']}{destination} via {endpoint}")
# Upload with the final cache policy before cleanup. Sync and deletion are # `sync --delete` handles new/changed/orphaned files. Partitioned
# scoped to the same current route prefix and cache partition. A route move # `cp --recursive` calls then re-upload each file once to refresh metadata
# leaves its old bucket partition intact but unreachable after the old # (cache-control, content-type) on objects sync skipped as unchanged.
# Ingress disappears, while stale mutable keys on the serving prefix are
# deleted. Immutable subtrees are structurally excluded. `cp --recursive`
# refreshes metadata atomically per object before `sync --delete` removes
# stale keys without ever exposing new bytes under a provisional policy.
# A no-op deploy therefore transfers the artifact bytes once. # A no-op deploy therefore transfers the artifact bytes once.
# AWS CLI guesses Content-Type from file extension on local→S3 uploads, # AWS CLI guesses Content-Type from file extension on local→S3 uploads,
# so a fresh upload always carries the right MIME type. # so a fresh upload always carries the right MIME type.
stage = None
sync_source = html_dir
sync_excludes = exclude_args
if object_prefix:
stage = tempfile.TemporaryDirectory()
sync_source = Path(stage.name)
staged_artifact = sync_source / object_prefix
staged_artifact.parent.mkdir(parents=True, exist_ok=True)
staged_artifact.symlink_to(html_dir.resolve(), target_is_directory=True)
sync_excludes = [arg for pattern in artifact["excludes"]
for arg in ("--exclude", f"{object_prefix}/{pattern}")]
try:
# Sync the complete bucket authority so moving a route prefix also
# deletes objects under its old prefix instead of leaving them public.
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{sync_source}/", bucket_destination,
"--delete", "--only-show-errors", "--cache-control", default_cache,
*sync_excludes], env=aws_env)
finally:
if stage:
stage.cleanup()
print("Re-stamping metadata on all objects...")
specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]] specific_paths = [rule["path"] for rule in artifact["cache_rules"] if rule["path"]]
default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")] default_filters = [arg for path in specific_paths for arg in ("--exclude", f"{path}/*")]
retired_filters = retired_immutable_filters(artifact, route, html_dir, previous_contract)
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination, run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", default_cache, "--recursive", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *retired_filters, *exclude_args], env=aws_env) *default_filters, *exclude_args], env=aws_env)
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--delete", "--only-show-errors", "--cache-control", default_cache,
*default_filters, *retired_filters, *exclude_args], env=aws_env)
for rule in artifact["cache_rules"]: for rule in artifact["cache_rules"]:
if not rule["path"]: if not rule["path"]:
continue continue
if _is_immutable(rule):
continue
include = f"{rule['path'].rstrip('/')}/*" include = f"{rule['path'].rstrip('/')}/*"
child_filters = [arg for path in specific_paths child_filters = [arg for path in specific_paths
if path.startswith(f"{rule['path'].rstrip('/')}/") if path.startswith(f"{rule['path'].rstrip('/')}/")
@@ -259,9 +119,6 @@ def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contr
run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination, run(["aws", "--endpoint-url", endpoint, "s3", "cp", f"{html_dir}/", destination,
"--recursive", "--only-show-errors", "--cache-control", rule["cache_control"], "--recursive", "--only-show-errors", "--cache-control", rule["cache_control"],
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env) "--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
run(["aws", "--endpoint-url", endpoint, "s3", "sync", f"{html_dir}/", destination,
"--delete", "--only-show-errors", "--cache-control", rule["cache_control"],
"--exclude", "*", "--include", include, *child_filters, *exclude_args], env=aws_env)
def garage_admin(method, path, token, body=None): def garage_admin(method, path, token, body=None):
@@ -332,99 +189,6 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
render_templates(action_dir, template_vars, app_dir, manifests_dir) render_templates(action_dir, template_vars, app_dir, manifests_dir)
def previous_route_contracts(app_dir):
"""Read the append-only bucket history kept beside generated manifests."""
path = app_dir / HISTORY_FILE
if not path.exists():
return {}
try:
document = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError(f"invalid site-publish route history in {path}") from exc
if (not isinstance(document, dict) or set(document) != {"schemaVersion", "buckets"}
or document["schemaVersion"] != 1 or not isinstance(document["buckets"], dict)):
raise RuntimeError(f"invalid site-publish route history in {path}")
for bucket, contract in document["buckets"].items():
if (not isinstance(bucket, str) or not isinstance(contract, dict)
or set(contract) != {"access", "artifact", "immutablePrefixes", "routePath"}
or contract["access"] not in {"legacy", "protected", "public"}
or not isinstance(contract["artifact"], str)
or not isinstance(contract["routePath"], str)
or not contract["routePath"].startswith("/")
or not isinstance(contract["immutablePrefixes"], list)
or any(not _valid_immutable_prefix(value)
for value in contract["immutablePrefixes"])
or len(contract["immutablePrefixes"]) != len(set(contract["immutablePrefixes"]))):
raise RuntimeError(f"invalid site-publish route history in {path}")
return {
bucket: {
"access": contract["access"],
"artifact": contract["artifact"],
"immutable_prefixes": sorted(contract["immutablePrefixes"]),
"path": contract["routePath"],
}
for bucket, contract in document["buckets"].items()
}
def next_route_contracts(cfg, previous_contracts):
"""Carry protected access and immutable prefixes forward for every known bucket."""
contracts = json.loads(json.dumps(previous_contracts))
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
for route in cfg["routes"]:
artifact = artifacts[route["artifact"]]
bucket = artifact["bucket"]
previous = previous_contracts.get(bucket)
access = "protected" if (
route["access"] == "protected" or previous and previous["access"] == "protected"
) else route["access"]
contracts[bucket] = {
"access": access,
"artifact": route["artifact"],
"immutable_prefixes": sorted(set(
(previous or {}).get("immutable_prefixes", []) + immutable_prefixes(artifact, route)
)),
"path": route["path"],
}
return contracts
def write_route_contracts(app_dir, contracts):
app_dir.mkdir(parents=True, exist_ok=True)
document = {
"schemaVersion": 1,
"buckets": {
bucket: {
"access": contract["access"],
"artifact": contract["artifact"],
"immutablePrefixes": contract["immutable_prefixes"],
"routePath": contract["path"],
}
for bucket, contract in sorted(contracts.items())
},
}
(app_dir / HISTORY_FILE).write_text(f"{json.dumps(document, indent=2, sort_keys=True)}\n")
def _valid_immutable_prefix(value):
return (isinstance(value, str) and value and not value.startswith("/")
and not value.endswith("/") and ".." not in PurePosixPath(value).parts)
def validate_route_migrations(cfg, previous_contracts):
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
for route in cfg["routes"]:
artifact = artifacts[route["artifact"]]
previous = previous_contracts.get(artifact["bucket"])
if (previous and previous["access"] == "protected"
and route["access"] in {"public", "legacy"}
):
raise RuntimeError(
f"artifact {route['artifact']} cannot become public while reusing protected "
f"bucket {artifact['bucket']}"
)
def deploy_static(site_name, site_dir, action_dir, token, cfg): def deploy_static(site_name, site_dir, action_dir, token, cfg):
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
credential_env_names = { credential_env_names = {
@@ -433,30 +197,18 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
validate_publication_environment(cfg) validate_publication_environment(cfg)
for artifact in cfg["artifacts"]: for artifact in cfg["artifacts"]:
validate_artifact_output(site_dir, artifact) validate_artifact_output(site_dir, artifact)
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
manifests_dir = app_dir / "manifests"
previous_contracts = previous_route_contracts(app_dir)
validate_route_migrations(cfg, previous_contracts)
# Complete immutable work across the whole publication before any route's
# mutable pointers can change. Partial immutable success is safe; mixing a
# new route with an old route after a later immutable failure is not.
for route in cfg["routes"]: for route in cfg["routes"]:
publish_route_immutables( s3_sync(artifact_by_name[route["artifact"]], route, site_dir, credential_env_names)
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
)
for route in cfg["routes"]:
s3_sync(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]),
)
if cfg["compatibility"]: if cfg["compatibility"]:
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN")) ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
write_route_contracts(app_dir, next_route_contracts(cfg, previous_contracts)) apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
manifests_dir = app_dir / "manifests"
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg) render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
commit_and_push(apps_dir, f"Deploy {site_name}", token) commit_and_push(apps_dir, f"Deploy {site_name}")
def decommission(site_name, token, buckets=None): def decommission(site_name, token, buckets=None):
@@ -466,13 +218,8 @@ def decommission(site_name, token, buckets=None):
if not site_path.exists(): if not site_path.exists():
print(f"No manifests for {site_name} — nothing to remove") print(f"No manifests for {site_name} — nothing to remove")
return return
history_path = site_path / HISTORY_FILE
history = history_path.read_bytes() if history_path.exists() else None
shutil.rmtree(site_path) shutil.rmtree(site_path)
if history is not None: commit_and_push(apps_dir, f"Decommission {site_name}")
site_path.mkdir(parents=True)
(site_path / HISTORY_FILE).write_bytes(history)
commit_and_push(apps_dir, f"Decommission {site_name}", token)
for bucket in buckets or [site_name]: for bucket in buckets or [site_name]:
print(f"Bucket {bucket} and its objects are NOT purged automatically.") print(f"Bucket {bucket} and its objects are NOT purged automatically.")
print(f" garage bucket delete {bucket} --yes") print(f" garage bucket delete {bucket} --yes")
@@ -492,5 +239,4 @@ def cmd_deploy():
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]]) decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
return return
validate_artifact_inputs(site_dir, cfg)
deploy_static(site_name, site_dir, action_dir, token, cfg) deploy_static(site_name, site_dir, action_dir, token, cfg)
+12 -70
View File
@@ -26,7 +26,6 @@ EXCLUDE_FILES = {
VALID_TYPES = {"static", "hugo", "mkdocs"} VALID_TYPES = {"static", "hugo", "mkdocs"}
NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
ENV_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$") ENV_RE = re.compile(r"^[A-Z_][A-Z0-9_]*$")
ACCESS_KEY_ENV_RE = re.compile(r"^([A-Z][A-Z0-9_]*)_S3_ACCESS_KEY$")
BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?$") BUCKET_RE = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?$")
MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9_-]{0,61}[A-Za-z0-9])?$") MIDDLEWARE_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9_-]{0,61}[A-Za-z0-9])?$")
@@ -239,7 +238,7 @@ def _artifact(item, index):
if not isinstance(name, str) or not NAME_RE.fullmatch(name): if not isinstance(name, str) or not NAME_RE.fullmatch(name):
raise ConfigError(f"{label}.name must be a DNS label") raise ConfigError(f"{label}.name must be a DNS label")
publish = _mapping(item.get("publish"), f"{label}.publish") publish = _mapping(item.get("publish"), f"{label}.publish")
_known_keys(publish, {"bucket", "credentials"}, f"{label}.publish") _known_keys(publish, {"bucket", "endpoint", "website_authority", "credentials"}, f"{label}.publish")
bucket = publish.get("bucket") bucket = publish.get("bucket")
if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket): if not isinstance(bucket, str) or not BUCKET_RE.fullmatch(bucket):
raise ConfigError(f"{label}.publish.bucket is not a valid bucket name") raise ConfigError(f"{label}.publish.bucket is not a valid bucket name")
@@ -251,15 +250,6 @@ def _artifact(item, index):
if not isinstance(value, str) or not ENV_RE.fullmatch(value): if not isinstance(value, str) or not ENV_RE.fullmatch(value):
raise ConfigError(f"{label}.publish.credentials.{key} must name an environment variable") raise ConfigError(f"{label}.publish.credentials.{key} must name an environment variable")
normalized_credentials[key] = value normalized_credentials[key] = value
access_match = ACCESS_KEY_ENV_RE.fullmatch(normalized_credentials["access_key_env"])
expected_secret = (
f"{access_match.group(1)}_S3_SECRET_KEY" if access_match else None
)
if normalized_credentials["secret_key_env"] != expected_secret:
raise ConfigError(
f"{label}.publish.credentials must be a matched "
"<NAME>_S3_ACCESS_KEY and <NAME>_S3_SECRET_KEY pair"
)
cache = _mapping(item.get("cache"), f"{label}.cache") cache = _mapping(item.get("cache"), f"{label}.cache")
_known_keys(cache, {"rules"}, f"{label}.cache") _known_keys(cache, {"rules"}, f"{label}.cache")
rules = _list(cache.get("rules"), f"{label}.cache.rules") rules = _list(cache.get("rules"), f"{label}.cache.rules")
@@ -280,15 +270,10 @@ def _artifact(item, index):
if "" not in paths: if "" not in paths:
raise ConfigError(f"{label}.cache.rules must declare a '/' default") raise ConfigError(f"{label}.cache.rules must declare a '/' default")
cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"])) cache_rules.sort(key=lambda rule: (len(PurePosixPath(rule["path"]).parts), rule["path"]))
immutable_paths = [rule["path"] for rule in cache_rules authority = _hostname(
if "immutable" in {part.strip().lower().split("=", 1)[0] publish.get("website_authority", f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}"),
for part in rule["cache_control"].split(",")}] f"{label}.publish.website_authority",
if "" in immutable_paths: )
raise ConfigError(f"{label}.cache.rules immutable paths must be narrower than '/'")
for path in immutable_paths:
if any(other.startswith(f"{path}/") for other in paths):
raise ConfigError(f"{label}.cache.rules cannot nest another policy under immutable /{path}")
authority = f"{bucket}.{DEFAULT_WEBSITE_SUFFIX}"
if not isinstance(item.get("tidy", True), bool): if not isinstance(item.get("tidy", True), bool):
raise ConfigError(f"{label}.tidy must be a boolean") raise ConfigError(f"{label}.tidy must be a boolean")
return { return {
@@ -299,7 +284,7 @@ def _artifact(item, index):
"excludes": _strings(item.get("excludes") or [], f"{label}.excludes"), "excludes": _strings(item.get("excludes") or [], f"{label}.excludes"),
"build_dir": f".site-publish/{name}/html", "build_dir": f".site-publish/{name}/html",
"bucket": bucket, "bucket": bucket,
"s3_endpoint": DEFAULT_S3_ENDPOINT, "s3_endpoint": _endpoint(publish.get("endpoint", DEFAULT_S3_ENDPOINT), f"{label}.publish.endpoint"),
"website_authority": authority, "website_authority": authority,
"credentials": normalized_credentials, "credentials": normalized_credentials,
"cache_rules": cache_rules, "cache_rules": cache_rules,
@@ -425,42 +410,6 @@ def normalize_site_config(raw, site_name):
return cfg return cfg
def validate_artifact_inputs(site_dir, cfg):
"""Reject source containment before a public or protected build starts."""
if cfg["compatibility"]:
return
root = Path(site_dir).resolve()
sources = []
for artifact in cfg["artifacts"]:
declared = root
for component in Path(artifact["content_dir"]).parts:
declared /= component
if declared.is_symlink():
raise ConfigError(
f"artifact {artifact['name']} content_dir contains symlink component: "
f"{declared.relative_to(root)}"
)
source = declared.resolve()
if source != root and root not in source.parents:
raise ConfigError(
f"artifact {artifact['name']} content_dir resolves outside the repository"
)
if source.exists():
symlink = next((path for path in source.rglob("*") if path.is_symlink()), None)
if symlink is not None:
raise ConfigError(
f"artifact {artifact['name']} build input contains symlink: "
f"{symlink.relative_to(root)}"
)
sources.append((artifact["name"], source))
for index, (name, source) in enumerate(sources):
for other_name, other_source in sources[index + 1:]:
if source == other_source or source in other_source.parents or other_source in source.parents:
raise ConfigError(
f"artifact build inputs overlap after resolution: {name} and {other_name}"
)
def parse_site_yaml(site_dir, site_name=None): def parse_site_yaml(site_dir, site_name=None):
path = Path(site_dir) / "site.yaml" path = Path(site_dir) / "site.yaml"
if not path.exists(): if not path.exists():
@@ -487,7 +436,10 @@ def clone_apps(token):
apps_dir = Path("/tmp/apps-deploy") apps_dir = Path("/tmp/apps-deploy")
if apps_dir.exists(): if apps_dir.exists():
shutil.rmtree(apps_dir) shutil.rmtree(apps_dir)
clone_env = git_auth_env(token) clone_env = os.environ.copy()
clone_env["CI_BOT_TOKEN"] = token
clone_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
clone_env["GIT_TERMINAL_PROMPT"] = "0"
url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git" url = f"https://{user}@{GITEA_HOST}/{APPS_REPO}.git"
run(["git", "clone", "--depth", "1", url, str(apps_dir)], run(["git", "clone", "--depth", "1", url, str(apps_dir)],
display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env) display=f"git clone --depth 1 https://{user}@{GITEA_HOST}/{APPS_REPO}.git {apps_dir}", env=clone_env)
@@ -525,23 +477,13 @@ def render_templates(action_dir, template_vars, app_dir, manifests_dir):
print(f" Rendered {out_name}.j2 -> {destination}") print(f" Rendered {out_name}.j2 -> {destination}")
def git_auth_env(token): def commit_and_push(apps_dir, message):
"""Return credential-safe Git authentication shared by clone and push."""
auth_env = os.environ.copy()
auth_env["CI_BOT_TOKEN"] = token
auth_env["GIT_ASKPASS"] = str(Path(__file__).with_name("git-askpass.sh"))
auth_env["GIT_TERMINAL_PROMPT"] = "0"
return auth_env
def commit_and_push(apps_dir, message, token=None):
run(["git", "-C", str(apps_dir), "add", "-A"]) run(["git", "-C", str(apps_dir), "add", "-A"])
result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False) result = subprocess.run(["git", "-C", str(apps_dir), "diff", "--cached", "--quiet"], check=False)
if result.returncode == 0: if result.returncode == 0:
print("No manifest changes to commit") print("No manifest changes to commit")
return False return False
run(["git", "-C", str(apps_dir), "commit", "-m", message]) run(["git", "-C", str(apps_dir), "commit", "-m", message])
push_env = git_auth_env(token) if token else None run(["git", "-C", str(apps_dir), "push"])
run(["git", "-C", str(apps_dir), "push"], env=push_env)
print("Manifests pushed — ArgoCD will sync") print("Manifests pushed — ArgoCD will sync")
return True return True
+13 -438
View File
@@ -1,19 +1,11 @@
import base64
import copy import copy
import hashlib
import io import io
import json
import os import os
import socket
import subprocess
import sys import sys
import tempfile import tempfile
import threading
import unittest import unittest
from contextlib import redirect_stderr, redirect_stdout from contextlib import redirect_stderr, redirect_stdout
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
from urllib.parse import urlsplit
from unittest.mock import patch from unittest.mock import patch
import yaml import yaml
@@ -24,72 +16,13 @@ sys.path.insert(0, str(ROOT / "scripts"))
import deploy import deploy
import build import build
import utils import utils
from utils import ConfigError, normalize_site_config, validate_artifact_inputs from utils import ConfigError, normalize_site_config
def fixture(name): def fixture(name):
return yaml.safe_load((ROOT / "tests" / "fixtures" / name).read_text()) return yaml.safe_load((ROOT / "tests" / "fixtures" / name).read_text())
class IPv6GitHTTPServer(ThreadingHTTPServer):
address_family = socket.AF_INET6
class AuthenticatedGitHandler(BaseHTTPRequestHandler):
project_root = None
expected_authorization = None
def log_message(self, _format, *_args):
pass
def do_GET(self):
self._git_backend()
def do_POST(self):
self._git_backend()
def _git_backend(self):
if self.headers.get("Authorization") != self.expected_authorization:
self.send_response(401)
self.send_header("WWW-Authenticate", 'Basic realm="test"')
self.end_headers()
return
parsed = urlsplit(self.path)
length = int(self.headers.get("Content-Length", "0"))
request_body = self.rfile.read(length) if length else b""
backend_env = os.environ.copy()
backend_env.update({
"GIT_PROJECT_ROOT": str(self.project_root),
"GIT_HTTP_EXPORT_ALL": "1",
"PATH_INFO": parsed.path,
"QUERY_STRING": parsed.query,
"REQUEST_METHOD": self.command,
"CONTENT_TYPE": self.headers.get("Content-Type", ""),
"CONTENT_LENGTH": str(length),
"REMOTE_USER": "ci-bot",
"REMOTE_ADDR": "::1",
"GATEWAY_INTERFACE": "CGI/1.1",
"SERVER_PROTOCOL": "HTTP/1.1",
})
result = subprocess.run(
["git", "http-backend"], input=request_body, env=backend_env,
capture_output=True, check=True,
)
raw_headers, response_body = result.stdout.split(b"\r\n\r\n", 1)
headers, status = [], 200
for line in raw_headers.decode().split("\r\n"):
name, value = line.split(":", 1)
if name.lower() == "status":
status = int(value.strip().split(" ", 1)[0])
else:
headers.append((name, value.strip()))
self.send_response(status)
for name, value in headers:
self.send_header(name, value)
self.end_headers()
self.wfile.write(response_body)
class ConfigContractTests(unittest.TestCase): class ConfigContractTests(unittest.TestCase):
def setUp(self): def setUp(self):
self.raw = fixture("split-site.yaml") self.raw = fixture("split-site.yaml")
@@ -151,11 +84,12 @@ class ConfigContractTests(unittest.TestCase):
) )
def test_publication_credentials_cannot_be_reused(self): def test_publication_credentials_cannot_be_reused(self):
def mutate(raw): self.assert_invalid(
raw["artifacts"][0]["publish"]["credentials"] = copy.deepcopy( lambda raw: raw["artifacts"][0]["publish"]["credentials"].__setitem__(
raw["artifacts"][1]["publish"]["credentials"] "access_key_env", "DIST_S3_ACCESS_KEY"
),
"publication credential DIST_S3_ACCESS_KEY is reused",
) )
self.assert_invalid(mutate, "publication credential DIST_S3_ACCESS_KEY is reused")
def test_cache_directive_contradiction_is_rejected(self): def test_cache_directive_contradiction_is_rejected(self):
self.assert_invalid( self.assert_invalid(
@@ -165,14 +99,6 @@ class ConfigContractTests(unittest.TestCase):
"immutable requires", "immutable requires",
) )
def test_cache_policy_cannot_nest_below_immutable_path(self):
def mutate(raw):
raw["artifacts"][1]["cache"]["rules"].append({
"path": "releases/candidates",
"cache_control": "public, max-age=0, must-revalidate",
})
self.assert_invalid(mutate, "cannot nest another policy under immutable /releases")
def test_public_cache_is_rejected_on_protected_route(self): def test_public_cache_is_rejected_on_protected_route(self):
self.assert_invalid( self.assert_invalid(
lambda raw: raw["artifacts"][0]["cache"]["rules"][0].__setitem__( lambda raw: raw["artifacts"][0]["cache"]["rules"][0].__setitem__(
@@ -190,56 +116,6 @@ class ConfigContractTests(unittest.TestCase):
"unknown fields: storage_bucket", "unknown fields: storage_bucket",
) )
def test_backend_authority_and_endpoint_are_derived(self):
for field, value in (
("endpoint", "https://attacker.example"),
("website_authority", "internal-api.default.svc.k8s.sjc001.fritzlab.net"),
):
with self.subTest(field=field):
self.assert_invalid(
lambda raw, field=field, value=value: raw["artifacts"][0]["publish"].__setitem__(field, value),
f"unknown fields: {field}",
)
def test_publication_credentials_use_dedicated_matched_names(self):
self.assert_invalid(
lambda raw: raw["artifacts"][0]["publish"]["credentials"].__setitem__(
"access_key_env", "CI_BOT_TOKEN"
),
"must be a matched <NAME>_S3_ACCESS_KEY",
)
def test_resolved_build_inputs_must_be_pairwise_disjoint(self):
raw = copy.deepcopy(self.raw)
raw["artifacts"][1]["content_dir"] = ""
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "portal" / "build").mkdir(parents=True)
with self.assertRaisesRegex(ConfigError, "build inputs overlap after resolution"):
validate_artifact_inputs(root, cfg)
def test_descendant_symlink_cannot_cross_artifact_boundary(self):
cfg = normalize_site_config(self.raw, "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "dist").mkdir()
(root / "portal" / "build").mkdir(parents=True)
(root / "portal" / "build" / "private.txt").write_text("private")
(root / "dist" / "portal-link").symlink_to(root / "portal" / "build")
with self.assertRaisesRegex(ConfigError, "build input contains symlink"):
validate_artifact_inputs(root, cfg)
def test_artifact_root_symlink_is_rejected_before_resolution(self):
cfg = normalize_site_config(self.raw, "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "dist-real").mkdir()
(root / "dist").symlink_to(root / "dist-real")
(root / "portal" / "build").mkdir(parents=True)
with self.assertRaisesRegex(ConfigError, "content_dir contains symlink component"):
validate_artifact_inputs(root, cfg)
class GenerationTests(unittest.TestCase): class GenerationTests(unittest.TestCase):
def render(self, raw): def render(self, raw):
@@ -273,18 +149,6 @@ class GenerationTests(unittest.TestCase):
self.assertNotIn("passhostheader", files["manifests/ingress-portal.yaml"]) self.assertNotIn("passhostheader", files["manifests/ingress-portal.yaml"])
self.assertIn("baseline-dist.web.sjc001.fritzlab.net", files["manifests/service-distributions.yaml"]) self.assertIn("baseline-dist.web.sjc001.fritzlab.net", files["manifests/service-distributions.yaml"])
def test_history_keeps_yaml_ambiguous_artifact_names_as_strings(self):
raw = fixture("split-site.yaml")
raw["artifacts"][0]["name"] = "yes"
raw["routes"][0]["artifact"] = "yes"
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
contracts = deploy.next_route_contracts(cfg, {})
with tempfile.TemporaryDirectory() as tmp:
app_dir = Path(tmp)
deploy.write_route_contracts(app_dir, contracts)
restored = deploy.previous_route_contracts(app_dir)
self.assertEqual("yes", restored["baseline-portal"]["artifact"])
def test_generation_is_deterministic_when_input_lists_are_reversed(self): def test_generation_is_deterministic_when_input_lists_are_reversed(self):
raw = fixture("split-site.yaml") raw = fixture("split-site.yaml")
first_tmp, _, first = self.render(raw) first_tmp, _, first = self.render(raw)
@@ -308,14 +172,12 @@ class GenerationTests(unittest.TestCase):
self.assertFalse(stale.exists()) self.assertFalse(stale.exists())
def test_legacy_names_and_garage_s3_target_are_preserved(self): def test_legacy_names_and_garage_s3_target_are_preserved(self):
tmp, app_dir, files = self.render(fixture("legacy-site.yaml")) tmp, _, files = self.render(fixture("legacy-site.yaml"))
self.addCleanup(tmp.cleanup) self.addCleanup(tmp.cleanup)
self.assertIn("manifests/service.yaml", files) self.assertIn("manifests/service.yaml", files)
self.assertIn("manifests/ingress.yaml", files) self.assertIn("manifests/ingress.yaml", files)
self.assertIn("garage-s3.storage.svc.k8s.sjc001.fritzlab.net", files["manifests/service.yaml"]) self.assertIn("garage-s3.storage.svc.k8s.sjc001.fritzlab.net", files["manifests/service.yaml"])
self.assertNotIn("passhostheader", files["manifests/ingress.yaml"]) self.assertNotIn("passhostheader", files["manifests/ingress.yaml"])
self.assertNotIn("site-publish.fritzlab.net", files["manifests/ingress.yaml"])
self.assertEqual({}, deploy.previous_route_contracts(app_dir))
class BuildTests(unittest.TestCase): class BuildTests(unittest.TestCase):
@@ -353,56 +215,6 @@ class PublishingTests(unittest.TestCase):
self.assertNotIn(secret, " ".join(command)) self.assertNotIn(secret, " ".join(command))
self.assertNotIn(secret, kwargs.get("display", "")) self.assertNotIn(secret, kwargs.get("display", ""))
def test_askpass_authenticates_clone_and_push_round_trip(self):
token = "round-trip-token"
expected = "Basic " + base64.b64encode(f"ci-bot:{token}".encode()).decode()
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
bare = root / "repo.git"
seed = root / "seed"
checkout = root / "checkout"
subprocess.run(["git", "init", "--bare", "--initial-branch=main", str(bare)], check=True,
stdout=subprocess.DEVNULL)
subprocess.run(["git", "-C", str(bare), "config", "http.receivepack", "true"], check=True)
subprocess.run(["git", "init", "--initial-branch=main", str(seed)], check=True,
stdout=subprocess.DEVNULL)
subprocess.run(["git", "-C", str(seed), "config", "user.name", "Test"], check=True)
subprocess.run(["git", "-C", str(seed), "config", "user.email", "test@example.invalid"], check=True)
(seed / "README.md").write_text("seed\n")
subprocess.run(["git", "-C", str(seed), "add", "README.md"], check=True)
subprocess.run(["git", "-C", str(seed), "commit", "-m", "seed"], check=True,
stdout=subprocess.DEVNULL)
subprocess.run(["git", "-C", str(seed), "push", str(bare), "main"], check=True,
stdout=subprocess.DEVNULL)
handler = type("GitHandler", (AuthenticatedGitHandler,), {
"project_root": root, "expected_authorization": expected,
})
server = IPv6GitHTTPServer(("::1", 0), handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
url = f"http://ci-bot@[::1]:{server.server_port}/repo.git"
with patch.dict(os.environ, {"NO_PROXY": "::1,[::1]", "no_proxy": "::1,[::1]"}, clear=False):
subprocess.run(
["git", "clone", url, str(checkout)], env=utils.git_auth_env(token),
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
subprocess.run(["git", "-C", str(checkout), "config", "user.name", "Test"], check=True)
subprocess.run(["git", "-C", str(checkout), "config", "user.email", "test@example.invalid"],
check=True)
(checkout / "roundtrip.txt").write_text("authenticated\n")
utils.commit_and_push(checkout, "authenticated round trip", token)
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
result = subprocess.run(
["git", "--git-dir", str(bare), "show", "main:roundtrip.txt"],
check=True, text=True, capture_output=True,
)
self.assertEqual("authenticated\n", result.stdout)
def test_cache_headers_credentials_and_route_prefix_are_separate(self): def test_cache_headers_credentials_and_route_prefix_are_separate(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions") artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
@@ -414,186 +226,31 @@ class PublishingTests(unittest.TestCase):
(html / "channels").mkdir() (html / "channels").mkdir()
(html / "releases" / "1.0.js").write_text("release") (html / "releases" / "1.0.js").write_text("release")
(html / "channels" / "stable.json").write_text("channel") (html / "channels" / "stable.json").write_text("channel")
commands, events = [], [] commands = []
def capture(command, **kwargs): def capture(command, **kwargs):
commands.append((command, kwargs["env"])) commands.append((command, kwargs["env"]))
events.append("mutable")
def publish_immutable(*_args):
events.append("immutable")
secret = "secret-must-not-appear" secret = "secret-must-not-appear"
output = io.StringIO() output = io.StringIO()
with patch.dict(os.environ, { with patch.dict(os.environ, {
"DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret "DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": secret
}, clear=False), patch.object(deploy, "run", side_effect=capture), \ }, clear=False), patch.object(deploy, "run", side_effect=capture), redirect_stdout(output):
patch.object(deploy, "publish_immutable_rule", side_effect=publish_immutable) as immutable_publish, \
redirect_stdout(output):
deploy.publish_route_immutables(artifact, route, root)
deploy.s3_sync(artifact, route, root) deploy.s3_sync(artifact, route, root)
self.assertTrue(all(secret not in " ".join(command) for command, _ in commands)) self.assertTrue(all(secret not in " ".join(command) for command, _ in commands))
self.assertEqual("immutable", events[0])
self.assertNotIn(secret, output.getvalue()) self.assertNotIn(secret, output.getvalue())
self.assertTrue(all(call_env["AWS_ACCESS_KEY_ID"] == "dist-key" for _, call_env in commands)) self.assertTrue(all(call_env["AWS_ACCESS_KEY_ID"] == "dist-key" for _, call_env in commands))
self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands)) self.assertTrue(all("DIST_S3_SECRET_KEY" not in call_env for _, call_env in commands))
rendered = [" ".join(command) for command, _ in commands] rendered = [" ".join(command) for command, _ in commands]
self.assertIn("s3://baseline-dist/dist/", rendered[0]) self.assertIn("s3://baseline-dist/", rendered[0])
self.assertIn("releases/*", rendered[0])
self.assertNotIn("--delete", rendered[0])
self.assertIn("--delete", rendered[1])
self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:])) self.assertTrue(all("s3://baseline-dist/dist/" in command for command in rendered[1:]))
self.assertTrue(any("releases/" in command and
"public, max-age=31536000, immutable" in command
for command in rendered))
self.assertTrue(any("channels/" in command and self.assertTrue(any("channels/" in command and
"public, max-age=0, must-revalidate" in command "public, max-age=0, must-revalidate" in command
for command in rendered)) for command in rendered))
immutable_publish.assert_called_once()
self.assertEqual(
"public, max-age=31536000, immutable",
immutable_publish.call_args.args[2]["cache_control"],
)
def test_root_move_preserves_only_actual_retired_immutable_prefix(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
route = {**next(item for item in cfg["routes"] if item["artifact"] == "distributions"),
"path": "/"}
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
html = root / artifact["build_dir"]
(html / "releases").mkdir(parents=True)
(html / "channels").mkdir()
(html / "docs" / "releases").mkdir(parents=True)
(html / "channels" / "stable.json").write_text("channel")
(html / "docs" / "releases" / "index.html").write_text("mutable")
commands = []
with patch.dict(os.environ, {
"DIST_S3_ACCESS_KEY": "dist-key", "DIST_S3_SECRET_KEY": "dist-secret"
}, clear=False), patch.object(
deploy, "run", side_effect=lambda command, **_: commands.append(command)
):
deploy.s3_sync(artifact, route, root, previous_contract={
"path": "/foo", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["foo/releases"],
})
rendered = [" ".join(command) for command in commands]
self.assertTrue(all("foo/releases/*" in command for command in rendered[:2]))
self.assertTrue(all("*/releases/*" not in command for command in rendered))
def test_protected_bucket_cannot_become_public_across_deployments(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
previous = {
"baseline-dist": {
"path": "/dist", "access": "protected", "artifact": "old-name",
"immutable_prefixes": ["dist/releases"],
}
}
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(cfg, previous)
renamed = copy.deepcopy(cfg)
artifact = next(item for item in renamed["artifacts"] if item["name"] == "distributions")
artifact["name"] = "downloads"
route = next(item for item in renamed["routes"] if item["artifact"] == "distributions")
route["artifact"] = "downloads"
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(renamed, previous)
previous = {"retired-protected-bucket": previous["baseline-dist"]}
deploy.validate_route_migrations(cfg, previous)
def test_protected_split_bucket_cannot_become_legacy_public(self):
cfg = normalize_site_config(fixture("legacy-site.yaml"), "baseline.fritzlab.net")
previous = {
"baseline.fritzlab.net": {
"path": "/portal", "access": "protected", "artifact": "portal",
"immutable_prefixes": [],
}
}
with self.assertRaisesRegex(RuntimeError, "cannot become public while reusing protected"):
deploy.validate_route_migrations(cfg, previous)
def test_removed_immutable_rule_preserves_prior_keys(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
artifact = next(item for item in cfg["artifacts"] if item["name"] == "distributions")
artifact["cache_rules"] = [
rule for rule in artifact["cache_rules"] if rule["path"] != "releases"
]
route = next(item for item in cfg["routes"] if item["artifact"] == "distributions")
with tempfile.TemporaryDirectory() as tmp:
html = Path(tmp)
filters = deploy.retired_immutable_filters(artifact, route, html, {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["dist/releases"],
})
self.assertEqual(["--exclude", "releases/*"], filters)
previous = {
"baseline-dist": {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["dist/releases"],
}
}
next_contracts = deploy.next_route_contracts(cfg, previous)
self.assertEqual(
["dist/releases"], next_contracts["baseline-dist"]["immutable_prefixes"],
)
self.assertEqual(
["--exclude", "releases/*"],
deploy.retired_immutable_filters(
artifact, route, html, next_contracts["baseline-dist"],
),
)
(html / "releases").mkdir()
(html / "releases" / "replacement.js").write_text("mutable")
with self.assertRaisesRegex(RuntimeError, "collides with retired immutable"):
deploy.retired_immutable_filters(artifact, route, html, {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_prefixes": ["dist/releases"],
})
def test_removed_route_history_remains_append_only(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
previous = {
"retired-bucket": {
"path": "/retired", "access": "protected", "artifact": "retired",
"immutable_prefixes": ["retired/releases"],
}
}
contracts = deploy.next_route_contracts(cfg, previous)
self.assertEqual(previous["retired-bucket"], contracts["retired-bucket"])
def test_decommission_preserves_history_for_an_unpurged_bucket(self):
with tempfile.TemporaryDirectory() as tmp:
apps = Path(tmp)
site = apps / "sjc001/websites/baseline"
site.mkdir(parents=True)
history = b'{"schemaVersion":1,"buckets":{}}\n'
(site / deploy.HISTORY_FILE).write_bytes(history)
(site / "app.yaml").write_text("live\n")
with patch.object(deploy, "clone_apps", return_value=apps), patch.object(
deploy, "commit_and_push"
) as commit:
deploy.decommission("baseline", "token", ["baseline-dist"])
self.assertEqual(history, (site / deploy.HISTORY_FILE).read_bytes())
self.assertFalse((site / "app.yaml").exists())
commit.assert_called_once_with(apps, "Decommission baseline", "token")
def test_later_route_immutable_failure_stops_all_mutable_publication(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
with patch.object(deploy, "validate_publication_environment"), \
patch.object(deploy, "validate_artifact_output"), \
patch.object(deploy, "clone_apps", return_value=root / "apps"), patch.object(
deploy, "publish_route_immutables",
side_effect=[None, RuntimeError("immutable failed")],
) as immutable_publish, patch.object(deploy, "s3_sync") as mutable_sync, \
self.assertRaisesRegex(
RuntimeError, "immutable failed"
):
deploy.deploy_static("baseline", root, root, "token", cfg)
self.assertEqual(2, immutable_publish.call_count)
mutable_sync.assert_not_called()
def test_absent_artifact_is_detected_before_publish(self): def test_absent_artifact_is_detected_before_publish(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
@@ -612,87 +269,5 @@ class PublishingTests(unittest.TestCase):
deploy.validate_artifact_output(Path(tmp), artifact) deploy.validate_artifact_output(Path(tmp), artifact)
class ImmutablePublicationTests(unittest.TestCase):
CACHE = "public, max-age=31536000, immutable"
def result(self, returncode, stdout="", stderr=""):
return subprocess.CompletedProcess([], returncode, stdout, stderr)
def key_and_digests(self, source):
content_type = "text/javascript"
content_digest, publication_digest = deploy._immutable_digests(
source, self.CACHE, content_type,
)
return f"dist/releases/release-{publication_digest}.js", content_digest, publication_digest
def test_new_key_uses_content_address_and_digest_metadata(self):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "release.js"
source.write_text("fixed release")
key, content_digest, publication_digest = self.key_and_digests(source)
calls = []
def capture(args, _env):
calls.append(args)
return self.result(1, stderr="404 Not Found") if len(calls) == 1 else self.result(0, "{}")
with patch.object(deploy, "_aws_capture", side_effect=capture):
created = deploy.publish_immutable_file(
"http://garage-s3.storage.svc:3900", "dist-bucket",
key, source, self.CACHE, {},
)
self.assertTrue(created)
put = calls[1]
self.assertEqual("put-object", put[4])
self.assertNotIn("--if-none-match", put)
self.assertEqual(
f"sha256={content_digest},publication-sha256={publication_digest}",
put[put.index("--metadata") + 1],
)
def test_identical_retry_converges_without_put(self):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "release.js"
source.write_text("fixed release")
key, content_digest, publication_digest = self.key_and_digests(source)
head = json.dumps({
"Metadata": {"sha256": content_digest, "publication-sha256": publication_digest},
"CacheControl": self.CACHE,
"ContentType": "text/javascript",
})
with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)) as request:
created = deploy.publish_immutable_file(
"http://garage-s3.storage.svc:3900", "dist-bucket",
key, source, self.CACHE, {},
)
self.assertFalse(created)
self.assertEqual(1, request.call_count)
def test_changed_immutable_key_is_refused(self):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "release.js"
source.write_text("changed release")
key, _, _ = self.key_and_digests(source)
head = json.dumps({"Metadata": {"sha256": "different"}, "CacheControl": self.CACHE})
with patch.object(deploy, "_aws_capture", return_value=self.result(0, head)), \
self.assertRaisesRegex(RuntimeError, "immutable object differs"):
deploy.publish_immutable_file(
"http://garage-s3.storage.svc:3900", "dist-bucket",
key, source, self.CACHE, {},
)
def test_immutable_key_without_publication_digest_is_refused_before_s3(self):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "release.js"
source.write_text("fixed release")
with patch.object(deploy, "_aws_capture") as request, \
self.assertRaisesRegex(RuntimeError, "must contain its one publication SHA-256"):
deploy.publish_immutable_file(
"http://garage-s3.storage.svc:3900", "dist-bucket",
"dist/releases/release.js", source, self.CACHE, {},
)
request.assert_not_called()
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()