fix(site-publish): retain immutable history across deploys
Authored-By: @architect <architect@fritzlab.net>
This commit is contained in:
Evelyn Chen
+103
-57
@@ -8,12 +8,10 @@ import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import yaml
|
||||
|
||||
from utils import (
|
||||
NAMESPACE,
|
||||
clone_apps,
|
||||
@@ -30,6 +28,7 @@ from utils import (
|
||||
GARAGE_ADMIN_ENDPOINT = os.environ.get(
|
||||
"GARAGE_ADMIN_ENDPOINT", "http://garage.storage.svc:3903"
|
||||
)
|
||||
HISTORY_FILE = "site-publish-history.json"
|
||||
|
||||
|
||||
def validate_artifact_output(site_dir, artifact):
|
||||
@@ -179,35 +178,38 @@ def publish_route_immutables(artifact, route, site_dir, credential_env_names=Non
|
||||
|
||||
|
||||
def retired_immutable_filters(artifact, route, html_dir, previous_contract):
|
||||
"""Protect immutable keys only when an old route falls inside the new scope."""
|
||||
"""Protect every historical immutable prefix inside the current sync scope."""
|
||||
if not previous_contract:
|
||||
return []
|
||||
current_prefix = route["path"].strip("/")
|
||||
previous_prefix = previous_contract["path"].strip("/")
|
||||
if current_prefix:
|
||||
marker = f"{current_prefix}/"
|
||||
if previous_prefix == current_prefix:
|
||||
previous_prefix = ""
|
||||
elif not previous_prefix.startswith(marker):
|
||||
return []
|
||||
else:
|
||||
previous_prefix = previous_prefix[len(marker):]
|
||||
filters = []
|
||||
current_immutable = {
|
||||
rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)
|
||||
}
|
||||
for immutable_path in previous_contract["immutable_paths"]:
|
||||
retired_path = "/".join(part for part in (previous_prefix, immutable_path) if part)
|
||||
collision = html_dir / retired_path
|
||||
if (immutable_path not in current_immutable and collision.exists()
|
||||
current_immutable = set(immutable_prefixes(artifact, route))
|
||||
for immutable_prefix in previous_contract["immutable_prefixes"]:
|
||||
if current_prefix:
|
||||
marker = f"{current_prefix}/"
|
||||
if not immutable_prefix.startswith(marker):
|
||||
continue
|
||||
relative_path = immutable_prefix[len(marker):]
|
||||
else:
|
||||
relative_path = immutable_prefix
|
||||
collision = html_dir / relative_path
|
||||
if (immutable_prefix not in current_immutable and collision.exists()
|
||||
and any(path.is_file() for path in collision.rglob("*"))):
|
||||
raise RuntimeError(
|
||||
f"current artifact collides with retired immutable partition: {retired_path}"
|
||||
f"current artifact collides with retired immutable partition: {relative_path}"
|
||||
)
|
||||
filters.extend(("--exclude", f"{retired_path}/*"))
|
||||
filters.extend(("--exclude", f"{relative_path}/*"))
|
||||
return filters
|
||||
|
||||
|
||||
def immutable_prefixes(artifact, route):
|
||||
route_prefix = route["path"].strip("/")
|
||||
return sorted({
|
||||
"/".join(part for part in (route_prefix, rule["path"]) if part)
|
||||
for rule in artifact["cache_rules"] if _is_immutable(rule)
|
||||
})
|
||||
|
||||
|
||||
def s3_sync(artifact, route, site_dir, credential_env_names=None, previous_contract=None):
|
||||
endpoint = artifact["s3_endpoint"]
|
||||
html_dir = site_dir / artifact["build_dir"]
|
||||
@@ -317,13 +319,7 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
|
||||
for route in cfg["routes"]:
|
||||
artifact = artifact_by_name[route["artifact"]]
|
||||
resource_name = k8s_name(site_name) if cfg["compatibility"] else f"{k8s_name(site_name)}-{route['name']}"
|
||||
immutable_paths = [
|
||||
rule["path"] for rule in artifact["cache_rules"] if _is_immutable(rule)
|
||||
]
|
||||
routes.append({
|
||||
**route, "resource_name": resource_name, "artifact_config": artifact,
|
||||
"immutable_paths_json": json.dumps(immutable_paths, separators=(",", ":")),
|
||||
})
|
||||
routes.append({**route, "resource_name": resource_name, "artifact_config": artifact})
|
||||
template_vars = {
|
||||
"site": site_name,
|
||||
"site_k8s": k8s_name(site_name),
|
||||
@@ -337,40 +333,84 @@ def render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg):
|
||||
|
||||
|
||||
def previous_route_contracts(app_dir):
|
||||
"""Read bucket-keyed route history from generated Ingresses."""
|
||||
contracts = {}
|
||||
manifests = app_dir / "manifests"
|
||||
if not manifests.exists():
|
||||
return contracts
|
||||
for path in sorted(manifests.glob("ingress*.yaml")):
|
||||
document = yaml.safe_load(path.read_text()) or {}
|
||||
annotations = document.get("metadata", {}).get("annotations", {})
|
||||
artifact = annotations.get("site-publish.fritzlab.net/artifact")
|
||||
access = annotations.get("site-publish.fritzlab.net/access")
|
||||
bucket = annotations.get("site-publish.fritzlab.net/bucket")
|
||||
immutable_paths_json = annotations.get("site-publish.fritzlab.net/immutable-paths")
|
||||
route_path = annotations.get("site-publish.fritzlab.net/route-path")
|
||||
values = (artifact, access, bucket, immutable_paths_json, route_path)
|
||||
if all(value is None for value in values):
|
||||
continue
|
||||
if (not all(isinstance(value, str) for value in values)
|
||||
or access not in {"public", "protected"} or not route_path.startswith("/")):
|
||||
"""Read the append-only bucket history kept beside generated manifests."""
|
||||
path = app_dir / HISTORY_FILE
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
document = json.loads(path.read_text())
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"invalid site-publish route history in {path}") from exc
|
||||
if (not isinstance(document, dict) or set(document) != {"schemaVersion", "buckets"}
|
||||
or document["schemaVersion"] != 1 or not isinstance(document["buckets"], dict)):
|
||||
raise RuntimeError(f"invalid site-publish route history in {path}")
|
||||
for bucket, contract in document["buckets"].items():
|
||||
if (not isinstance(bucket, str) or not isinstance(contract, dict)
|
||||
or set(contract) != {"access", "artifact", "immutablePrefixes", "routePath"}
|
||||
or contract["access"] not in {"legacy", "protected", "public"}
|
||||
or not isinstance(contract["artifact"], str)
|
||||
or not isinstance(contract["routePath"], str)
|
||||
or not contract["routePath"].startswith("/")
|
||||
or not isinstance(contract["immutablePrefixes"], list)
|
||||
or any(not _valid_immutable_prefix(value)
|
||||
for value in contract["immutablePrefixes"])
|
||||
or len(contract["immutablePrefixes"]) != len(set(contract["immutablePrefixes"]))):
|
||||
raise RuntimeError(f"invalid site-publish route history in {path}")
|
||||
try:
|
||||
immutable_paths = json.loads(immutable_paths_json)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"invalid site-publish route history in {path}") from exc
|
||||
if not isinstance(immutable_paths, list) or any(not isinstance(item, str) for item in immutable_paths):
|
||||
raise RuntimeError(f"invalid site-publish route history in {path}")
|
||||
if bucket in contracts:
|
||||
raise RuntimeError(f"duplicate site-publish route history for bucket {bucket}")
|
||||
return {
|
||||
bucket: {
|
||||
"access": contract["access"],
|
||||
"artifact": contract["artifact"],
|
||||
"immutable_prefixes": sorted(contract["immutablePrefixes"]),
|
||||
"path": contract["routePath"],
|
||||
}
|
||||
for bucket, contract in document["buckets"].items()
|
||||
}
|
||||
|
||||
|
||||
def next_route_contracts(cfg, previous_contracts):
|
||||
"""Carry protected access and immutable prefixes forward for every known bucket."""
|
||||
contracts = json.loads(json.dumps(previous_contracts))
|
||||
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
||||
for route in cfg["routes"]:
|
||||
artifact = artifacts[route["artifact"]]
|
||||
bucket = artifact["bucket"]
|
||||
previous = previous_contracts.get(bucket)
|
||||
access = "protected" if (
|
||||
route["access"] == "protected" or previous and previous["access"] == "protected"
|
||||
) else route["access"]
|
||||
contracts[bucket] = {
|
||||
"path": route_path, "access": access, "artifact": artifact,
|
||||
"immutable_paths": immutable_paths,
|
||||
"access": access,
|
||||
"artifact": route["artifact"],
|
||||
"immutable_prefixes": sorted(set(
|
||||
(previous or {}).get("immutable_prefixes", []) + immutable_prefixes(artifact, route)
|
||||
)),
|
||||
"path": route["path"],
|
||||
}
|
||||
return contracts
|
||||
|
||||
|
||||
def write_route_contracts(app_dir, contracts):
|
||||
app_dir.mkdir(parents=True, exist_ok=True)
|
||||
document = {
|
||||
"schemaVersion": 1,
|
||||
"buckets": {
|
||||
bucket: {
|
||||
"access": contract["access"],
|
||||
"artifact": contract["artifact"],
|
||||
"immutablePrefixes": contract["immutable_prefixes"],
|
||||
"routePath": contract["path"],
|
||||
}
|
||||
for bucket, contract in sorted(contracts.items())
|
||||
},
|
||||
}
|
||||
(app_dir / HISTORY_FILE).write_text(f"{json.dumps(document, indent=2, sort_keys=True)}\n")
|
||||
|
||||
|
||||
def _valid_immutable_prefix(value):
|
||||
return (isinstance(value, str) and value and not value.startswith("/")
|
||||
and not value.endswith("/") and ".." not in PurePosixPath(value).parts)
|
||||
|
||||
|
||||
def validate_route_migrations(cfg, previous_contracts):
|
||||
artifacts = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
|
||||
for route in cfg["routes"]:
|
||||
@@ -413,6 +453,7 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
|
||||
if cfg["compatibility"]:
|
||||
ensure_bucket_aliases(site_name, cfg["aliases"], os.environ.get("GARAGE_ADMIN_TOKEN"))
|
||||
|
||||
write_route_contracts(app_dir, next_route_contracts(cfg, previous_contracts))
|
||||
render_site_manifests(site_name, action_dir, app_dir, manifests_dir, cfg)
|
||||
|
||||
commit_and_push(apps_dir, f"Deploy {site_name}", token)
|
||||
@@ -425,7 +466,12 @@ def decommission(site_name, token, buckets=None):
|
||||
if not site_path.exists():
|
||||
print(f"No manifests for {site_name} — nothing to remove")
|
||||
return
|
||||
history_path = site_path / HISTORY_FILE
|
||||
history = history_path.read_bytes() if history_path.exists() else None
|
||||
shutil.rmtree(site_path)
|
||||
if history is not None:
|
||||
site_path.mkdir(parents=True)
|
||||
(site_path / HISTORY_FILE).write_bytes(history)
|
||||
commit_and_push(apps_dir, f"Decommission {site_name}", token)
|
||||
for bucket in buckets or [site_name]:
|
||||
print(f"Bucket {bucket} and its objects are NOT purged automatically.")
|
||||
|
||||
Reference in New Issue
Block a user