feat(site-publish): scope publication with an artifacts selection #7

Merged
dfritz merged 3 commits from artifacts-input into main 2026-09-06 21:31:51 +00:00
6 changed files with 352 additions and 9 deletions
+36
View File
@@ -234,10 +234,46 @@ my-site.fritzlab.net 300 IN CNAME gateway.sjc001.fritzlab.net.
| `garage-admin-token` | legacy aliases only | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) |
| `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint |
| `username` | no | `ci-bot` | Gitea username |
| `artifacts` | no | every declared artifact | Space- or comma-separated subset of `site.yaml` artifacts to build and publish |
Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`,
`GARAGE_S3_SECRET_KEY`, `GARAGE_ADMIN_TOKEN`.
### Publishing a subset of artifacts
A repository whose branches ship on different cadences names the ones this run
owns:
```yaml
- uses: https://code.fritzlab.net/action/site-publish@<sha>
with:
token: ${{ secrets.CI_BOT_TOKEN }}
artifacts: catalogue
```
Selection scopes the build and the S3 publication only. Ingresses, Services,
Certificates and the immutable-path history are always rendered from the whole
`site.yaml`, so a scoped run cannot retire another artifact's route or delete
its bucket contents. Credentials are resolved for the selected artifacts alone,
so a workflow need not carry secrets for artifacts it does not publish.
What a scoped run may **not** do is move an artifact it is not publishing. The
generated route contract in `site-publish-history.yaml` is written from the
whole `site.yaml`, and `access` there is a replacement rather than a union — so
recording a change nothing published would turn intent into a fact that
`validate_route_migrations` then refuses to undo. A run whose `site.yaml`
changes an unselected artifact's route path, access or artifact name is refused
before the first bucket is touched, naming both contracts: publish that artifact
in the same run. For the same reason a scoped run cannot introduce an artifact
that has no published history yet.
Two more refusals, both before any bucket changes: an undeclared name, and
`enabled: false` with a selection, because decommissioning is whole-site.
One consequence to know: CORS reconciliation is scoped too, since a scoped run
holds no credential for the other bucket. A `cors_origins:` change lands with
that artifact's next publish, not on the merge that edits `site.yaml`.
## Tools
- **`new-site.sh`** — create a new site: Gitea repo, Garage bucket, web hosting enabled.
+9
View File
@@ -28,6 +28,13 @@ inputs:
description: Gitea username for git operations
required: false
default: ci-bot
artifacts:
# Scopes building and publishing only. Routes, Ingresses and the immutable
# -path history are always rendered from the whole site.yaml, so a scoped
# run never retires another artifact's route.
description: Space- or comma-separated subset of site.yaml artifacts to build and publish (default is every declared artifact)
required: false
default: ''
runs:
using: composite
steps:
@@ -44,6 +51,7 @@ runs:
ACTION_DIR: ${{ github.action_path }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
CI_BOT_USER: ${{ inputs.username }}
SITE_ARTIFACTS: ${{ inputs.artifacts }}
- name: Deploy
shell: bash
@@ -61,3 +69,4 @@ runs:
GARAGE_ADMIN_ENDPOINT: ${{ inputs.garage-admin-endpoint }}
GARAGE_ADMIN_TOKEN: ${{ inputs.garage-admin-token }}
GITHUB_RUN_NUMBER: ${{ github.run_number }}
SITE_ARTIFACTS: ${{ inputs.artifacts }}
+9 -2
View File
@@ -5,7 +5,14 @@ import subprocess
import tempfile
from pathlib import Path
from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs
from utils import (
EXCLUDE_FILES,
env,
parse_site_yaml,
run,
selected_artifacts,
validate_artifact_inputs,
)
def build_artifact(site_dir, artifact):
@@ -66,5 +73,5 @@ def cmd_build():
validate_artifact_inputs(site_dir, cfg)
for artifact in cfg["artifacts"]:
for artifact in selected_artifacts(cfg):
build_artifact(site_dir, artifact)
+52 -6
View File
@@ -25,6 +25,8 @@ from utils import (
parse_site_yaml,
render_templates,
run,
selected_artifacts,
selected_routes,
validate_artifact_inputs,
)
@@ -47,8 +49,8 @@ def validate_artifact_output(site_dir, artifact):
def validate_publication_environment(cfg):
"""Resolve every declared credential before the first bucket is changed."""
for artifact in cfg["artifacts"]:
"""Resolve every credential this run needs before the first bucket is changed."""
for artifact in selected_artifacts(cfg):
env(artifact["credentials"]["access_key_env"])
env(artifact["credentials"]["secret_key_env"])
if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
@@ -533,31 +535,73 @@ def validate_route_migrations(cfg, previous_contracts):
)
def validate_scoped_history(cfg, previous_contracts):
"""A scoped run may not record a route contract it did not publish.
render_site_manifests advances the stored contract for every route in
site.yaml, and `access` is overwritten rather than unioned the way
immutable_paths is. Without this, a catalogue-only publish could write a
protected access for the distributions bucket that nothing published, and
validate_route_migrations would then refuse to put that bucket back —
unpublished intent turned into an irreversible fact.
"""
chosen = set(cfg["selected"])
if len(chosen) == len(cfg["artifacts"]):
return
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
for route in cfg["routes"]:
if route["artifact"] in chosen:
continue
artifact = artifact_by_name[route["artifact"]]
previous = previous_contracts.get(artifact["bucket"])
if previous is None:
raise RuntimeError(
f"a scoped run cannot introduce artifact {route['artifact']}; "
f"publish it in the same run"
)
current = {
"path": route["path"],
"access": "public" if route["access"] == "legacy" else route["access"],
"artifact": route["artifact"],
}
recorded = {key: previous[key] for key in current}
if recorded != current:
raise RuntimeError(
f"a scoped run may not change unselected artifact {route['artifact']}'s route "
f"contract ({recorded} -> {current}); publish it in the same run"
)
def deploy_static(site_name, site_dir, action_dir, token, cfg):
artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
credential_env_names = {
name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
}
# Publication is scoped to the selected artifacts; the rendered route
# contract is not. Manifests and immutable-path history stay whole, so a
# partial publish can never retire another artifact's route or bucket.
publishing = selected_routes(cfg)
validate_publication_environment(cfg)
for artifact in cfg["artifacts"]:
for artifact in selected_artifacts(cfg):
validate_artifact_output(site_dir, artifact)
apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name
manifests_dir = app_dir / "manifests"
previous_contracts = previous_route_contracts(app_dir)
validate_route_migrations(cfg, previous_contracts)
validate_scoped_history(cfg, previous_contracts)
# Complete immutable work across the whole publication before any route's
# mutable pointers can change. Partial immutable success is safe; mixing a
# new route with an old route after a later immutable failure is not.
for route in cfg["routes"]:
for route in publishing:
publish_route_immutables(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
)
# Reconcile every browser-read policy before publishing mutable content.
# A CORS failure therefore cannot leave a new channel pointing at a release
# whose cross-origin assets browsers cannot consume.
reconcile_artifact_cors(cfg["artifacts"], credential_env_names)
for route in cfg["routes"]:
reconcile_artifact_cors(selected_artifacts(cfg), credential_env_names)
for route in publishing:
s3_sync(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]),
@@ -596,6 +640,8 @@ def cmd_deploy():
cfg = parse_site_yaml(site_dir)
if not cfg["enabled"]:
if len(cfg["selected"]) != len(cfg["artifacts"]):
die("a disabled site decommissions whole; drop the artifacts selection")
print("Site disabled — running decommission...")
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
return
+31
View File
@@ -274,6 +274,7 @@ def _legacy_config(raw, site_name):
"aliases": _aliases(raw.get("aliases"), raw["domain"]),
"enabled": raw.get("enabled", True),
"artifacts": [artifact],
"selected": ["site"],
"routes": [{
"name": "site", "path": "/", "artifact": "site", "access": "legacy",
"access_middleware": None,
@@ -457,6 +458,33 @@ def _validate_multi(cfg):
raise ConfigError("a public '/' catch-all would expose unmatched protected content")
def _artifact_selection(cfg, requested):
"""Resolve the subset of declared artifacts this run publishes."""
declared = [artifact["name"] for artifact in cfg["artifacts"]]
if not requested or not requested.strip():
return declared
names = [part for part in re.split(r"[,\s]+", requested.strip()) if part]
unknown = sorted({name for name in names if name not in declared})
if unknown:
raise ConfigError(
"artifacts selects undeclared artifact(s) " + ", ".join(unknown)
+ "; site.yaml declares " + ", ".join(declared)
)
return [name for name in declared if name in set(names)]
def selected_artifacts(cfg):
"""Artifacts this run builds and publishes, in declaration order."""
chosen = set(cfg["selected"])
return [artifact for artifact in cfg["artifacts"] if artifact["name"] in chosen]
def selected_routes(cfg):
"""Routes whose artifact this run publishes, in route order."""
chosen = set(cfg["selected"])
return [route for route in cfg["routes"] if route["artifact"] in chosen]
def normalize_site_config(raw, site_name):
raw = _mapping(raw, "site.yaml")
domain = _hostname(raw.get("domain"), "domain")
@@ -482,6 +510,7 @@ def normalize_site_config(raw, site_name):
"routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])),
}
_validate_multi(cfg)
cfg["selected"] = [artifact["name"] for artifact in cfg["artifacts"]]
for route in cfg["routes"]:
if len(f"{k8s_name(site_name)}-{route['name']}") > 63:
raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters")
@@ -534,12 +563,14 @@ def parse_site_yaml(site_dir, site_name=None):
try:
with open(path) as stream:
cfg = normalize_site_config(yaml.safe_load(stream), site_name)
cfg["selected"] = _artifact_selection(cfg, os.environ.get("SITE_ARTIFACTS"))
except (ConfigError, yaml.YAMLError) as exc:
die(str(exc))
print("Site config:")
print(f" domain: {cfg['domain']}")
print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}")
print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}")
print(f" publishing: {cfg['selected']}")
print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}")
return cfg
+215 -1
View File
@@ -24,7 +24,12 @@ sys.path.insert(0, str(ROOT / "scripts"))
import deploy
import build
import utils
from utils import ConfigError, normalize_site_config, validate_artifact_inputs
from utils import (
ConfigError,
normalize_site_config,
parse_site_yaml,
validate_artifact_inputs,
)
def fixture(name):
@@ -404,6 +409,215 @@ class BuildTests(unittest.TestCase):
)
class ArtifactSelectionTests(unittest.TestCase):
"""The `artifacts` input scopes publication, never the route contract."""
def parse(self, selection):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "site.yaml").write_text(
(ROOT / "tests" / "fixtures" / "split-site.yaml").read_text()
)
environment = {"SITE_REPO": "fritzlab/baseline"}
if selection is not None:
environment["SITE_ARTIFACTS"] = selection
with patch.dict(os.environ, environment, clear=False), \
redirect_stdout(io.StringIO()):
if selection is None:
os.environ.pop("SITE_ARTIFACTS", None)
return parse_site_yaml(root)
def test_absent_selection_publishes_every_declared_artifact(self):
self.assertEqual(["distributions", "portal"], self.parse(None)["selected"])
self.assertEqual(["distributions", "portal"], self.parse(" ")["selected"])
def test_selection_accepts_comma_and_whitespace_separated_names(self):
self.assertEqual(["portal"], self.parse("portal")["selected"])
self.assertEqual(
["distributions", "portal"], self.parse("portal, distributions")["selected"]
)
self.assertEqual(
["distributions", "portal"], self.parse("portal distributions")["selected"]
)
def test_undeclared_selection_fails_before_any_publication(self):
stderr = io.StringIO()
with redirect_stderr(stderr), self.assertRaises(SystemExit):
self.parse("catalogue")
self.assertIn("undeclared artifact", stderr.getvalue())
self.assertIn("catalogue", stderr.getvalue())
def test_build_skips_unselected_artifacts(self):
raw = fixture("split-site.yaml")
for artifact in raw["artifacts"]:
artifact["tidy"] = False
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
cfg["selected"] = ["portal"]
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "portal" / "build").mkdir(parents=True)
(root / "portal" / "build" / "index.html").write_text("portal")
built = []
with patch.object(build, "build_artifact",
side_effect=lambda _root, artifact: built.append(artifact["name"])), \
patch.dict(os.environ, {"SITE_DIR": str(root)}, clear=False), \
patch.object(build, "parse_site_yaml", return_value=cfg), \
redirect_stdout(io.StringIO()):
build.cmd_build()
self.assertEqual(["portal"], built)
def deploy_selection(self, selection, environment):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
cfg["selected"] = selection
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
apps_dir = root / "apps"
app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net"
app_dir.mkdir(parents=True)
# A scoped run refuses an artifact it has no published history for, so
# the fixture starts from the whole publication this one narrows.
(app_dir / "site-publish-history.yaml").write_text(yaml.safe_dump({
"version": 1,
"buckets": {
artifact["bucket"]: {
"path": route["path"],
"access": "public" if route["access"] == "legacy" else route["access"],
"artifact": route["artifact"],
"immutable_paths": deploy.immutable_key_prefixes(artifact, route),
}
for route in cfg["routes"]
for artifact in [next(item for item in cfg["artifacts"]
if item["name"] == route["artifact"])]
},
}, sort_keys=True))
for artifact in cfg["artifacts"]:
if artifact["name"] not in set(selection):
continue
html = root / artifact["build_dir"]
html.mkdir(parents=True)
(html / "index.html").write_text(artifact["name"])
for rule in artifact["cache_rules"]:
if rule["path"]:
(html / rule["path"]).mkdir(parents=True, exist_ok=True)
(html / rule["path"] / "keep.json").write_text("{}")
synced, immutables, cors = [], [], []
with patch.dict(os.environ, environment, clear=False), \
patch.object(deploy, "clone_apps", return_value=apps_dir), \
patch.object(deploy, "commit_and_push"), \
patch.object(deploy, "publish_route_immutables",
side_effect=lambda artifact, *_a, **_k: immutables.append(artifact["name"])), \
patch.object(deploy, "reconcile_artifact_cors",
side_effect=lambda artifacts, *_a: cors.extend(item["name"] for item in artifacts)), \
patch.object(deploy, "s3_sync",
side_effect=lambda artifact, *_a, **_k: synced.append(artifact["name"])), \
redirect_stdout(io.StringIO()):
deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg)
files = {path.relative_to(app_dir).as_posix(): path.read_text()
for path in sorted(app_dir.rglob("*.yaml"))}
return {"synced": synced, "immutables": immutables, "cors": cors, "files": files}
def test_selected_publication_leaves_the_whole_route_contract_rendered(self):
# Only the portal credentials exist: a scoped run must not demand the
# secrets of an artifact it is not publishing.
environment = {"PORTAL_S3_ACCESS_KEY": "portal-key",
"PORTAL_S3_SECRET_KEY": "portal-secret"}
for name in ("DIST_S3_ACCESS_KEY", "DIST_S3_SECRET_KEY"):
os.environ.pop(name, None)
result = self.deploy_selection(["portal"], environment)
self.assertEqual(["portal"], result["synced"])
self.assertEqual(["portal"], result["immutables"])
self.assertEqual(["portal"], result["cors"])
self.assertIn("manifests/ingress-distributions.yaml", result["files"])
self.assertIn("manifests/service-distributions.yaml", result["files"])
history = yaml.safe_load(result["files"]["site-publish-history.yaml"])
self.assertEqual({"baseline-dist", "baseline-portal"}, set(history["buckets"]))
self.assertEqual(
["dist/releases"], history["buckets"]["baseline-dist"]["immutable_paths"]
)
def test_scoped_run_refuses_to_record_a_contract_it_did_not_publish(self):
"""The reviewer's scenario: distributions goes protected in site.yaml while
a catalogue-only run publishes. Recording that access would make the
bucket unreturnable through validate_route_migrations."""
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
published = {
"baseline-portal": {
"path": "/", "access": "protected", "artifact": "portal", "immutable_paths": [],
},
"baseline-dist": {
"path": "/dist", "access": "public", "artifact": "distributions",
"immutable_paths": ["dist/releases"],
},
}
# A whole publish agrees with what is recorded, scoped or not.
deploy.validate_scoped_history(cfg, published)
cfg["selected"] = ["portal"]
deploy.validate_scoped_history(cfg, published)
moved = copy.deepcopy(cfg)
route = next(item for item in moved["routes"] if item["artifact"] == "distributions")
route["access"] = "protected"
route["access_middleware"] = "authentik-forwardauth"
with self.assertRaises(RuntimeError) as refused:
deploy.validate_scoped_history(moved, published)
self.assertIn("may not change unselected artifact distributions", str(refused.exception))
# The same change published in the same run is allowed to proceed.
moved["selected"] = ["distributions", "portal"]
deploy.validate_scoped_history(moved, published)
def test_scoped_run_refuses_an_artifact_with_no_published_history(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
cfg["selected"] = ["portal"]
with self.assertRaises(RuntimeError) as refused:
deploy.validate_scoped_history(cfg, {})
self.assertIn("cannot introduce artifact distributions", str(refused.exception))
def test_scoped_deploy_runs_the_history_guard_before_touching_a_bucket(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
cfg["selected"] = ["portal"]
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
apps_dir = root / "apps"
app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net"
app_dir.mkdir(parents=True)
html = root / next(
item for item in cfg["artifacts"] if item["name"] == "portal"
)["build_dir"]
html.mkdir(parents=True)
(html / "index.html").write_text("portal")
with patch.dict(os.environ, {
"PORTAL_S3_ACCESS_KEY": "portal-key", "PORTAL_S3_SECRET_KEY": "portal-secret",
}, clear=False), \
patch.object(deploy, "clone_apps", return_value=apps_dir), \
patch.object(deploy, "commit_and_push"), \
patch.object(deploy, "publish_route_immutables") as immutables, \
patch.object(deploy, "reconcile_artifact_cors") as cors, \
patch.object(deploy, "s3_sync") as sync, \
redirect_stdout(io.StringIO()), \
self.assertRaises(RuntimeError):
deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg)
immutables.assert_not_called()
cors.assert_not_called()
sync.assert_not_called()
def test_disabled_site_refuses_a_partial_selection(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
cfg["enabled"] = False
cfg["selected"] = ["portal"]
with patch.object(deploy, "parse_site_yaml", return_value=cfg), \
patch.dict(os.environ, {
"SITE_REPO": "fritzlab/baseline.fritzlab.net", "SITE_DIR": ".",
"ACTION_DIR": str(ROOT), "CI_BOT_TOKEN": "token",
}, clear=False), \
patch.object(deploy, "decommission") as decommission, \
redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \
self.assertRaises(SystemExit):
deploy.cmd_deploy()
decommission.assert_not_called()
class PublishingTests(unittest.TestCase):
def test_cors_policy_is_reconciled_as_read_only_browser_access(self):
captured = {}