diff --git a/README.md b/README.md index cf534f8..fe0a54f 100644 --- a/README.md +++ b/README.md @@ -234,10 +234,46 @@ my-site.fritzlab.net 300 IN CNAME gateway.sjc001.fritzlab.net. | `garage-admin-token` | legacy aliases only | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) | | `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint | | `username` | no | `ci-bot` | Gitea username | +| `artifacts` | no | every declared artifact | Space- or comma-separated subset of `site.yaml` artifacts to build and publish | Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`, `GARAGE_S3_SECRET_KEY`, `GARAGE_ADMIN_TOKEN`. +### Publishing a subset of artifacts + +A repository whose branches ship on different cadences names the ones this run +owns: + +```yaml +- uses: https://code.fritzlab.net/action/site-publish@ + with: + token: ${{ secrets.CI_BOT_TOKEN }} + artifacts: catalogue +``` + +Selection scopes the build and the S3 publication only. Ingresses, Services, +Certificates and the immutable-path history are always rendered from the whole +`site.yaml`, so a scoped run cannot retire another artifact's route or delete +its bucket contents. Credentials are resolved for the selected artifacts alone, +so a workflow need not carry secrets for artifacts it does not publish. + +What a scoped run may **not** do is move an artifact it is not publishing. The +generated route contract in `site-publish-history.yaml` is written from the +whole `site.yaml`, and `access` there is a replacement rather than a union — so +recording a change nothing published would turn intent into a fact that +`validate_route_migrations` then refuses to undo. A run whose `site.yaml` +changes an unselected artifact's route path, access or artifact name is refused +before the first bucket is touched, naming both contracts: publish that artifact +in the same run. For the same reason a scoped run cannot introduce an artifact +that has no published history yet. + +Two more refusals, both before any bucket changes: an undeclared name, and +`enabled: false` with a selection, because decommissioning is whole-site. + +One consequence to know: CORS reconciliation is scoped too, since a scoped run +holds no credential for the other bucket. A `cors_origins:` change lands with +that artifact's next publish, not on the merge that edits `site.yaml`. + ## Tools - **`new-site.sh`** — create a new site: Gitea repo, Garage bucket, web hosting enabled. diff --git a/action.yaml b/action.yaml index 7d287b2..0ffb29b 100644 --- a/action.yaml +++ b/action.yaml @@ -28,6 +28,13 @@ inputs: description: Gitea username for git operations required: false default: ci-bot + artifacts: + # Scopes building and publishing only. Routes, Ingresses and the immutable + # -path history are always rendered from the whole site.yaml, so a scoped + # run never retires another artifact's route. + description: Space- or comma-separated subset of site.yaml artifacts to build and publish (default is every declared artifact) + required: false + default: '' runs: using: composite steps: @@ -44,6 +51,7 @@ runs: ACTION_DIR: ${{ github.action_path }} GITHUB_RUN_NUMBER: ${{ github.run_number }} CI_BOT_USER: ${{ inputs.username }} + SITE_ARTIFACTS: ${{ inputs.artifacts }} - name: Deploy shell: bash @@ -61,3 +69,4 @@ runs: GARAGE_ADMIN_ENDPOINT: ${{ inputs.garage-admin-endpoint }} GARAGE_ADMIN_TOKEN: ${{ inputs.garage-admin-token }} GITHUB_RUN_NUMBER: ${{ github.run_number }} + SITE_ARTIFACTS: ${{ inputs.artifacts }} diff --git a/scripts/build.py b/scripts/build.py index 6530e28..524f1e9 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -5,7 +5,14 @@ import subprocess import tempfile from pathlib import Path -from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs +from utils import ( + EXCLUDE_FILES, + env, + parse_site_yaml, + run, + selected_artifacts, + validate_artifact_inputs, +) def build_artifact(site_dir, artifact): @@ -66,5 +73,5 @@ def cmd_build(): validate_artifact_inputs(site_dir, cfg) - for artifact in cfg["artifacts"]: + for artifact in selected_artifacts(cfg): build_artifact(site_dir, artifact) diff --git a/scripts/deploy.py b/scripts/deploy.py index 45e25bb..e3a0eb5 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -25,6 +25,8 @@ from utils import ( parse_site_yaml, render_templates, run, + selected_artifacts, + selected_routes, validate_artifact_inputs, ) @@ -47,8 +49,8 @@ def validate_artifact_output(site_dir, artifact): def validate_publication_environment(cfg): - """Resolve every declared credential before the first bucket is changed.""" - for artifact in cfg["artifacts"]: + """Resolve every credential this run needs before the first bucket is changed.""" + for artifact in selected_artifacts(cfg): env(artifact["credentials"]["access_key_env"]) env(artifact["credentials"]["secret_key_env"]) if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"): @@ -533,31 +535,73 @@ def validate_route_migrations(cfg, previous_contracts): ) +def validate_scoped_history(cfg, previous_contracts): + """A scoped run may not record a route contract it did not publish. + + render_site_manifests advances the stored contract for every route in + site.yaml, and `access` is overwritten rather than unioned the way + immutable_paths is. Without this, a catalogue-only publish could write a + protected access for the distributions bucket that nothing published, and + validate_route_migrations would then refuse to put that bucket back — + unpublished intent turned into an irreversible fact. + """ + chosen = set(cfg["selected"]) + if len(chosen) == len(cfg["artifacts"]): + return + artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} + for route in cfg["routes"]: + if route["artifact"] in chosen: + continue + artifact = artifact_by_name[route["artifact"]] + previous = previous_contracts.get(artifact["bucket"]) + if previous is None: + raise RuntimeError( + f"a scoped run cannot introduce artifact {route['artifact']}; " + f"publish it in the same run" + ) + current = { + "path": route["path"], + "access": "public" if route["access"] == "legacy" else route["access"], + "artifact": route["artifact"], + } + recorded = {key: previous[key] for key in current} + if recorded != current: + raise RuntimeError( + f"a scoped run may not change unselected artifact {route['artifact']}'s route " + f"contract ({recorded} -> {current}); publish it in the same run" + ) + + def deploy_static(site_name, site_dir, action_dir, token, cfg): artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]} credential_env_names = { name for artifact in cfg["artifacts"] for name in artifact["credentials"].values() } + # Publication is scoped to the selected artifacts; the rendered route + # contract is not. Manifests and immutable-path history stay whole, so a + # partial publish can never retire another artifact's route or bucket. + publishing = selected_routes(cfg) validate_publication_environment(cfg) - for artifact in cfg["artifacts"]: + for artifact in selected_artifacts(cfg): validate_artifact_output(site_dir, artifact) apps_dir = clone_apps(token) app_dir = apps_dir / "sjc001" / "websites" / site_name manifests_dir = app_dir / "manifests" previous_contracts = previous_route_contracts(app_dir) validate_route_migrations(cfg, previous_contracts) + validate_scoped_history(cfg, previous_contracts) # Complete immutable work across the whole publication before any route's # mutable pointers can change. Partial immutable success is safe; mixing a # new route with an old route after a later immutable failure is not. - for route in cfg["routes"]: + for route in publishing: publish_route_immutables( artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, ) # Reconcile every browser-read policy before publishing mutable content. # A CORS failure therefore cannot leave a new channel pointing at a release # whose cross-origin assets browsers cannot consume. - reconcile_artifact_cors(cfg["artifacts"], credential_env_names) - for route in cfg["routes"]: + reconcile_artifact_cors(selected_artifacts(cfg), credential_env_names) + for route in publishing: s3_sync( artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]), @@ -596,6 +640,8 @@ def cmd_deploy(): cfg = parse_site_yaml(site_dir) if not cfg["enabled"]: + if len(cfg["selected"]) != len(cfg["artifacts"]): + die("a disabled site decommissions whole; drop the artifacts selection") print("Site disabled — running decommission...") decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]]) return diff --git a/scripts/utils.py b/scripts/utils.py index f499731..46391e6 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -274,6 +274,7 @@ def _legacy_config(raw, site_name): "aliases": _aliases(raw.get("aliases"), raw["domain"]), "enabled": raw.get("enabled", True), "artifacts": [artifact], + "selected": ["site"], "routes": [{ "name": "site", "path": "/", "artifact": "site", "access": "legacy", "access_middleware": None, @@ -457,6 +458,33 @@ def _validate_multi(cfg): raise ConfigError("a public '/' catch-all would expose unmatched protected content") +def _artifact_selection(cfg, requested): + """Resolve the subset of declared artifacts this run publishes.""" + declared = [artifact["name"] for artifact in cfg["artifacts"]] + if not requested or not requested.strip(): + return declared + names = [part for part in re.split(r"[,\s]+", requested.strip()) if part] + unknown = sorted({name for name in names if name not in declared}) + if unknown: + raise ConfigError( + "artifacts selects undeclared artifact(s) " + ", ".join(unknown) + + "; site.yaml declares " + ", ".join(declared) + ) + return [name for name in declared if name in set(names)] + + +def selected_artifacts(cfg): + """Artifacts this run builds and publishes, in declaration order.""" + chosen = set(cfg["selected"]) + return [artifact for artifact in cfg["artifacts"] if artifact["name"] in chosen] + + +def selected_routes(cfg): + """Routes whose artifact this run publishes, in route order.""" + chosen = set(cfg["selected"]) + return [route for route in cfg["routes"] if route["artifact"] in chosen] + + def normalize_site_config(raw, site_name): raw = _mapping(raw, "site.yaml") domain = _hostname(raw.get("domain"), "domain") @@ -482,6 +510,7 @@ def normalize_site_config(raw, site_name): "routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])), } _validate_multi(cfg) + cfg["selected"] = [artifact["name"] for artifact in cfg["artifacts"]] for route in cfg["routes"]: if len(f"{k8s_name(site_name)}-{route['name']}") > 63: raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters") @@ -534,12 +563,14 @@ def parse_site_yaml(site_dir, site_name=None): try: with open(path) as stream: cfg = normalize_site_config(yaml.safe_load(stream), site_name) + cfg["selected"] = _artifact_selection(cfg, os.environ.get("SITE_ARTIFACTS")) except (ConfigError, yaml.YAMLError) as exc: die(str(exc)) print("Site config:") print(f" domain: {cfg['domain']}") print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}") print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}") + print(f" publishing: {cfg['selected']}") print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}") return cfg diff --git a/tests/test_contract.py b/tests/test_contract.py index f09deb9..2a07b30 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -24,7 +24,12 @@ sys.path.insert(0, str(ROOT / "scripts")) import deploy import build import utils -from utils import ConfigError, normalize_site_config, validate_artifact_inputs +from utils import ( + ConfigError, + normalize_site_config, + parse_site_yaml, + validate_artifact_inputs, +) def fixture(name): @@ -404,6 +409,215 @@ class BuildTests(unittest.TestCase): ) +class ArtifactSelectionTests(unittest.TestCase): + """The `artifacts` input scopes publication, never the route contract.""" + + def parse(self, selection): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "site.yaml").write_text( + (ROOT / "tests" / "fixtures" / "split-site.yaml").read_text() + ) + environment = {"SITE_REPO": "fritzlab/baseline"} + if selection is not None: + environment["SITE_ARTIFACTS"] = selection + with patch.dict(os.environ, environment, clear=False), \ + redirect_stdout(io.StringIO()): + if selection is None: + os.environ.pop("SITE_ARTIFACTS", None) + return parse_site_yaml(root) + + def test_absent_selection_publishes_every_declared_artifact(self): + self.assertEqual(["distributions", "portal"], self.parse(None)["selected"]) + self.assertEqual(["distributions", "portal"], self.parse(" ")["selected"]) + + def test_selection_accepts_comma_and_whitespace_separated_names(self): + self.assertEqual(["portal"], self.parse("portal")["selected"]) + self.assertEqual( + ["distributions", "portal"], self.parse("portal, distributions")["selected"] + ) + self.assertEqual( + ["distributions", "portal"], self.parse("portal distributions")["selected"] + ) + + def test_undeclared_selection_fails_before_any_publication(self): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit): + self.parse("catalogue") + self.assertIn("undeclared artifact", stderr.getvalue()) + self.assertIn("catalogue", stderr.getvalue()) + + def test_build_skips_unselected_artifacts(self): + raw = fixture("split-site.yaml") + for artifact in raw["artifacts"]: + artifact["tidy"] = False + cfg = normalize_site_config(raw, "baseline.fritzlab.net") + cfg["selected"] = ["portal"] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "portal" / "build").mkdir(parents=True) + (root / "portal" / "build" / "index.html").write_text("portal") + built = [] + with patch.object(build, "build_artifact", + side_effect=lambda _root, artifact: built.append(artifact["name"])), \ + patch.dict(os.environ, {"SITE_DIR": str(root)}, clear=False), \ + patch.object(build, "parse_site_yaml", return_value=cfg), \ + redirect_stdout(io.StringIO()): + build.cmd_build() + self.assertEqual(["portal"], built) + + def deploy_selection(self, selection, environment): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + cfg["selected"] = selection + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + apps_dir = root / "apps" + app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net" + app_dir.mkdir(parents=True) + # A scoped run refuses an artifact it has no published history for, so + # the fixture starts from the whole publication this one narrows. + (app_dir / "site-publish-history.yaml").write_text(yaml.safe_dump({ + "version": 1, + "buckets": { + artifact["bucket"]: { + "path": route["path"], + "access": "public" if route["access"] == "legacy" else route["access"], + "artifact": route["artifact"], + "immutable_paths": deploy.immutable_key_prefixes(artifact, route), + } + for route in cfg["routes"] + for artifact in [next(item for item in cfg["artifacts"] + if item["name"] == route["artifact"])] + }, + }, sort_keys=True)) + for artifact in cfg["artifacts"]: + if artifact["name"] not in set(selection): + continue + html = root / artifact["build_dir"] + html.mkdir(parents=True) + (html / "index.html").write_text(artifact["name"]) + for rule in artifact["cache_rules"]: + if rule["path"]: + (html / rule["path"]).mkdir(parents=True, exist_ok=True) + (html / rule["path"] / "keep.json").write_text("{}") + synced, immutables, cors = [], [], [] + with patch.dict(os.environ, environment, clear=False), \ + patch.object(deploy, "clone_apps", return_value=apps_dir), \ + patch.object(deploy, "commit_and_push"), \ + patch.object(deploy, "publish_route_immutables", + side_effect=lambda artifact, *_a, **_k: immutables.append(artifact["name"])), \ + patch.object(deploy, "reconcile_artifact_cors", + side_effect=lambda artifacts, *_a: cors.extend(item["name"] for item in artifacts)), \ + patch.object(deploy, "s3_sync", + side_effect=lambda artifact, *_a, **_k: synced.append(artifact["name"])), \ + redirect_stdout(io.StringIO()): + deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg) + files = {path.relative_to(app_dir).as_posix(): path.read_text() + for path in sorted(app_dir.rglob("*.yaml"))} + return {"synced": synced, "immutables": immutables, "cors": cors, "files": files} + + def test_selected_publication_leaves_the_whole_route_contract_rendered(self): + # Only the portal credentials exist: a scoped run must not demand the + # secrets of an artifact it is not publishing. + environment = {"PORTAL_S3_ACCESS_KEY": "portal-key", + "PORTAL_S3_SECRET_KEY": "portal-secret"} + for name in ("DIST_S3_ACCESS_KEY", "DIST_S3_SECRET_KEY"): + os.environ.pop(name, None) + result = self.deploy_selection(["portal"], environment) + self.assertEqual(["portal"], result["synced"]) + self.assertEqual(["portal"], result["immutables"]) + self.assertEqual(["portal"], result["cors"]) + self.assertIn("manifests/ingress-distributions.yaml", result["files"]) + self.assertIn("manifests/service-distributions.yaml", result["files"]) + history = yaml.safe_load(result["files"]["site-publish-history.yaml"]) + self.assertEqual({"baseline-dist", "baseline-portal"}, set(history["buckets"])) + self.assertEqual( + ["dist/releases"], history["buckets"]["baseline-dist"]["immutable_paths"] + ) + + def test_scoped_run_refuses_to_record_a_contract_it_did_not_publish(self): + """The reviewer's scenario: distributions goes protected in site.yaml while + a catalogue-only run publishes. Recording that access would make the + bucket unreturnable through validate_route_migrations.""" + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + published = { + "baseline-portal": { + "path": "/", "access": "protected", "artifact": "portal", "immutable_paths": [], + }, + "baseline-dist": { + "path": "/dist", "access": "public", "artifact": "distributions", + "immutable_paths": ["dist/releases"], + }, + } + # A whole publish agrees with what is recorded, scoped or not. + deploy.validate_scoped_history(cfg, published) + cfg["selected"] = ["portal"] + deploy.validate_scoped_history(cfg, published) + + moved = copy.deepcopy(cfg) + route = next(item for item in moved["routes"] if item["artifact"] == "distributions") + route["access"] = "protected" + route["access_middleware"] = "authentik-forwardauth" + with self.assertRaises(RuntimeError) as refused: + deploy.validate_scoped_history(moved, published) + self.assertIn("may not change unselected artifact distributions", str(refused.exception)) + # The same change published in the same run is allowed to proceed. + moved["selected"] = ["distributions", "portal"] + deploy.validate_scoped_history(moved, published) + + def test_scoped_run_refuses_an_artifact_with_no_published_history(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + cfg["selected"] = ["portal"] + with self.assertRaises(RuntimeError) as refused: + deploy.validate_scoped_history(cfg, {}) + self.assertIn("cannot introduce artifact distributions", str(refused.exception)) + + def test_scoped_deploy_runs_the_history_guard_before_touching_a_bucket(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + cfg["selected"] = ["portal"] + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + apps_dir = root / "apps" + app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net" + app_dir.mkdir(parents=True) + html = root / next( + item for item in cfg["artifacts"] if item["name"] == "portal" + )["build_dir"] + html.mkdir(parents=True) + (html / "index.html").write_text("portal") + with patch.dict(os.environ, { + "PORTAL_S3_ACCESS_KEY": "portal-key", "PORTAL_S3_SECRET_KEY": "portal-secret", + }, clear=False), \ + patch.object(deploy, "clone_apps", return_value=apps_dir), \ + patch.object(deploy, "commit_and_push"), \ + patch.object(deploy, "publish_route_immutables") as immutables, \ + patch.object(deploy, "reconcile_artifact_cors") as cors, \ + patch.object(deploy, "s3_sync") as sync, \ + redirect_stdout(io.StringIO()), \ + self.assertRaises(RuntimeError): + deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg) + immutables.assert_not_called() + cors.assert_not_called() + sync.assert_not_called() + + def test_disabled_site_refuses_a_partial_selection(self): + cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net") + cfg["enabled"] = False + cfg["selected"] = ["portal"] + with patch.object(deploy, "parse_site_yaml", return_value=cfg), \ + patch.dict(os.environ, { + "SITE_REPO": "fritzlab/baseline.fritzlab.net", "SITE_DIR": ".", + "ACTION_DIR": str(ROOT), "CI_BOT_TOKEN": "token", + }, clear=False), \ + patch.object(deploy, "decommission") as decommission, \ + redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \ + self.assertRaises(SystemExit): + deploy.cmd_deploy() + decommission.assert_not_called() + + class PublishingTests(unittest.TestCase): def test_cors_policy_is_reconciled_as_read_only_browser_access(self): captured = {}