Merge pull request 'feat(site-publish): scope publication with an artifacts selection' (#7) from artifacts-input into main

Reviewed-on: #7

README.md

@@ -234,10 +234,46 @@ my-site.fritzlab.net  300  IN  CNAME  gateway.sjc001.fritzlab.net.
 | `garage-admin-token` | legacy aliases only | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) |
 | `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint |
 | `username` | no | `ci-bot` | Gitea username |
+| `artifacts` | no | every declared artifact | Space- or comma-separated subset of `site.yaml` artifacts to build and publish |
 
 Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`,
 `GARAGE_S3_SECRET_KEY`, `GARAGE_ADMIN_TOKEN`.
 
+### Publishing a subset of artifacts
+
+A repository whose branches ship on different cadences names the ones this run
+owns:
+
+```yaml
+- uses: https://code.fritzlab.net/action/site-publish@<sha>
+  with:
+    token: ${{ secrets.CI_BOT_TOKEN }}
+    artifacts: catalogue
+```
+
+Selection scopes the build and the S3 publication only. Ingresses, Services,
+Certificates and the immutable-path history are always rendered from the whole
+`site.yaml`, so a scoped run cannot retire another artifact's route or delete
+its bucket contents. Credentials are resolved for the selected artifacts alone,
+so a workflow need not carry secrets for artifacts it does not publish.
+
+What a scoped run may **not** do is move an artifact it is not publishing. The
+generated route contract in `site-publish-history.yaml` is written from the
+whole `site.yaml`, and `access` there is a replacement rather than a union — so
+recording a change nothing published would turn intent into a fact that
+`validate_route_migrations` then refuses to undo. A run whose `site.yaml`
+changes an unselected artifact's route path, access or artifact name is refused
+before the first bucket is touched, naming both contracts: publish that artifact
+in the same run. For the same reason a scoped run cannot introduce an artifact
+that has no published history yet.
+
+Two more refusals, both before any bucket changes: an undeclared name, and
+`enabled: false` with a selection, because decommissioning is whole-site.
+
+One consequence to know: CORS reconciliation is scoped too, since a scoped run
+holds no credential for the other bucket. A `cors_origins:` change lands with
+that artifact's next publish, not on the merge that edits `site.yaml`.
+
 ## Tools
 
 - **`new-site.sh`** — create a new site: Gitea repo, Garage bucket, web hosting enabled.

action.yaml

@@ -28,6 +28,13 @@ inputs:
     description: Gitea username for git operations
     required: false
     default: ci-bot
+  artifacts:
+    # Scopes building and publishing only. Routes, Ingresses and the immutable
+    # -path history are always rendered from the whole site.yaml, so a scoped
+    # run never retires another artifact's route.
+    description: Space- or comma-separated subset of site.yaml artifacts to build and publish (default is every declared artifact)
+    required: false
+    default: ''
 runs:
   using: composite
   steps:
@@ -44,6 +51,7 @@ runs:
         ACTION_DIR: ${{ github.action_path }}
         GITHUB_RUN_NUMBER: ${{ github.run_number }}
         CI_BOT_USER: ${{ inputs.username }}
+        SITE_ARTIFACTS: ${{ inputs.artifacts }}
 
     - name: Deploy
       shell: bash
@@ -61,3 +69,4 @@ runs:
         GARAGE_ADMIN_ENDPOINT: ${{ inputs.garage-admin-endpoint }}
         GARAGE_ADMIN_TOKEN: ${{ inputs.garage-admin-token }}
         GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        SITE_ARTIFACTS: ${{ inputs.artifacts }}

scripts/build.py

@@ -5,7 +5,14 @@ import subprocess
 import tempfile
 from pathlib import Path
 
-from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs
+from utils import (
+    EXCLUDE_FILES,
+    env,
+    parse_site_yaml,
+    run,
+    selected_artifacts,
+    validate_artifact_inputs,
+)
 
 
 def build_artifact(site_dir, artifact):
@@ -66,5 +73,5 @@ def cmd_build():
 
     validate_artifact_inputs(site_dir, cfg)
 
-    for artifact in cfg["artifacts"]:
+    for artifact in selected_artifacts(cfg):
         build_artifact(site_dir, artifact)

scripts/deploy.py

@@ -25,6 +25,8 @@ from utils import (
     parse_site_yaml,
     render_templates,
     run,
+    selected_artifacts,
+    selected_routes,
     validate_artifact_inputs,
 )
 
@@ -47,8 +49,8 @@ def validate_artifact_output(site_dir, artifact):
 
 
 def validate_publication_environment(cfg):
-    """Resolve every declared credential before the first bucket is changed."""
-    for artifact in cfg["artifacts"]:
+    """Resolve every credential this run needs before the first bucket is changed."""
+    for artifact in selected_artifacts(cfg):
         env(artifact["credentials"]["access_key_env"])
         env(artifact["credentials"]["secret_key_env"])
     if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
@@ -533,31 +535,73 @@ def validate_route_migrations(cfg, previous_contracts):
             )
 
 
+def validate_scoped_history(cfg, previous_contracts):
+    """A scoped run may not record a route contract it did not publish.
+
+    render_site_manifests advances the stored contract for every route in
+    site.yaml, and `access` is overwritten rather than unioned the way
+    immutable_paths is. Without this, a catalogue-only publish could write a
+    protected access for the distributions bucket that nothing published, and
+    validate_route_migrations would then refuse to put that bucket back —
+    unpublished intent turned into an irreversible fact.
+    """
+    chosen = set(cfg["selected"])
+    if len(chosen) == len(cfg["artifacts"]):
+        return
+    artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
+    for route in cfg["routes"]:
+        if route["artifact"] in chosen:
+            continue
+        artifact = artifact_by_name[route["artifact"]]
+        previous = previous_contracts.get(artifact["bucket"])
+        if previous is None:
+            raise RuntimeError(
+                f"a scoped run cannot introduce artifact {route['artifact']}; "
+                f"publish it in the same run"
+            )
+        current = {
+            "path": route["path"],
+            "access": "public" if route["access"] == "legacy" else route["access"],
+            "artifact": route["artifact"],
+        }
+        recorded = {key: previous[key] for key in current}
+        if recorded != current:
+            raise RuntimeError(
+                f"a scoped run may not change unselected artifact {route['artifact']}'s route "
+                f"contract ({recorded} -> {current}); publish it in the same run"
+            )
+
+
 def deploy_static(site_name, site_dir, action_dir, token, cfg):
     artifact_by_name = {artifact["name"]: artifact for artifact in cfg["artifacts"]}
     credential_env_names = {
         name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
     }
+    # Publication is scoped to the selected artifacts; the rendered route
+    # contract is not. Manifests and immutable-path history stay whole, so a
+    # partial publish can never retire another artifact's route or bucket.
+    publishing = selected_routes(cfg)
     validate_publication_environment(cfg)
-    for artifact in cfg["artifacts"]:
+    for artifact in selected_artifacts(cfg):
         validate_artifact_output(site_dir, artifact)
     apps_dir = clone_apps(token)
     app_dir = apps_dir / "sjc001" / "websites" / site_name
     manifests_dir = app_dir / "manifests"
     previous_contracts = previous_route_contracts(app_dir)
     validate_route_migrations(cfg, previous_contracts)
+    validate_scoped_history(cfg, previous_contracts)
     # Complete immutable work across the whole publication before any route's
     # mutable pointers can change. Partial immutable success is safe; mixing a
     # new route with an old route after a later immutable failure is not.
-    for route in cfg["routes"]:
+    for route in publishing:
         publish_route_immutables(
             artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
         )
     # Reconcile every browser-read policy before publishing mutable content.
     # A CORS failure therefore cannot leave a new channel pointing at a release
     # whose cross-origin assets browsers cannot consume.
-    reconcile_artifact_cors(cfg["artifacts"], credential_env_names)
-    for route in cfg["routes"]:
+    reconcile_artifact_cors(selected_artifacts(cfg), credential_env_names)
+    for route in publishing:
         s3_sync(
             artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
             previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]),
@@ -596,6 +640,8 @@ def cmd_deploy():
     cfg = parse_site_yaml(site_dir)
 
     if not cfg["enabled"]:
+        if len(cfg["selected"]) != len(cfg["artifacts"]):
+            die("a disabled site decommissions whole; drop the artifacts selection")
         print("Site disabled — running decommission...")
         decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
         return

scripts/utils.py

@@ -274,6 +274,7 @@ def _legacy_config(raw, site_name):
         "aliases": _aliases(raw.get("aliases"), raw["domain"]),
         "enabled": raw.get("enabled", True),
         "artifacts": [artifact],
+        "selected": ["site"],
         "routes": [{
             "name": "site", "path": "/", "artifact": "site", "access": "legacy",
             "access_middleware": None,
@@ -457,6 +458,33 @@ def _validate_multi(cfg):
         raise ConfigError("a public '/' catch-all would expose unmatched protected content")
 
 
+def _artifact_selection(cfg, requested):
+    """Resolve the subset of declared artifacts this run publishes."""
+    declared = [artifact["name"] for artifact in cfg["artifacts"]]
+    if not requested or not requested.strip():
+        return declared
+    names = [part for part in re.split(r"[,\s]+", requested.strip()) if part]
+    unknown = sorted({name for name in names if name not in declared})
+    if unknown:
+        raise ConfigError(
+            "artifacts selects undeclared artifact(s) " + ", ".join(unknown)
+            + "; site.yaml declares " + ", ".join(declared)
+        )
+    return [name for name in declared if name in set(names)]
+
+
+def selected_artifacts(cfg):
+    """Artifacts this run builds and publishes, in declaration order."""
+    chosen = set(cfg["selected"])
+    return [artifact for artifact in cfg["artifacts"] if artifact["name"] in chosen]
+
+
+def selected_routes(cfg):
+    """Routes whose artifact this run publishes, in route order."""
+    chosen = set(cfg["selected"])
+    return [route for route in cfg["routes"] if route["artifact"] in chosen]
+
+
 def normalize_site_config(raw, site_name):
     raw = _mapping(raw, "site.yaml")
     domain = _hostname(raw.get("domain"), "domain")
@@ -482,6 +510,7 @@ def normalize_site_config(raw, site_name):
         "routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])),
     }
     _validate_multi(cfg)
+    cfg["selected"] = [artifact["name"] for artifact in cfg["artifacts"]]
     for route in cfg["routes"]:
         if len(f"{k8s_name(site_name)}-{route['name']}") > 63:
             raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters")
@@ -534,12 +563,14 @@ def parse_site_yaml(site_dir, site_name=None):
     try:
         with open(path) as stream:
             cfg = normalize_site_config(yaml.safe_load(stream), site_name)
+        cfg["selected"] = _artifact_selection(cfg, os.environ.get("SITE_ARTIFACTS"))
     except (ConfigError, yaml.YAMLError) as exc:
         die(str(exc))
     print("Site config:")
     print(f"  domain: {cfg['domain']}")
     print(f"  contract: {cfg['compatibility'] or 'split-surface-v2'}")
     print(f"  artifacts: {[item['name'] for item in cfg['artifacts']]}")
+    print(f"  publishing: {cfg['selected']}")
     print(f"  routes: {[(item['path'], item['access']) for item in cfg['routes']]}")
     return cfg
 

tests/test_contract.py

@@ -24,7 +24,12 @@ sys.path.insert(0, str(ROOT / "scripts"))
 import deploy
 import build
 import utils
-from utils import ConfigError, normalize_site_config, validate_artifact_inputs
+from utils import (
+    ConfigError,
+    normalize_site_config,
+    parse_site_yaml,
+    validate_artifact_inputs,
+)
 
 
 def fixture(name):
@@ -404,6 +409,215 @@ class BuildTests(unittest.TestCase):
             )
 
 
+class ArtifactSelectionTests(unittest.TestCase):
+    """The `artifacts` input scopes publication, never the route contract."""
+
+    def parse(self, selection):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "site.yaml").write_text(
+                (ROOT / "tests" / "fixtures" / "split-site.yaml").read_text()
+            )
+            environment = {"SITE_REPO": "fritzlab/baseline"}
+            if selection is not None:
+                environment["SITE_ARTIFACTS"] = selection
+            with patch.dict(os.environ, environment, clear=False), \
+                    redirect_stdout(io.StringIO()):
+                if selection is None:
+                    os.environ.pop("SITE_ARTIFACTS", None)
+                return parse_site_yaml(root)
+
+    def test_absent_selection_publishes_every_declared_artifact(self):
+        self.assertEqual(["distributions", "portal"], self.parse(None)["selected"])
+        self.assertEqual(["distributions", "portal"], self.parse("  ")["selected"])
+
+    def test_selection_accepts_comma_and_whitespace_separated_names(self):
+        self.assertEqual(["portal"], self.parse("portal")["selected"])
+        self.assertEqual(
+            ["distributions", "portal"], self.parse("portal, distributions")["selected"]
+        )
+        self.assertEqual(
+            ["distributions", "portal"], self.parse("portal distributions")["selected"]
+        )
+
+    def test_undeclared_selection_fails_before_any_publication(self):
+        stderr = io.StringIO()
+        with redirect_stderr(stderr), self.assertRaises(SystemExit):
+            self.parse("catalogue")
+        self.assertIn("undeclared artifact", stderr.getvalue())
+        self.assertIn("catalogue", stderr.getvalue())
+
+    def test_build_skips_unselected_artifacts(self):
+        raw = fixture("split-site.yaml")
+        for artifact in raw["artifacts"]:
+            artifact["tidy"] = False
+        cfg = normalize_site_config(raw, "baseline.fritzlab.net")
+        cfg["selected"] = ["portal"]
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "portal" / "build").mkdir(parents=True)
+            (root / "portal" / "build" / "index.html").write_text("portal")
+            built = []
+            with patch.object(build, "build_artifact",
+                              side_effect=lambda _root, artifact: built.append(artifact["name"])), \
+                    patch.dict(os.environ, {"SITE_DIR": str(root)}, clear=False), \
+                    patch.object(build, "parse_site_yaml", return_value=cfg), \
+                    redirect_stdout(io.StringIO()):
+                build.cmd_build()
+            self.assertEqual(["portal"], built)
+
+    def deploy_selection(self, selection, environment):
+        cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
+        cfg["selected"] = selection
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        root = Path(tmp.name)
+        apps_dir = root / "apps"
+        app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net"
+        app_dir.mkdir(parents=True)
+        # A scoped run refuses an artifact it has no published history for, so
+        # the fixture starts from the whole publication this one narrows.
+        (app_dir / "site-publish-history.yaml").write_text(yaml.safe_dump({
+            "version": 1,
+            "buckets": {
+                artifact["bucket"]: {
+                    "path": route["path"],
+                    "access": "public" if route["access"] == "legacy" else route["access"],
+                    "artifact": route["artifact"],
+                    "immutable_paths": deploy.immutable_key_prefixes(artifact, route),
+                }
+                for route in cfg["routes"]
+                for artifact in [next(item for item in cfg["artifacts"]
+                                      if item["name"] == route["artifact"])]
+            },
+        }, sort_keys=True))
+        for artifact in cfg["artifacts"]:
+            if artifact["name"] not in set(selection):
+                continue
+            html = root / artifact["build_dir"]
+            html.mkdir(parents=True)
+            (html / "index.html").write_text(artifact["name"])
+            for rule in artifact["cache_rules"]:
+                if rule["path"]:
+                    (html / rule["path"]).mkdir(parents=True, exist_ok=True)
+                    (html / rule["path"] / "keep.json").write_text("{}")
+        synced, immutables, cors = [], [], []
+        with patch.dict(os.environ, environment, clear=False), \
+                patch.object(deploy, "clone_apps", return_value=apps_dir), \
+                patch.object(deploy, "commit_and_push"), \
+                patch.object(deploy, "publish_route_immutables",
+                             side_effect=lambda artifact, *_a, **_k: immutables.append(artifact["name"])), \
+                patch.object(deploy, "reconcile_artifact_cors",
+                             side_effect=lambda artifacts, *_a: cors.extend(item["name"] for item in artifacts)), \
+                patch.object(deploy, "s3_sync",
+                             side_effect=lambda artifact, *_a, **_k: synced.append(artifact["name"])), \
+                redirect_stdout(io.StringIO()):
+            deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg)
+        files = {path.relative_to(app_dir).as_posix(): path.read_text()
+                 for path in sorted(app_dir.rglob("*.yaml"))}
+        return {"synced": synced, "immutables": immutables, "cors": cors, "files": files}
+
+    def test_selected_publication_leaves_the_whole_route_contract_rendered(self):
+        # Only the portal credentials exist: a scoped run must not demand the
+        # secrets of an artifact it is not publishing.
+        environment = {"PORTAL_S3_ACCESS_KEY": "portal-key",
+                       "PORTAL_S3_SECRET_KEY": "portal-secret"}
+        for name in ("DIST_S3_ACCESS_KEY", "DIST_S3_SECRET_KEY"):
+            os.environ.pop(name, None)
+        result = self.deploy_selection(["portal"], environment)
+        self.assertEqual(["portal"], result["synced"])
+        self.assertEqual(["portal"], result["immutables"])
+        self.assertEqual(["portal"], result["cors"])
+        self.assertIn("manifests/ingress-distributions.yaml", result["files"])
+        self.assertIn("manifests/service-distributions.yaml", result["files"])
+        history = yaml.safe_load(result["files"]["site-publish-history.yaml"])
+        self.assertEqual({"baseline-dist", "baseline-portal"}, set(history["buckets"]))
+        self.assertEqual(
+            ["dist/releases"], history["buckets"]["baseline-dist"]["immutable_paths"]
+        )
+
+    def test_scoped_run_refuses_to_record_a_contract_it_did_not_publish(self):
+        """The reviewer's scenario: distributions goes protected in site.yaml while
+        a catalogue-only run publishes. Recording that access would make the
+        bucket unreturnable through validate_route_migrations."""
+        cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
+        published = {
+            "baseline-portal": {
+                "path": "/", "access": "protected", "artifact": "portal", "immutable_paths": [],
+            },
+            "baseline-dist": {
+                "path": "/dist", "access": "public", "artifact": "distributions",
+                "immutable_paths": ["dist/releases"],
+            },
+        }
+        # A whole publish agrees with what is recorded, scoped or not.
+        deploy.validate_scoped_history(cfg, published)
+        cfg["selected"] = ["portal"]
+        deploy.validate_scoped_history(cfg, published)
+
+        moved = copy.deepcopy(cfg)
+        route = next(item for item in moved["routes"] if item["artifact"] == "distributions")
+        route["access"] = "protected"
+        route["access_middleware"] = "authentik-forwardauth"
+        with self.assertRaises(RuntimeError) as refused:
+            deploy.validate_scoped_history(moved, published)
+        self.assertIn("may not change unselected artifact distributions", str(refused.exception))
+        # The same change published in the same run is allowed to proceed.
+        moved["selected"] = ["distributions", "portal"]
+        deploy.validate_scoped_history(moved, published)
+
+    def test_scoped_run_refuses_an_artifact_with_no_published_history(self):
+        cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
+        cfg["selected"] = ["portal"]
+        with self.assertRaises(RuntimeError) as refused:
+            deploy.validate_scoped_history(cfg, {})
+        self.assertIn("cannot introduce artifact distributions", str(refused.exception))
+
+    def test_scoped_deploy_runs_the_history_guard_before_touching_a_bucket(self):
+        cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
+        cfg["selected"] = ["portal"]
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        root = Path(tmp.name)
+        apps_dir = root / "apps"
+        app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net"
+        app_dir.mkdir(parents=True)
+        html = root / next(
+            item for item in cfg["artifacts"] if item["name"] == "portal"
+        )["build_dir"]
+        html.mkdir(parents=True)
+        (html / "index.html").write_text("portal")
+        with patch.dict(os.environ, {
+            "PORTAL_S3_ACCESS_KEY": "portal-key", "PORTAL_S3_SECRET_KEY": "portal-secret",
+        }, clear=False), \
+                patch.object(deploy, "clone_apps", return_value=apps_dir), \
+                patch.object(deploy, "commit_and_push"), \
+                patch.object(deploy, "publish_route_immutables") as immutables, \
+                patch.object(deploy, "reconcile_artifact_cors") as cors, \
+                patch.object(deploy, "s3_sync") as sync, \
+                redirect_stdout(io.StringIO()), \
+                self.assertRaises(RuntimeError):
+            deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg)
+        immutables.assert_not_called()
+        cors.assert_not_called()
+        sync.assert_not_called()
+
+    def test_disabled_site_refuses_a_partial_selection(self):
+        cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
+        cfg["enabled"] = False
+        cfg["selected"] = ["portal"]
+        with patch.object(deploy, "parse_site_yaml", return_value=cfg), \
+                patch.dict(os.environ, {
+                    "SITE_REPO": "fritzlab/baseline.fritzlab.net", "SITE_DIR": ".",
+                    "ACTION_DIR": str(ROOT), "CI_BOT_TOKEN": "token",
+                }, clear=False), \
+                patch.object(deploy, "decommission") as decommission, \
+                redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \
+                self.assertRaises(SystemExit):
+            deploy.cmd_deploy()
+        decommission.assert_not_called()
+
+
 class PublishingTests(unittest.TestCase):
     def test_cors_policy_is_reconciled_as_read_only_browser_access(self):
         captured = {}