feat(site-publish): scope publication with an artifacts selection #7

Merged
dfritz merged 3 commits from artifacts-input into main 2026-09-06 21:31:51 +00:00
6 changed files with 217 additions and 9 deletions
Showing only changes of commit 3dfee64335 - Show all commits
+21
View File
@@ -234,10 +234,31 @@ my-site.fritzlab.net 300 IN CNAME gateway.sjc001.fritzlab.net.
| `garage-admin-token` | legacy aliases only | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) | | `garage-admin-token` | legacy aliases only | | Garage admin API token (`admin-token` from `garage-rpc-secret` in `storage` ns) |
| `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint | | `garage-admin-endpoint` | no | `http://garage.storage.svc:3903` | Garage admin API endpoint |
| `username` | no | `ci-bot` | Gitea username | | `username` | no | `ci-bot` | Gitea username |
| `artifacts` | no | every declared artifact | Space- or comma-separated subset of `site.yaml` artifacts to build and publish |
Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`, Org secrets in `websites`: `CI_BOT_TOKEN`, `GARAGE_S3_ACCESS_KEY`,
`GARAGE_S3_SECRET_KEY`, `GARAGE_ADMIN_TOKEN`. `GARAGE_S3_SECRET_KEY`, `GARAGE_ADMIN_TOKEN`.
### Publishing a subset of artifacts
A repository whose branches ship on different cadences names the ones this run
owns:
```yaml
- uses: https://code.fritzlab.net/action/site-publish@<sha>
with:
token: ${{ secrets.CI_BOT_TOKEN }}
artifacts: catalogue
```
Selection scopes the build and the S3 publication only. Ingresses, Services,
Certificates and the immutable-path history are always rendered from the whole
`site.yaml`, so a scoped run cannot retire another artifact's route or delete
its bucket contents. Credentials are resolved for the selected artifacts alone,
so a workflow need not carry secrets for artifacts it does not publish. An
undeclared name fails before the first bucket is touched, and `enabled: false`
refuses a selection because decommissioning is whole-site.
## Tools ## Tools
- **`new-site.sh`** — create a new site: Gitea repo, Garage bucket, web hosting enabled. - **`new-site.sh`** — create a new site: Gitea repo, Garage bucket, web hosting enabled.
+9
View File
@@ -28,6 +28,13 @@ inputs:
description: Gitea username for git operations description: Gitea username for git operations
required: false required: false
default: ci-bot default: ci-bot
artifacts:
# Scopes building and publishing only. Routes, Ingresses and the immutable
# -path history are always rendered from the whole site.yaml, so a scoped
# run never retires another artifact's route.
description: Space- or comma-separated subset of site.yaml artifacts to build and publish (default is every declared artifact)
required: false
default: ''
runs: runs:
using: composite using: composite
steps: steps:
@@ -44,6 +51,7 @@ runs:
ACTION_DIR: ${{ github.action_path }} ACTION_DIR: ${{ github.action_path }}
GITHUB_RUN_NUMBER: ${{ github.run_number }} GITHUB_RUN_NUMBER: ${{ github.run_number }}
CI_BOT_USER: ${{ inputs.username }} CI_BOT_USER: ${{ inputs.username }}
SITE_ARTIFACTS: ${{ inputs.artifacts }}
- name: Deploy - name: Deploy
shell: bash shell: bash
@@ -61,3 +69,4 @@ runs:
GARAGE_ADMIN_ENDPOINT: ${{ inputs.garage-admin-endpoint }} GARAGE_ADMIN_ENDPOINT: ${{ inputs.garage-admin-endpoint }}
GARAGE_ADMIN_TOKEN: ${{ inputs.garage-admin-token }} GARAGE_ADMIN_TOKEN: ${{ inputs.garage-admin-token }}
GITHUB_RUN_NUMBER: ${{ github.run_number }} GITHUB_RUN_NUMBER: ${{ github.run_number }}
SITE_ARTIFACTS: ${{ inputs.artifacts }}
+9 -2
View File
@@ -5,7 +5,14 @@ import subprocess
import tempfile import tempfile
from pathlib import Path from pathlib import Path
from utils import EXCLUDE_FILES, env, parse_site_yaml, run, validate_artifact_inputs from utils import (
EXCLUDE_FILES,
env,
parse_site_yaml,
run,
selected_artifacts,
validate_artifact_inputs,
)
def build_artifact(site_dir, artifact): def build_artifact(site_dir, artifact):
@@ -66,5 +73,5 @@ def cmd_build():
validate_artifact_inputs(site_dir, cfg) validate_artifact_inputs(site_dir, cfg)
for artifact in cfg["artifacts"]: for artifact in selected_artifacts(cfg):
build_artifact(site_dir, artifact) build_artifact(site_dir, artifact)
+14 -6
View File
@@ -25,6 +25,8 @@ from utils import (
parse_site_yaml, parse_site_yaml,
render_templates, render_templates,
run, run,
selected_artifacts,
selected_routes,
validate_artifact_inputs, validate_artifact_inputs,
) )
@@ -47,8 +49,8 @@ def validate_artifact_output(site_dir, artifact):
def validate_publication_environment(cfg): def validate_publication_environment(cfg):
"""Resolve every declared credential before the first bucket is changed.""" """Resolve every credential this run needs before the first bucket is changed."""
for artifact in cfg["artifacts"]: for artifact in selected_artifacts(cfg):
env(artifact["credentials"]["access_key_env"]) env(artifact["credentials"]["access_key_env"])
env(artifact["credentials"]["secret_key_env"]) env(artifact["credentials"]["secret_key_env"])
if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"): if cfg["compatibility"] and cfg["aliases"] and not os.environ.get("GARAGE_ADMIN_TOKEN"):
@@ -538,8 +540,12 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
credential_env_names = { credential_env_names = {
name for artifact in cfg["artifacts"] for name in artifact["credentials"].values() name for artifact in cfg["artifacts"] for name in artifact["credentials"].values()
} }
# Publication is scoped to the selected artifacts; the rendered route
# contract is not. Manifests and immutable-path history stay whole, so a
# partial publish can never retire another artifact's route or bucket.
publishing = selected_routes(cfg)
validate_publication_environment(cfg) validate_publication_environment(cfg)
for artifact in cfg["artifacts"]: for artifact in selected_artifacts(cfg):
validate_artifact_output(site_dir, artifact) validate_artifact_output(site_dir, artifact)
apps_dir = clone_apps(token) apps_dir = clone_apps(token)
app_dir = apps_dir / "sjc001" / "websites" / site_name app_dir = apps_dir / "sjc001" / "websites" / site_name
@@ -549,15 +555,15 @@ def deploy_static(site_name, site_dir, action_dir, token, cfg):
# Complete immutable work across the whole publication before any route's # Complete immutable work across the whole publication before any route's
# mutable pointers can change. Partial immutable success is safe; mixing a # mutable pointers can change. Partial immutable success is safe; mixing a
# new route with an old route after a later immutable failure is not. # new route with an old route after a later immutable failure is not.
for route in cfg["routes"]: for route in publishing:
publish_route_immutables( publish_route_immutables(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
) )
# Reconcile every browser-read policy before publishing mutable content. # Reconcile every browser-read policy before publishing mutable content.
# A CORS failure therefore cannot leave a new channel pointing at a release # A CORS failure therefore cannot leave a new channel pointing at a release
# whose cross-origin assets browsers cannot consume. # whose cross-origin assets browsers cannot consume.
reconcile_artifact_cors(cfg["artifacts"], credential_env_names) reconcile_artifact_cors(selected_artifacts(cfg), credential_env_names)
for route in cfg["routes"]: for route in publishing:
s3_sync( s3_sync(
artifact_by_name[route["artifact"]], route, site_dir, credential_env_names, artifact_by_name[route["artifact"]], route, site_dir, credential_env_names,
previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]), previous_contracts.get(artifact_by_name[route["artifact"]]["bucket"]),
@@ -596,6 +602,8 @@ def cmd_deploy():
cfg = parse_site_yaml(site_dir) cfg = parse_site_yaml(site_dir)
if not cfg["enabled"]: if not cfg["enabled"]:
if len(cfg["selected"]) != len(cfg["artifacts"]):
die("a disabled site decommissions whole; drop the artifacts selection")
print("Site disabled — running decommission...") print("Site disabled — running decommission...")
decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]]) decommission(site_name, token, [artifact["bucket"] for artifact in cfg["artifacts"]])
return return
+31
View File
@@ -274,6 +274,7 @@ def _legacy_config(raw, site_name):
"aliases": _aliases(raw.get("aliases"), raw["domain"]), "aliases": _aliases(raw.get("aliases"), raw["domain"]),
"enabled": raw.get("enabled", True), "enabled": raw.get("enabled", True),
"artifacts": [artifact], "artifacts": [artifact],
"selected": ["site"],
"routes": [{ "routes": [{
"name": "site", "path": "/", "artifact": "site", "access": "legacy", "name": "site", "path": "/", "artifact": "site", "access": "legacy",
"access_middleware": None, "access_middleware": None,
@@ -457,6 +458,33 @@ def _validate_multi(cfg):
raise ConfigError("a public '/' catch-all would expose unmatched protected content") raise ConfigError("a public '/' catch-all would expose unmatched protected content")
def _artifact_selection(cfg, requested):
"""Resolve the subset of declared artifacts this run publishes."""
declared = [artifact["name"] for artifact in cfg["artifacts"]]
if not requested or not requested.strip():
return declared
names = [part for part in re.split(r"[,\s]+", requested.strip()) if part]
unknown = sorted({name for name in names if name not in declared})
if unknown:
raise ConfigError(
"artifacts selects undeclared artifact(s) " + ", ".join(unknown)
+ "; site.yaml declares " + ", ".join(declared)
)
return [name for name in declared if name in set(names)]
def selected_artifacts(cfg):
"""Artifacts this run builds and publishes, in declaration order."""
chosen = set(cfg["selected"])
return [artifact for artifact in cfg["artifacts"] if artifact["name"] in chosen]
def selected_routes(cfg):
"""Routes whose artifact this run publishes, in route order."""
chosen = set(cfg["selected"])
return [route for route in cfg["routes"] if route["artifact"] in chosen]
def normalize_site_config(raw, site_name): def normalize_site_config(raw, site_name):
raw = _mapping(raw, "site.yaml") raw = _mapping(raw, "site.yaml")
domain = _hostname(raw.get("domain"), "domain") domain = _hostname(raw.get("domain"), "domain")
@@ -482,6 +510,7 @@ def normalize_site_config(raw, site_name):
"routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])), "routes": sorted(routes, key=lambda item: (-len(item["path"]), item["path"], item["name"])),
} }
_validate_multi(cfg) _validate_multi(cfg)
cfg["selected"] = [artifact["name"] for artifact in cfg["artifacts"]]
for route in cfg["routes"]: for route in cfg["routes"]:
if len(f"{k8s_name(site_name)}-{route['name']}") > 63: if len(f"{k8s_name(site_name)}-{route['name']}") > 63:
raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters") raise ConfigError(f"route {route['name']} makes the generated Service name exceed 63 characters")
@@ -534,12 +563,14 @@ def parse_site_yaml(site_dir, site_name=None):
try: try:
with open(path) as stream: with open(path) as stream:
cfg = normalize_site_config(yaml.safe_load(stream), site_name) cfg = normalize_site_config(yaml.safe_load(stream), site_name)
cfg["selected"] = _artifact_selection(cfg, os.environ.get("SITE_ARTIFACTS"))
except (ConfigError, yaml.YAMLError) as exc: except (ConfigError, yaml.YAMLError) as exc:
die(str(exc)) die(str(exc))
print("Site config:") print("Site config:")
print(f" domain: {cfg['domain']}") print(f" domain: {cfg['domain']}")
print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}") print(f" contract: {cfg['compatibility'] or 'split-surface-v2'}")
print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}") print(f" artifacts: {[item['name'] for item in cfg['artifacts']]}")
print(f" publishing: {cfg['selected']}")
print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}") print(f" routes: {[(item['path'], item['access']) for item in cfg['routes']]}")
return cfg return cfg
+133 -1
View File
@@ -24,7 +24,12 @@ sys.path.insert(0, str(ROOT / "scripts"))
import deploy import deploy
import build import build
import utils import utils
from utils import ConfigError, normalize_site_config, validate_artifact_inputs from utils import (
ConfigError,
normalize_site_config,
parse_site_yaml,
validate_artifact_inputs,
)
def fixture(name): def fixture(name):
@@ -404,6 +409,133 @@ class BuildTests(unittest.TestCase):
) )
class ArtifactSelectionTests(unittest.TestCase):
"""The `artifacts` input scopes publication, never the route contract."""
def parse(self, selection):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "site.yaml").write_text(
(ROOT / "tests" / "fixtures" / "split-site.yaml").read_text()
)
environment = {"SITE_REPO": "fritzlab/baseline"}
if selection is not None:
environment["SITE_ARTIFACTS"] = selection
with patch.dict(os.environ, environment, clear=False), \
redirect_stdout(io.StringIO()):
if selection is None:
os.environ.pop("SITE_ARTIFACTS", None)
return parse_site_yaml(root)
def test_absent_selection_publishes_every_declared_artifact(self):
self.assertEqual(["distributions", "portal"], self.parse(None)["selected"])
self.assertEqual(["distributions", "portal"], self.parse(" ")["selected"])
def test_selection_accepts_comma_and_whitespace_separated_names(self):
self.assertEqual(["portal"], self.parse("portal")["selected"])
self.assertEqual(
["distributions", "portal"], self.parse("portal, distributions")["selected"]
)
self.assertEqual(
["distributions", "portal"], self.parse("portal distributions")["selected"]
)
def test_undeclared_selection_fails_before_any_publication(self):
stderr = io.StringIO()
with redirect_stderr(stderr), self.assertRaises(SystemExit):
self.parse("catalogue")
self.assertIn("undeclared artifact", stderr.getvalue())
self.assertIn("catalogue", stderr.getvalue())
def test_build_skips_unselected_artifacts(self):
raw = fixture("split-site.yaml")
for artifact in raw["artifacts"]:
artifact["tidy"] = False
cfg = normalize_site_config(raw, "baseline.fritzlab.net")
cfg["selected"] = ["portal"]
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "portal" / "build").mkdir(parents=True)
(root / "portal" / "build" / "index.html").write_text("portal")
built = []
with patch.object(build, "build_artifact",
side_effect=lambda _root, artifact: built.append(artifact["name"])), \
patch.dict(os.environ, {"SITE_DIR": str(root)}, clear=False), \
patch.object(build, "parse_site_yaml", return_value=cfg), \
redirect_stdout(io.StringIO()):
build.cmd_build()
self.assertEqual(["portal"], built)
def deploy_selection(self, selection, environment):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
cfg["selected"] = selection
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
apps_dir = root / "apps"
app_dir = apps_dir / "sjc001" / "websites" / "baseline.fritzlab.net"
app_dir.mkdir(parents=True)
for artifact in cfg["artifacts"]:
if artifact["name"] not in set(selection):
continue
html = root / artifact["build_dir"]
html.mkdir(parents=True)
(html / "index.html").write_text(artifact["name"])
for rule in artifact["cache_rules"]:
if rule["path"]:
(html / rule["path"]).mkdir(parents=True, exist_ok=True)
(html / rule["path"] / "keep.json").write_text("{}")
synced, immutables, cors = [], [], []
with patch.dict(os.environ, environment, clear=False), \
patch.object(deploy, "clone_apps", return_value=apps_dir), \
patch.object(deploy, "commit_and_push"), \
patch.object(deploy, "publish_route_immutables",
side_effect=lambda artifact, *_a, **_k: immutables.append(artifact["name"])), \
patch.object(deploy, "reconcile_artifact_cors",
side_effect=lambda artifacts, *_a: cors.extend(item["name"] for item in artifacts)), \
patch.object(deploy, "s3_sync",
side_effect=lambda artifact, *_a, **_k: synced.append(artifact["name"])), \
redirect_stdout(io.StringIO()):
deploy.deploy_static("baseline.fritzlab.net", root, ROOT, "token", cfg)
files = {path.relative_to(app_dir).as_posix(): path.read_text()
for path in sorted(app_dir.rglob("*.yaml"))}
return {"synced": synced, "immutables": immutables, "cors": cors, "files": files}
def test_selected_publication_leaves_the_whole_route_contract_rendered(self):
# Only the portal credentials exist: a scoped run must not demand the
# secrets of an artifact it is not publishing.
environment = {"PORTAL_S3_ACCESS_KEY": "portal-key",
"PORTAL_S3_SECRET_KEY": "portal-secret"}
for name in ("DIST_S3_ACCESS_KEY", "DIST_S3_SECRET_KEY"):
os.environ.pop(name, None)
result = self.deploy_selection(["portal"], environment)
self.assertEqual(["portal"], result["synced"])
self.assertEqual(["portal"], result["immutables"])
self.assertEqual(["portal"], result["cors"])
self.assertIn("manifests/ingress-distributions.yaml", result["files"])
self.assertIn("manifests/service-distributions.yaml", result["files"])
history = yaml.safe_load(result["files"]["site-publish-history.yaml"])
self.assertEqual({"baseline-dist", "baseline-portal"}, set(history["buckets"]))
self.assertEqual(
["dist/releases"], history["buckets"]["baseline-dist"]["immutable_paths"]
)
def test_disabled_site_refuses_a_partial_selection(self):
cfg = normalize_site_config(fixture("split-site.yaml"), "baseline.fritzlab.net")
cfg["enabled"] = False
cfg["selected"] = ["portal"]
with patch.object(deploy, "parse_site_yaml", return_value=cfg), \
patch.dict(os.environ, {
"SITE_REPO": "fritzlab/baseline.fritzlab.net", "SITE_DIR": ".",
"ACTION_DIR": str(ROOT), "CI_BOT_TOKEN": "token",
}, clear=False), \
patch.object(deploy, "decommission") as decommission, \
redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \
self.assertRaises(SystemExit):
deploy.cmd_deploy()
decommission.assert_not_called()
class PublishingTests(unittest.TestCase): class PublishingTests(unittest.TestCase):
def test_cors_policy_is_reconciled_as_read_only_browser_access(self): def test_cors_policy_is_reconciled_as_read_only_browser_access(self):
captured = {} captured = {}