[bug-0zkpwxbsdxcn] Own canonical offline private Go module tooling #2

Merged
dfritz merged 7 commits from architect/bug-0zkpwxbsdxcn/private-module-tooling into main 2026-09-09 02:15:55 +00:00
7 changed files with 626 additions and 0 deletions
Showing only changes of commit 320cab62a0 - Show all commits
+14
View File
@@ -0,0 +1,14 @@
name: Private module tooling
on:
pull_request:
push:
branches: [main]
jobs:
contract:
runs-on: fritzlab
steps:
- uses: actions/checkout@v4
- name: Canonical export and native offline boundaries
run: |
python3 -m unittest discover -s tests
GO111MODULE=off GOTOOLCHAIN=local GOFLAGS= go test -race ./tools/private-modules
+1
View File
@@ -0,0 +1 @@
__pycache__/
+34
View File
@@ -9,6 +9,40 @@ PR build that pulls a private base image still needs a registry token limited to
the `read:package` capability; public-base builds need no token. The action logs
in as `ci-bot`, so the token must be issued to that account.
## Private Go module builds
`tools/private-modules/` owns the stdlib-only offline generator and verifier for
native Go file-proxy bundles. Consumers export exact committed source bytes so
their first check needs no Git, network, module dependency, or credential.
Changes belong here; consumers must not edit their generated copies.
From this checkout, export a reviewed full commit ID into a service checkout:
```sh
python3 tools/export-private-modules.py /path/to/service --revision FULL_COMMIT_ID
python3 tools/export-private-modules.py /path/to/service --revision FULL_COMMIT_ID --check
```
The export includes source revision/path/digest metadata and a SHA256 lock.
Service Make and Docker builds run `sha256sum -c tools/private-modules.sha256`
before `GO111MODULE=off GOTOOLCHAIN=local GOFLAGS= go run tools/private-modules.go`.
Reviewers can reproduce `--check` from that pinned source commit. The local hash
check detects accidental drift; Git review establishes the trusted source pin.
Populate selected private versions through the configured authenticated Go
client once, then run the generated command with `-write` to copy unchanged
`.info`, `.mod`, and `.zip` cache artifacts into `third_party/go-proxy`. The
generator itself enforces offline resolution and a local toolchain. It preserves
the complete original archives, including any licenses and notices. Normal Go
downloads still validate their content against the consumer's `go.sum`.
Consumer resolution uses `GONOPROXY=none`, `GONOSUMDB=code.fritzlab.net`,
`GOFLAGS=-mod=readonly`, `GOTOOLCHAIN=local`, and
`GOPROXY=file:///absolute/service/third_party/go-proxy,https://proxy.golang.org`.
Run the checker before and after downloads; its exact go.mod/go.sum fingerprint
rejects stale dependency closures. Public modules stay on the public proxy.
Registry publication remains a separate authenticated operation.
## Usage
```yaml
+48
View File
@@ -0,0 +1,48 @@
import importlib.util
import pathlib
import subprocess
import tempfile
import unittest
ROOT = pathlib.Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("exporter", ROOT / "tools/export-private-modules.py")
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class ExportTest(unittest.TestCase):
def test_committed_bytes_and_provenance_are_exact(self):
with tempfile.TemporaryDirectory() as temporary:
base = pathlib.Path(temporary)
source, target = base / "source", base / "target"
source.mkdir()
target.mkdir()
subprocess.run(["git", "init", "-q", str(source)], check=True)
for path in MODULE.FILES:
file = source / path
file.parent.mkdir(parents=True, exist_ok=True)
file.write_text("package main\n")
subprocess.run(["git", "-C", str(source), "add", "."], check=True)
subprocess.run(["git", "-C", str(source), "-c", "user.name=Fixture",
"-c", "user.email=fixture@example.invalid", "commit", "-qm", "fixture"], check=True)
revision = MODULE.committed(source, "rev-parse", "HEAD").decode().strip()
(source / next(iter(MODULE.FILES))).write_text("dirty worktree must be ignored")
MODULE.export(source, target, revision)
MODULE.export(source, target, revision, True)
subprocess.run(["sha256sum", "-c", "tools/private-modules.sha256"],
cwd=target, check=True, stdout=subprocess.DEVNULL)
destination = target / "tools/private-modules.go"
self.assertEqual(destination.read_text(), "package main\n")
destination.write_text("changed")
with self.assertRaises(ValueError):
MODULE.export(source, target, revision, True)
destination.unlink()
destination.symlink_to(source / next(iter(MODULE.FILES)))
with self.assertRaises(ValueError):
MODULE.export(source, target, revision)
with self.assertRaises(ValueError):
MODULE.export(source, target, "HEAD")
if __name__ == "__main__":
unittest.main()
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Export exact committed offline-bootstrap tooling; never download anything."""
import argparse
import hashlib
import json
import pathlib
import re
import subprocess
SOURCE = "https://code.fritzlab.net/action/image-build"
FILES = {
"tools/private-modules/main.go": "tools/private-modules.go",
"tools/private-modules/main_test.go": "tools/private-modules_test.go",
}
def committed(root, *args):
return subprocess.check_output(["git", "-C", str(root), *args])
def export(root, target, revision, check=False):
if not re.fullmatch(r"[0-9a-f]{40}", revision):
raise ValueError("source revision must be a full Git commit ID")
resolved = committed(root, "rev-parse", revision + "^{commit}").decode().strip()
if resolved != revision:
raise ValueError("source revision is not a commit")
result = {}
entries = []
for source, destination in FILES.items():
tree = committed(root, "ls-tree", revision, "--", source).decode()
if not tree.startswith("100644 blob ") or not tree.endswith("\t" + source + "\n"):
raise ValueError("source must be one regular committed file")
data = committed(root, "show", revision + ":" + source)
if not data or len(data) > 128 * 1024:
raise ValueError("source exceeds its bounded export")
result[destination] = data
entries.append({"source": source, "destination": destination,
"sha256": hashlib.sha256(data).hexdigest()})
result["tools/private-modules.source.json"] = (json.dumps({
"version": 1, "repository": SOURCE, "revision": revision, "files": entries,
}, indent=2) + "\n").encode()
result["tools/private-modules.sha256"] = "".join(
hashlib.sha256(data).hexdigest() + " " + path + "\n"
for path, data in sorted(result.items())
).encode()
target = target.resolve(strict=True)
tools_dir = target / "tools"
if tools_dir.is_symlink():
raise ValueError("target tools directory must not be a symlink")
if not check:
tools_dir.mkdir(exist_ok=True)
for path, data in result.items():
destination = target / path
if destination.is_symlink():
raise ValueError("target file must not be a symlink")
if check:
if destination.read_bytes() != data:
raise ValueError("export differs from pinned canonical source: " + path)
else:
destination.write_bytes(data)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("target", type=pathlib.Path)
parser.add_argument("--revision", required=True)
parser.add_argument("--check", action="store_true")
args = parser.parse_args()
export(pathlib.Path(__file__).resolve().parents[1], args.target,
args.revision, args.check)
if __name__ == "__main__":
main()
+281
View File
@@ -0,0 +1,281 @@
// Command private-modules maintains exact private Go proxy artifacts from an
// already authenticated local module cache. It never downloads dependencies.
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
)
const bundleRoot = "third_party/go-proxy"
const privatePrefix = "code.fritzlab.net/"
type artifact struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Size int64 `json:"size_bytes"`
}
type module struct {
Path string `json:"path"`
Version string `json:"version"`
Files []artifact `json:"files"`
}
type manifest struct {
Version int `json:"version"`
GoModSHA256 string `json:"go_mod_sha256"`
GoSumSHA256 string `json:"go_sum_sha256"`
Modules []module `json:"modules"`
}
func digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
func inputDigest(name string) (string, error) {
b, e := os.ReadFile(name)
if e != nil {
return "", e
}
return digest(b), nil
}
func nativeGo(args ...string) ([]byte, error) {
c := exec.Command("go", args...)
c.Env = append(os.Environ(), "GOPROXY=off", "GONOPROXY=none", "GOTOOLCHAIN=local", "GOSUMDB=off", "GO111MODULE=on", "GOFLAGS=", "GOWORK=off")
return c.Output()
}
func generate() error {
m := manifest{Version: 1, Modules: []module{}}
var err error
if m.GoModSHA256, err = inputDigest("go.mod"); err != nil {
return err
}
if m.GoSumSHA256, err = inputDigest("go.sum"); err != nil {
return err
}
cache, err := nativeGo("env", "GOMODCACHE")
if err != nil {
return err
}
cacheRoot := filepath.Join(strings.TrimSpace(string(cache)), "cache", "download")
raw, err := nativeGo("list", "-m", "-json", "all")
if err != nil {
return fmt.Errorf("module metadata must already be cached: %w", err)
}
parent := filepath.Dir(bundleRoot)
if err = os.MkdirAll(parent, 0755); err != nil {
return err
}
temp, err := os.MkdirTemp(parent, ".go-proxy-")
if err != nil {
return err
}
defer os.RemoveAll(temp)
decoder := json.NewDecoder(bytes.NewReader(raw))
var total int64
for {
var src struct {
Path, Version, GoMod string
Main bool
Replace json.RawMessage
}
if err = decoder.Decode(&src); errors.Is(err, io.EOF) {
break
} else if err != nil {
return err
}
if src.Main || !strings.HasPrefix(src.Path, privatePrefix) {
continue
}
if len(src.Replace) != 0 || src.Version == "" || len(m.Modules) >= 64 {
return errors.New("private module must have exact unreplaced version")
}
mod := module{Path: src.Path, Version: src.Version, Files: []artifact{}}
rel, err := filepath.Rel(cacheRoot, src.GoMod)
if err != nil || !safePath(rel) || !strings.HasSuffix(rel, ".mod") || !strings.HasSuffix(filepath.Dir(rel), "/@v") {
return errors.New("unexpected native module cache layout")
}
stem, err := moduleStem(src.Path, src.Version)
if err != nil || filepath.ToSlash(rel) != stem+".mod" {
return errors.New("native artifact path differs from module identity")
}
for _, ext := range []string{".info", ".mod", ".zip"} {
name := stem + ext
source := filepath.Join(cacheRoot, name)
info, err := os.Lstat(source)
if err != nil || !info.Mode().IsRegular() || info.Size() > 64<<20 {
return errors.New("private module artifact absent or invalid; populate authenticated cache first")
}
total += info.Size()
if total > 256<<20 {
return errors.New("private module bundle too large")
}
b, err := os.ReadFile(source)
if err != nil {
return err
}
target := filepath.Join(temp, name)
if err = os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
}
if err = os.WriteFile(target, b, 0444); err != nil {
return err
}
mod.Files = append(mod.Files, artifact{Path: filepath.ToSlash(name), SHA256: digest(b), Size: int64(len(b))})
}
m.Modules = append(m.Modules, mod)
}
sort.Slice(m.Modules, func(i, j int) bool { return m.Modules[i].Path < m.Modules[j].Path })
if len(m.Modules) == 0 {
return errors.New("no private modules")
}
encoded, err := json.MarshalIndent(m, "", " ")
if err != nil {
return err
}
if err = os.WriteFile(filepath.Join(temp, "manifest.json"), append(encoded, '\n'), 0444); err != nil {
return err
}
// This path is exclusively generated. Refuse to replace an unrelated tree.
if _, err = os.Stat(bundleRoot); err == nil {
if _, err = os.Stat(filepath.Join(bundleRoot, "manifest.json")); err != nil {
return errors.New("refuse to replace unrecognized bundle")
}
if err = os.RemoveAll(bundleRoot); err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
return os.Rename(temp, bundleRoot)
}
var modulePath = regexp.MustCompile(`^code[.]fritzlab[.]net/[A-Za-z0-9._~-]+(/[A-Za-z0-9._~-]+)+$`)
var moduleVersion = regexp.MustCompile(`^v[0-9]+[.][0-9]+[.][0-9]+(-[0-9A-Za-z.-]+)?([+]incompatible)?$`)
// Standard Go proxy escaping replaces each ASCII capital with !lowercase.
func moduleStem(path, version string) (string, error) {
if len(path) > 512 || len(version) > 256 || !modulePath.MatchString(path) || !moduleVersion.MatchString(version) {
return "", errors.New("invalid private module identity")
}
for _, part := range strings.Split(path, "/") {
if part == "." || part == ".." {
return "", errors.New("invalid private module path")
}
}
escape := func(value string) string {
var out strings.Builder
for _, r := range value {
if r >= 'A' && r <= 'Z' {
out.WriteByte('!')
r += 'a' - 'A'
}
out.WriteRune(r)
}
return out.String()
}
return escape(path) + "/@v/" + escape(version), nil
}
func safePath(p string) bool {
return p != "" && filepath.Clean(p) == p && !filepath.IsAbs(p) && p != ".." && !strings.HasPrefix(p, "../") && !strings.ContainsAny(p, "\\\x00")
}
func check() error {
raw, err := os.ReadFile(filepath.Join(bundleRoot, "manifest.json"))
if err != nil || len(raw) > 1<<20 {
return errors.New("private module manifest missing or oversized")
}
var m manifest
d := json.NewDecoder(bytes.NewReader(raw))
d.DisallowUnknownFields()
if err = d.Decode(&m); err != nil || m.Version != 1 || len(m.Modules) == 0 || len(m.Modules) > 64 {
return errors.New("invalid private module manifest")
}
var trailing any
if err = d.Decode(&trailing); !errors.Is(err, io.EOF) {
return errors.New("trailing private module manifest data")
}
mod, err := inputDigest("go.mod")
if err != nil {
return err
}
sum, err := inputDigest("go.sum")
if err != nil {
return err
}
if mod != m.GoModSHA256 || sum != m.GoSumSHA256 {
return errors.New("private module closure stale; regenerate after go.mod/go.sum changes")
}
expected := map[string]bool{"manifest.json": true}
last := ""
var total int64
for _, m := range m.Modules {
if !strings.HasPrefix(m.Path, privatePrefix) || m.Path <= last || m.Version == "" || len(m.Files) != 3 {
return errors.New("invalid private module closure")
}
last = m.Path
stem, err := moduleStem(m.Path, m.Version)
if err != nil {
return err
}
for i, f := range m.Files {
if !safePath(f.Path) || expected[f.Path] || f.Path != stem+[]string{".info", ".mod", ".zip"}[i] || f.Size < 0 || f.Size > 64<<20 {
return errors.New("invalid module artifact")
}
total += f.Size
if total > 256<<20 {
return errors.New("private module bundle too large")
}
full := filepath.Join(bundleRoot, f.Path)
info, e := os.Lstat(full)
if e != nil || !info.Mode().IsRegular() || info.Size() != f.Size {
return errors.New("private module artifact missing or invalid")
}
b, e := os.ReadFile(full)
if e != nil || digest(b) != f.SHA256 {
return errors.New("private module artifact digest mismatch")
}
expected[f.Path] = true
}
}
err = filepath.WalkDir(bundleRoot, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.Type()&os.ModeSymlink != 0 {
return errors.New("symlink in private module bundle")
}
if entry.IsDir() {
return nil
}
rel, e := filepath.Rel(bundleRoot, path)
if e != nil || !expected[filepath.ToSlash(rel)] {
return errors.New("unexpected private module artifact")
}
return nil
})
return err
}
func main() {
write := flag.Bool("write", false, "regenerate from authenticated local module cache (no network)")
flag.Parse()
var err error
if *write {
err = generate()
} else {
err = check()
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Println("private module artifacts verified")
}
+174
View File
@@ -0,0 +1,174 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
func TestBundleRejectsStaleMissingModifiedAndUnexpectedArtifacts(t *testing.T) {
t.Chdir(t.TempDir())
os.WriteFile("go.mod", []byte("module example.invalid/test\n"), 0600)
os.WriteFile("go.sum", []byte("recorded sum\n"), 0600)
mod, _ := inputDigest("go.mod")
sum, _ := inputDigest("go.sum")
m := manifest{Version: 1, GoModSHA256: mod, GoSumSHA256: sum, Modules: []module{{Path: "code.fritzlab.net/agenthub/example", Version: "v0.1.0"}}}
for _, ext := range []string{".info", ".mod", ".zip"} {
name := "code.fritzlab.net/agenthub/example/@v/v0.1.0" + ext
b := []byte("canonical artifact " + ext)
os.MkdirAll(filepath.Dir(filepath.Join(bundleRoot, name)), 0700)
os.WriteFile(filepath.Join(bundleRoot, name), b, 0600)
m.Modules[0].Files = append(m.Modules[0].Files, artifact{Path: name, Size: int64(len(b)), SHA256: digest(b)})
}
b, _ := json.Marshal(m)
os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), b, 0600)
if err := check(); err != nil {
t.Fatal(err)
}
originalPath, originalVersion := m.Modules[0].Path, m.Modules[0].Version
for _, change := range []string{"path", "version"} {
if change == "path" {
m.Modules[0].Path = "code.fritzlab.net/agenthub/other"
} else {
m.Modules[0].Version = "v0.2.0"
}
changed, _ := json.Marshal(m)
os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), changed, 0600)
if check() == nil {
t.Fatal("artifact accepted under different module identity")
}
m.Modules[0].Path, m.Modules[0].Version = originalPath, originalVersion
}
os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), b, 0600)
os.WriteFile("go.mod", []byte("changed graph\n"), 0600)
if check() == nil {
t.Fatal("stale dependency closure accepted")
}
os.WriteFile("go.mod", []byte("module example.invalid/test\n"), 0600)
path := filepath.Join(bundleRoot, m.Modules[0].Files[2].Path)
original, _ := os.ReadFile(path)
os.WriteFile(path, []byte("corrupted zip"), 0600)
if check() == nil {
t.Fatal("modified artifact accepted")
}
os.Remove(path)
if check() == nil {
t.Fatal("missing artifact accepted")
}
os.WriteFile(path, original, 0600)
extra := filepath.Join(bundleRoot, "extra")
os.WriteFile(extra, []byte("unlisted"), 0600)
if check() == nil {
t.Fatal("unlisted artifact accepted")
}
os.Remove(extra)
os.Symlink("manifest.json", extra)
if check() == nil {
t.Fatal("symlink accepted")
}
}
func TestNativeGoSumRejectsChangedModuleArtifact(t *testing.T) {
repository, err := filepath.Abs("..")
if err != nil {
t.Fatal(err)
}
var m manifest
raw, err := os.ReadFile(filepath.Join(repository, bundleRoot, "manifest.json"))
if err != nil {
t.Fatal(err)
}
if err = json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
selected := m.Modules[0]
root := t.TempDir()
proxy := filepath.Join(root, "proxy")
for _, f := range selected.Files {
b, err := os.ReadFile(filepath.Join(repository, bundleRoot, f.Path))
if err != nil {
t.Fatal(err)
}
target := filepath.Join(proxy, f.Path)
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
t.Fatal(err)
}
if strings.HasSuffix(f.Path, ".mod") {
b = append(b, '\n')
}
if err = os.WriteFile(target, b, 0600); err != nil {
t.Fatal(err)
}
}
sum, err := os.ReadFile(filepath.Join(repository, "go.sum"))
if err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(root, "go.sum"), sum, 0600)
os.WriteFile(filepath.Join(root, "go.mod"), []byte("module example.invalid/checksum\n\ngo 1.27.0\n"), 0600)
cmd := exec.Command("go", "mod", "download", selected.Path+"@"+selected.Version)
cmd.Dir = root
cmd.Env = append(os.Environ(), "GOMODCACHE="+filepath.Join(root, "cache"), "GOPROXY=file://"+proxy, "GONOPROXY=none", "GONOSUMDB="+privatePrefix, "GOSUMDB=off", "GOTOOLCHAIN=local", "GOFLAGS=", "GOWORK=off")
output, err := cmd.CombinedOutput()
if err == nil || !bytes.Contains(output, []byte("checksum mismatch")) {
t.Fatalf("native module checksum guard failed: %v: %s", err, output)
}
}
func TestGeneratorMissingPrivateMetadataNeverInvokesGit(t *testing.T) {
root := t.TempDir()
t.Chdir(root)
os.WriteFile("go.mod", []byte("module example.invalid/offline\n\ngo 1.27.0\n\nrequire code.fritzlab.net/agenthub/missing v0.0.1\n"), 0600)
os.WriteFile("go.sum", nil, 0600)
bin := filepath.Join(root, "bin")
os.Mkdir(bin, 0700)
marker := filepath.Join(root, "git-invoked")
os.WriteFile(filepath.Join(bin, "git"), []byte("#!/bin/sh\nprintf invoked > \"$PRIVATE_GIT_PROBE\"\nexit 99\n"), 0700)
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("PRIVATE_GIT_PROBE", marker)
t.Setenv("GOMODCACHE", filepath.Join(root, "empty-cache"))
t.Setenv("GOPRIVATE", "code.fritzlab.net")
t.Setenv("GONOPROXY", "code.fritzlab.net")
t.Setenv("GOTOOLCHAIN", "auto")
err := generate()
if err == nil {
t.Fatal("missing cached module accepted")
}
var failure *exec.ExitError
if !errors.As(err, &failure) || !bytes.Contains(failure.Stderr, []byte("module lookup disabled by GOPROXY=off")) {
t.Fatalf("not a proven offline refusal: %v", err)
}
if _, err = os.Stat(marker); !os.IsNotExist(err) {
t.Fatal("Git invoked during offline generation")
}
if _, err = os.Stat(bundleRoot); !os.IsNotExist(err) {
t.Fatal("failed generation published output")
}
os.WriteFile("go.mod", []byte("module example.invalid/offline\n\ngo 1.999.0\n"), 0600)
_, err = nativeGo("list", "-m", "-json", "all")
if !errors.As(err, &failure) || !bytes.Contains(failure.Stderr, []byte("GOTOOLCHAIN=local")) {
t.Fatalf("toolchain download not disabled: %v", err)
}
}
func TestCanonicalModuleArtifactPathBinding(t *testing.T) {
for _, test := range []struct{ path, version, want string }{
{"code.fritzlab.net/agenthub/Provider", "v1.2.3-RC.1", "code.fritzlab.net/agenthub/!provider/@v/v1.2.3-!r!c.1"},
{"code.fritzlab.net/agenthub/provider", "v2.0.0+incompatible", "code.fritzlab.net/agenthub/provider/@v/v2.0.0+incompatible"},
} {
got, err := moduleStem(test.path, test.version)
if err != nil || got != test.want {
t.Fatalf("stem %q %v", got, err)
}
}
for _, path := range []string{"code.fritzlab.net/agenthub/../provider", "code.fritzlab.net/agenthub/provider!", "other.invalid/agenthub/provider"} {
if _, err := moduleStem(path, "v1.0.0"); err == nil {
t.Fatal("invalid path accepted")
}
}
}