Own canonical offline private Go module build tooling
Authored-By: Codex (GPT-6) <noreply@openai.com>
This commit is contained in:
Evelyn Chen
@@ -0,0 +1,281 @@
|
||||
// Command private-modules maintains exact private Go proxy artifacts from an
|
||||
// already authenticated local module cache. It never downloads dependencies.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const bundleRoot = "third_party/go-proxy"
|
||||
const privatePrefix = "code.fritzlab.net/"
|
||||
|
||||
type artifact struct {
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size_bytes"`
|
||||
}
|
||||
type module struct {
|
||||
Path string `json:"path"`
|
||||
Version string `json:"version"`
|
||||
Files []artifact `json:"files"`
|
||||
}
|
||||
type manifest struct {
|
||||
Version int `json:"version"`
|
||||
GoModSHA256 string `json:"go_mod_sha256"`
|
||||
GoSumSHA256 string `json:"go_sum_sha256"`
|
||||
Modules []module `json:"modules"`
|
||||
}
|
||||
|
||||
func digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
func inputDigest(name string) (string, error) {
|
||||
b, e := os.ReadFile(name)
|
||||
if e != nil {
|
||||
return "", e
|
||||
}
|
||||
return digest(b), nil
|
||||
}
|
||||
func nativeGo(args ...string) ([]byte, error) {
|
||||
c := exec.Command("go", args...)
|
||||
c.Env = append(os.Environ(), "GOPROXY=off", "GONOPROXY=none", "GOTOOLCHAIN=local", "GOSUMDB=off", "GO111MODULE=on", "GOFLAGS=", "GOWORK=off")
|
||||
return c.Output()
|
||||
}
|
||||
func generate() error {
|
||||
m := manifest{Version: 1, Modules: []module{}}
|
||||
var err error
|
||||
if m.GoModSHA256, err = inputDigest("go.mod"); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.GoSumSHA256, err = inputDigest("go.sum"); err != nil {
|
||||
return err
|
||||
}
|
||||
cache, err := nativeGo("env", "GOMODCACHE")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cacheRoot := filepath.Join(strings.TrimSpace(string(cache)), "cache", "download")
|
||||
raw, err := nativeGo("list", "-m", "-json", "all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("module metadata must already be cached: %w", err)
|
||||
}
|
||||
parent := filepath.Dir(bundleRoot)
|
||||
if err = os.MkdirAll(parent, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
temp, err := os.MkdirTemp(parent, ".go-proxy-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(temp)
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
var total int64
|
||||
for {
|
||||
var src struct {
|
||||
Path, Version, GoMod string
|
||||
Main bool
|
||||
Replace json.RawMessage
|
||||
}
|
||||
if err = decoder.Decode(&src); errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if src.Main || !strings.HasPrefix(src.Path, privatePrefix) {
|
||||
continue
|
||||
}
|
||||
if len(src.Replace) != 0 || src.Version == "" || len(m.Modules) >= 64 {
|
||||
return errors.New("private module must have exact unreplaced version")
|
||||
}
|
||||
mod := module{Path: src.Path, Version: src.Version, Files: []artifact{}}
|
||||
rel, err := filepath.Rel(cacheRoot, src.GoMod)
|
||||
if err != nil || !safePath(rel) || !strings.HasSuffix(rel, ".mod") || !strings.HasSuffix(filepath.Dir(rel), "/@v") {
|
||||
return errors.New("unexpected native module cache layout")
|
||||
}
|
||||
stem, err := moduleStem(src.Path, src.Version)
|
||||
if err != nil || filepath.ToSlash(rel) != stem+".mod" {
|
||||
return errors.New("native artifact path differs from module identity")
|
||||
}
|
||||
for _, ext := range []string{".info", ".mod", ".zip"} {
|
||||
name := stem + ext
|
||||
source := filepath.Join(cacheRoot, name)
|
||||
info, err := os.Lstat(source)
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() > 64<<20 {
|
||||
return errors.New("private module artifact absent or invalid; populate authenticated cache first")
|
||||
}
|
||||
total += info.Size()
|
||||
if total > 256<<20 {
|
||||
return errors.New("private module bundle too large")
|
||||
}
|
||||
b, err := os.ReadFile(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := filepath.Join(temp, name)
|
||||
if err = os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.WriteFile(target, b, 0444); err != nil {
|
||||
return err
|
||||
}
|
||||
mod.Files = append(mod.Files, artifact{Path: filepath.ToSlash(name), SHA256: digest(b), Size: int64(len(b))})
|
||||
}
|
||||
m.Modules = append(m.Modules, mod)
|
||||
}
|
||||
sort.Slice(m.Modules, func(i, j int) bool { return m.Modules[i].Path < m.Modules[j].Path })
|
||||
if len(m.Modules) == 0 {
|
||||
return errors.New("no private modules")
|
||||
}
|
||||
encoded, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(temp, "manifest.json"), append(encoded, '\n'), 0444); err != nil {
|
||||
return err
|
||||
}
|
||||
// This path is exclusively generated. Refuse to replace an unrelated tree.
|
||||
if _, err = os.Stat(bundleRoot); err == nil {
|
||||
if _, err = os.Stat(filepath.Join(bundleRoot, "manifest.json")); err != nil {
|
||||
return errors.New("refuse to replace unrecognized bundle")
|
||||
}
|
||||
if err = os.RemoveAll(bundleRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temp, bundleRoot)
|
||||
}
|
||||
|
||||
var modulePath = regexp.MustCompile(`^code[.]fritzlab[.]net/[A-Za-z0-9._~-]+(/[A-Za-z0-9._~-]+)+$`)
|
||||
var moduleVersion = regexp.MustCompile(`^v[0-9]+[.][0-9]+[.][0-9]+(-[0-9A-Za-z.-]+)?([+]incompatible)?$`)
|
||||
|
||||
// Standard Go proxy escaping replaces each ASCII capital with !lowercase.
|
||||
func moduleStem(path, version string) (string, error) {
|
||||
if len(path) > 512 || len(version) > 256 || !modulePath.MatchString(path) || !moduleVersion.MatchString(version) {
|
||||
return "", errors.New("invalid private module identity")
|
||||
}
|
||||
for _, part := range strings.Split(path, "/") {
|
||||
if part == "." || part == ".." {
|
||||
return "", errors.New("invalid private module path")
|
||||
}
|
||||
}
|
||||
escape := func(value string) string {
|
||||
var out strings.Builder
|
||||
for _, r := range value {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
out.WriteByte('!')
|
||||
r += 'a' - 'A'
|
||||
}
|
||||
out.WriteRune(r)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
return escape(path) + "/@v/" + escape(version), nil
|
||||
}
|
||||
|
||||
func safePath(p string) bool {
|
||||
return p != "" && filepath.Clean(p) == p && !filepath.IsAbs(p) && p != ".." && !strings.HasPrefix(p, "../") && !strings.ContainsAny(p, "\\\x00")
|
||||
}
|
||||
func check() error {
|
||||
raw, err := os.ReadFile(filepath.Join(bundleRoot, "manifest.json"))
|
||||
if err != nil || len(raw) > 1<<20 {
|
||||
return errors.New("private module manifest missing or oversized")
|
||||
}
|
||||
var m manifest
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.DisallowUnknownFields()
|
||||
if err = d.Decode(&m); err != nil || m.Version != 1 || len(m.Modules) == 0 || len(m.Modules) > 64 {
|
||||
return errors.New("invalid private module manifest")
|
||||
}
|
||||
var trailing any
|
||||
if err = d.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return errors.New("trailing private module manifest data")
|
||||
}
|
||||
mod, err := inputDigest("go.mod")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum, err := inputDigest("go.sum")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mod != m.GoModSHA256 || sum != m.GoSumSHA256 {
|
||||
return errors.New("private module closure stale; regenerate after go.mod/go.sum changes")
|
||||
}
|
||||
expected := map[string]bool{"manifest.json": true}
|
||||
last := ""
|
||||
var total int64
|
||||
for _, m := range m.Modules {
|
||||
if !strings.HasPrefix(m.Path, privatePrefix) || m.Path <= last || m.Version == "" || len(m.Files) != 3 {
|
||||
return errors.New("invalid private module closure")
|
||||
}
|
||||
last = m.Path
|
||||
stem, err := moduleStem(m.Path, m.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, f := range m.Files {
|
||||
if !safePath(f.Path) || expected[f.Path] || f.Path != stem+[]string{".info", ".mod", ".zip"}[i] || f.Size < 0 || f.Size > 64<<20 {
|
||||
return errors.New("invalid module artifact")
|
||||
}
|
||||
total += f.Size
|
||||
if total > 256<<20 {
|
||||
return errors.New("private module bundle too large")
|
||||
}
|
||||
full := filepath.Join(bundleRoot, f.Path)
|
||||
info, e := os.Lstat(full)
|
||||
if e != nil || !info.Mode().IsRegular() || info.Size() != f.Size {
|
||||
return errors.New("private module artifact missing or invalid")
|
||||
}
|
||||
b, e := os.ReadFile(full)
|
||||
if e != nil || digest(b) != f.SHA256 {
|
||||
return errors.New("private module artifact digest mismatch")
|
||||
}
|
||||
expected[f.Path] = true
|
||||
}
|
||||
}
|
||||
err = filepath.WalkDir(bundleRoot, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return errors.New("symlink in private module bundle")
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, e := filepath.Rel(bundleRoot, path)
|
||||
if e != nil || !expected[filepath.ToSlash(rel)] {
|
||||
return errors.New("unexpected private module artifact")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
func main() {
|
||||
write := flag.Bool("write", false, "regenerate from authenticated local module cache (no network)")
|
||||
flag.Parse()
|
||||
var err error
|
||||
if *write {
|
||||
err = generate()
|
||||
} else {
|
||||
err = check()
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("private module artifacts verified")
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBundleRejectsStaleMissingModifiedAndUnexpectedArtifacts(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
os.WriteFile("go.mod", []byte("module example.invalid/test\n"), 0600)
|
||||
os.WriteFile("go.sum", []byte("recorded sum\n"), 0600)
|
||||
mod, _ := inputDigest("go.mod")
|
||||
sum, _ := inputDigest("go.sum")
|
||||
m := manifest{Version: 1, GoModSHA256: mod, GoSumSHA256: sum, Modules: []module{{Path: "code.fritzlab.net/agenthub/example", Version: "v0.1.0"}}}
|
||||
for _, ext := range []string{".info", ".mod", ".zip"} {
|
||||
name := "code.fritzlab.net/agenthub/example/@v/v0.1.0" + ext
|
||||
b := []byte("canonical artifact " + ext)
|
||||
os.MkdirAll(filepath.Dir(filepath.Join(bundleRoot, name)), 0700)
|
||||
os.WriteFile(filepath.Join(bundleRoot, name), b, 0600)
|
||||
m.Modules[0].Files = append(m.Modules[0].Files, artifact{Path: name, Size: int64(len(b)), SHA256: digest(b)})
|
||||
}
|
||||
b, _ := json.Marshal(m)
|
||||
os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), b, 0600)
|
||||
if err := check(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originalPath, originalVersion := m.Modules[0].Path, m.Modules[0].Version
|
||||
for _, change := range []string{"path", "version"} {
|
||||
if change == "path" {
|
||||
m.Modules[0].Path = "code.fritzlab.net/agenthub/other"
|
||||
} else {
|
||||
m.Modules[0].Version = "v0.2.0"
|
||||
}
|
||||
changed, _ := json.Marshal(m)
|
||||
os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), changed, 0600)
|
||||
if check() == nil {
|
||||
t.Fatal("artifact accepted under different module identity")
|
||||
}
|
||||
m.Modules[0].Path, m.Modules[0].Version = originalPath, originalVersion
|
||||
}
|
||||
os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), b, 0600)
|
||||
os.WriteFile("go.mod", []byte("changed graph\n"), 0600)
|
||||
if check() == nil {
|
||||
t.Fatal("stale dependency closure accepted")
|
||||
}
|
||||
os.WriteFile("go.mod", []byte("module example.invalid/test\n"), 0600)
|
||||
path := filepath.Join(bundleRoot, m.Modules[0].Files[2].Path)
|
||||
original, _ := os.ReadFile(path)
|
||||
os.WriteFile(path, []byte("corrupted zip"), 0600)
|
||||
if check() == nil {
|
||||
t.Fatal("modified artifact accepted")
|
||||
}
|
||||
os.Remove(path)
|
||||
if check() == nil {
|
||||
t.Fatal("missing artifact accepted")
|
||||
}
|
||||
os.WriteFile(path, original, 0600)
|
||||
extra := filepath.Join(bundleRoot, "extra")
|
||||
os.WriteFile(extra, []byte("unlisted"), 0600)
|
||||
if check() == nil {
|
||||
t.Fatal("unlisted artifact accepted")
|
||||
}
|
||||
os.Remove(extra)
|
||||
os.Symlink("manifest.json", extra)
|
||||
if check() == nil {
|
||||
t.Fatal("symlink accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeGoSumRejectsChangedModuleArtifact(t *testing.T) {
|
||||
repository, err := filepath.Abs("..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
raw, err := os.ReadFile(filepath.Join(repository, bundleRoot, "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected := m.Modules[0]
|
||||
root := t.TempDir()
|
||||
proxy := filepath.Join(root, "proxy")
|
||||
for _, f := range selected.Files {
|
||||
b, err := os.ReadFile(filepath.Join(repository, bundleRoot, f.Path))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(proxy, f.Path)
|
||||
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.HasSuffix(f.Path, ".mod") {
|
||||
b = append(b, '\n')
|
||||
}
|
||||
if err = os.WriteFile(target, b, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sum, err := os.ReadFile(filepath.Join(repository, "go.sum"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.WriteFile(filepath.Join(root, "go.sum"), sum, 0600)
|
||||
os.WriteFile(filepath.Join(root, "go.mod"), []byte("module example.invalid/checksum\n\ngo 1.27.0\n"), 0600)
|
||||
cmd := exec.Command("go", "mod", "download", selected.Path+"@"+selected.Version)
|
||||
cmd.Dir = root
|
||||
cmd.Env = append(os.Environ(), "GOMODCACHE="+filepath.Join(root, "cache"), "GOPROXY=file://"+proxy, "GONOPROXY=none", "GONOSUMDB="+privatePrefix, "GOSUMDB=off", "GOTOOLCHAIN=local", "GOFLAGS=", "GOWORK=off")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err == nil || !bytes.Contains(output, []byte("checksum mismatch")) {
|
||||
t.Fatalf("native module checksum guard failed: %v: %s", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratorMissingPrivateMetadataNeverInvokesGit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
t.Chdir(root)
|
||||
os.WriteFile("go.mod", []byte("module example.invalid/offline\n\ngo 1.27.0\n\nrequire code.fritzlab.net/agenthub/missing v0.0.1\n"), 0600)
|
||||
os.WriteFile("go.sum", nil, 0600)
|
||||
bin := filepath.Join(root, "bin")
|
||||
os.Mkdir(bin, 0700)
|
||||
marker := filepath.Join(root, "git-invoked")
|
||||
os.WriteFile(filepath.Join(bin, "git"), []byte("#!/bin/sh\nprintf invoked > \"$PRIVATE_GIT_PROBE\"\nexit 99\n"), 0700)
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
t.Setenv("PRIVATE_GIT_PROBE", marker)
|
||||
t.Setenv("GOMODCACHE", filepath.Join(root, "empty-cache"))
|
||||
t.Setenv("GOPRIVATE", "code.fritzlab.net")
|
||||
t.Setenv("GONOPROXY", "code.fritzlab.net")
|
||||
t.Setenv("GOTOOLCHAIN", "auto")
|
||||
err := generate()
|
||||
if err == nil {
|
||||
t.Fatal("missing cached module accepted")
|
||||
}
|
||||
var failure *exec.ExitError
|
||||
if !errors.As(err, &failure) || !bytes.Contains(failure.Stderr, []byte("module lookup disabled by GOPROXY=off")) {
|
||||
t.Fatalf("not a proven offline refusal: %v", err)
|
||||
}
|
||||
if _, err = os.Stat(marker); !os.IsNotExist(err) {
|
||||
t.Fatal("Git invoked during offline generation")
|
||||
}
|
||||
if _, err = os.Stat(bundleRoot); !os.IsNotExist(err) {
|
||||
t.Fatal("failed generation published output")
|
||||
}
|
||||
os.WriteFile("go.mod", []byte("module example.invalid/offline\n\ngo 1.999.0\n"), 0600)
|
||||
_, err = nativeGo("list", "-m", "-json", "all")
|
||||
if !errors.As(err, &failure) || !bytes.Contains(failure.Stderr, []byte("GOTOOLCHAIN=local")) {
|
||||
t.Fatalf("toolchain download not disabled: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalModuleArtifactPathBinding(t *testing.T) {
|
||||
for _, test := range []struct{ path, version, want string }{
|
||||
{"code.fritzlab.net/agenthub/Provider", "v1.2.3-RC.1", "code.fritzlab.net/agenthub/!provider/@v/v1.2.3-!r!c.1"},
|
||||
{"code.fritzlab.net/agenthub/provider", "v2.0.0+incompatible", "code.fritzlab.net/agenthub/provider/@v/v2.0.0+incompatible"},
|
||||
} {
|
||||
got, err := moduleStem(test.path, test.version)
|
||||
if err != nil || got != test.want {
|
||||
t.Fatalf("stem %q %v", got, err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"code.fritzlab.net/agenthub/../provider", "code.fritzlab.net/agenthub/provider!", "other.invalid/agenthub/provider"} {
|
||||
if _, err := moduleStem(path, "v1.0.0"); err == nil {
|
||||
t.Fatal("invalid path accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user