flock M1 scaffold: CNI plugin + agent + NodeConfig CRD
Build flock Image / build (push) Has been cancelled
Build flock Image / build (push) Has been cancelled
- cmd/flock + cmd/flock-agent: build cleanly; CNI ADD/DEL/CHECK return ErrInternal stubs until M2; agent boots, opens unix socket, logs JSON. - pkg/agent/state.go: durable allocations.json (atomic write + fsync + parent fsync); pending/committed lifecycle. Tests cover round-trip, replace-by-cid, version mismatch, no-leak-on-tmp. - pkg/embed/suffix.go: ip-algo IID embedding. Tests cover the /48-/96 nibble distribution table from the design doc, determinism, prefix preservation, N-nibble isolation, digest-vs-fallback divergence. - pkg/api/v1alpha1: minimal NodeConfig types (no controller-runtime yet). - deploy/: NodeConfig CRD, empty ServiceAccount/ClusterRole, DaemonSet pinned to flock.fritzlab.net/agent="" label so it only runs on opted-in nodes. - .gitea/workflows/main.yaml + Dockerfile: build + push to code.fritzlab.net/fritzlab/flock; runs go test in CI. Design doc: dfritzlab/k8s-manager/dfritz-cni.md. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Donavan Fritz
@@ -0,0 +1,100 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// SocketPath is the unix socket on which flock-agent serves RPCs from the
|
||||
// CNI plugin. Mirrors pkg/cni.SocketPath; kept as a separate constant so the
|
||||
// agent package has no import-cycle on the CNI package.
|
||||
const SocketPath = "/run/flock/flock.sock"
|
||||
|
||||
// Server is the agent's runtime container: state store, kubernetes informers,
|
||||
// netlink, BIRD, nftables. M1 wires only the state store and a placeholder
|
||||
// listener so the binary boots and exits cleanly under a context.
|
||||
type Server struct {
|
||||
Node string
|
||||
Store *Store
|
||||
Logger *slog.Logger
|
||||
socket string
|
||||
closeCh chan struct{}
|
||||
}
|
||||
|
||||
// Config configures NewServer.
|
||||
type Config struct {
|
||||
Node string
|
||||
StatePath string // typically /var/lib/flock/allocations.json
|
||||
Socket string // typically /run/flock/flock.sock
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewServer constructs a Server. It does NOT start any goroutines; call Run.
|
||||
func NewServer(cfg Config) (*Server, error) {
|
||||
if cfg.Node == "" {
|
||||
return nil, fmt.Errorf("Node must be set")
|
||||
}
|
||||
if cfg.StatePath == "" {
|
||||
cfg.StatePath = "/var/lib/flock/allocations.json"
|
||||
}
|
||||
if cfg.Socket == "" {
|
||||
cfg.Socket = SocketPath
|
||||
}
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(cfg.StatePath), 0o750); err != nil {
|
||||
return nil, fmt.Errorf("mkdir state dir: %w", err)
|
||||
}
|
||||
store, err := NewStore(cfg.StatePath, cfg.Node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open store: %w", err)
|
||||
}
|
||||
return &Server{
|
||||
Node: cfg.Node,
|
||||
Store: store,
|
||||
Logger: cfg.Logger,
|
||||
socket: cfg.Socket,
|
||||
closeCh: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run starts the agent and blocks until ctx is cancelled. M1 only opens the
|
||||
// unix listener (proving permissions/path); the RPC handler is a no-op
|
||||
// returning ENOSYS until M2.
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
if err := os.MkdirAll(filepath.Dir(s.socket), 0o750); err != nil {
|
||||
return fmt.Errorf("mkdir socket dir: %w", err)
|
||||
}
|
||||
_ = os.Remove(s.socket)
|
||||
l, err := net.Listen("unix", s.socket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", s.socket, err)
|
||||
}
|
||||
defer l.Close()
|
||||
|
||||
s.Logger.Info("flock-agent started",
|
||||
"node", s.Node,
|
||||
"socket", s.socket,
|
||||
"allocations", len(s.Store.Snapshot()),
|
||||
)
|
||||
|
||||
// Accept loop: M1 closes every accepted conn immediately. M2 will dispatch.
|
||||
go func() {
|
||||
for {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
return // listener closed
|
||||
}
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
s.Logger.Info("flock-agent stopping")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
// Package agent owns the in-process flock-agent runtime: IPAM, netns, state,
|
||||
// anycast, and NetworkPolicy. This file implements the durable per-node
|
||||
// allocation file at /var/lib/flock/allocations.json.
|
||||
package agent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AllocationState is the lifecycle marker for an entry in allocations.json.
|
||||
//
|
||||
// pending — IPAM picked addresses; netlink work may be incomplete.
|
||||
// On agent startup these are GC'd: addrs/routes/veth removed.
|
||||
// committed — netlink ops finished; entry is the source of truth.
|
||||
type AllocationState string
|
||||
|
||||
const (
|
||||
StatePending AllocationState = "pending"
|
||||
StateCommitted AllocationState = "committed"
|
||||
)
|
||||
|
||||
// Allocation is a single per-pod entry persisted in allocations.json.
|
||||
type Allocation struct {
|
||||
ContainerID string `json:"container_id"`
|
||||
Namespace string `json:"namespace"`
|
||||
PodName string `json:"pod_name"`
|
||||
OwnerUID string `json:"owner_uid"`
|
||||
IP6 string `json:"ip6,omitempty"`
|
||||
IP4 string `json:"ip4,omitempty"`
|
||||
Anycast []string `json:"anycast,omitempty"`
|
||||
State AllocationState `json:"state"`
|
||||
AllocatedAt time.Time `json:"allocated_at"`
|
||||
}
|
||||
|
||||
// State is the on-disk file shape. Version is bumped on incompatible changes.
|
||||
type State struct {
|
||||
Version int `json:"version"`
|
||||
Node string `json:"node"`
|
||||
Allocations []Allocation `json:"allocations"`
|
||||
}
|
||||
|
||||
const stateVersion = 1
|
||||
|
||||
// Store is the durable allocation store.
|
||||
//
|
||||
// All public methods are safe for concurrent use. They serialize through
|
||||
// a single mutex so that the on-disk file is always consistent and so that
|
||||
// the CNI ADD/DEL critical sections (which mutate kernel state alongside the
|
||||
// file) can rely on snapshot semantics.
|
||||
type Store struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
node string
|
||||
data State
|
||||
}
|
||||
|
||||
// NewStore opens (or creates) a per-node store. The directory containing
|
||||
// `path` must already exist; we do not create it because in production it is
|
||||
// /var/lib/flock managed by the DaemonSet, not the agent process.
|
||||
func NewStore(path, node string) (*Store, error) {
|
||||
s := &Store{path: path, node: node}
|
||||
if err := s.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Store) load() error {
|
||||
b, err := os.ReadFile(s.path)
|
||||
if os.IsNotExist(err) {
|
||||
s.data = State{Version: stateVersion, Node: s.node, Allocations: []Allocation{}}
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read state: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(b, &s.data); err != nil {
|
||||
return fmt.Errorf("parse state: %w", err)
|
||||
}
|
||||
if s.data.Version != stateVersion {
|
||||
return fmt.Errorf("state version %d, want %d", s.data.Version, stateVersion)
|
||||
}
|
||||
if s.data.Allocations == nil {
|
||||
s.data.Allocations = []Allocation{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// flushLocked writes the in-memory state to disk durably:
|
||||
//
|
||||
// 1. write to <path>.tmp
|
||||
// 2. fsync(tmpfd)
|
||||
// 3. rename to <path>
|
||||
// 4. fsync(parent dir) — required so the rename survives power loss.
|
||||
//
|
||||
// Caller MUST hold s.mu.
|
||||
func (s *Store) flushLocked() error {
|
||||
b, err := json.MarshalIndent(s.data, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal state: %w", err)
|
||||
}
|
||||
|
||||
tmp := s.path + ".tmp"
|
||||
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open tmp: %w", err)
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("write tmp: %w", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("fsync tmp: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("close tmp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, s.path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("rename: %w", err)
|
||||
}
|
||||
|
||||
dir, err := os.Open(filepath.Dir(s.path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open parent: %w", err)
|
||||
}
|
||||
defer dir.Close()
|
||||
if err := dir.Sync(); err != nil {
|
||||
return fmt.Errorf("fsync parent: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the entry for containerID, ok=false if absent. Returned value
|
||||
// is a copy; mutations do not affect store state.
|
||||
func (s *Store) Get(containerID string) (Allocation, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, a := range s.data.Allocations {
|
||||
if a.ContainerID == containerID {
|
||||
return a, true
|
||||
}
|
||||
}
|
||||
return Allocation{}, false
|
||||
}
|
||||
|
||||
// Upsert inserts or replaces the entry for a.ContainerID and flushes.
|
||||
func (s *Store) Upsert(a Allocation) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for i, ex := range s.data.Allocations {
|
||||
if ex.ContainerID == a.ContainerID {
|
||||
s.data.Allocations[i] = a
|
||||
return s.flushLocked()
|
||||
}
|
||||
}
|
||||
s.data.Allocations = append(s.data.Allocations, a)
|
||||
return s.flushLocked()
|
||||
}
|
||||
|
||||
// Delete removes the entry for containerID (no-op if absent) and flushes.
|
||||
func (s *Store) Delete(containerID string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for i, a := range s.data.Allocations {
|
||||
if a.ContainerID == containerID {
|
||||
s.data.Allocations = append(s.data.Allocations[:i], s.data.Allocations[i+1:]...)
|
||||
return s.flushLocked()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Snapshot returns a defensive copy of all allocations.
|
||||
func (s *Store) Snapshot() []Allocation {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]Allocation, len(s.data.Allocations))
|
||||
copy(out, s.data.Allocations)
|
||||
return out
|
||||
}
|
||||
|
||||
// PendingContainerIDs returns container IDs whose entries are State==pending.
|
||||
// Used by agent startup GC.
|
||||
func (s *Store) PendingContainerIDs() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for _, a := range s.data.Allocations {
|
||||
if a.State == StatePending {
|
||||
out = append(out, a.ContainerID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newStore(t *testing.T) (*Store, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "allocations.json")
|
||||
s, err := NewStore(path, "host001")
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore: %v", err)
|
||||
}
|
||||
return s, path
|
||||
}
|
||||
|
||||
func TestStore_EmptyOnFirstOpen(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
if got := len(s.Snapshot()); got != 0 {
|
||||
t.Fatalf("Snapshot len = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_UpsertGetDelete(t *testing.T) {
|
||||
s, path := newStore(t)
|
||||
a := Allocation{
|
||||
ContainerID: "abc",
|
||||
Namespace: "mail",
|
||||
PodName: "stalwart-0",
|
||||
OwnerUID: "uid-1",
|
||||
IP6: "2602:817:3000:f001::1",
|
||||
State: StateCommitted,
|
||||
AllocatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}
|
||||
if err := s.Upsert(a); err != nil {
|
||||
t.Fatalf("Upsert: %v", err)
|
||||
}
|
||||
|
||||
got, ok := s.Get("abc")
|
||||
if !ok || got.PodName != "stalwart-0" {
|
||||
t.Fatalf("Get after Upsert: ok=%v got=%+v", ok, got)
|
||||
}
|
||||
|
||||
// Round-trip: a fresh Store reading the same path sees the entry.
|
||||
s2, err := NewStore(path, "host001")
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
if got, ok := s2.Get("abc"); !ok || got.IP6 != a.IP6 {
|
||||
t.Fatalf("reopen Get: ok=%v got=%+v", ok, got)
|
||||
}
|
||||
|
||||
if err := s.Delete("abc"); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if _, ok := s.Get("abc"); ok {
|
||||
t.Fatalf("entry still present after Delete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_UpsertReplacesByContainerID(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
must(s.Upsert(Allocation{ContainerID: "abc", IP6: "::1", State: StatePending}))
|
||||
must(s.Upsert(Allocation{ContainerID: "abc", IP6: "::2", State: StateCommitted}))
|
||||
if got := len(s.Snapshot()); got != 1 {
|
||||
t.Fatalf("len = %d, want 1 (Upsert should replace)", got)
|
||||
}
|
||||
if a, _ := s.Get("abc"); a.IP6 != "::2" || a.State != StateCommitted {
|
||||
t.Fatalf("Upsert did not replace: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_PendingContainerIDs(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
_ = s.Upsert(Allocation{ContainerID: "p1", State: StatePending})
|
||||
_ = s.Upsert(Allocation{ContainerID: "c1", State: StateCommitted})
|
||||
_ = s.Upsert(Allocation{ContainerID: "p2", State: StatePending})
|
||||
|
||||
pend := s.PendingContainerIDs()
|
||||
if len(pend) != 2 {
|
||||
t.Fatalf("PendingContainerIDs len = %d, want 2", len(pend))
|
||||
}
|
||||
have := map[string]bool{pend[0]: true, pend[1]: true}
|
||||
if !have["p1"] || !have["p2"] {
|
||||
t.Fatalf("PendingContainerIDs = %v, want p1,p2", pend)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_RejectsWrongVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "allocations.json")
|
||||
if err := os.WriteFile(path, []byte(`{"version":99,"node":"x","allocations":[]}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := NewStore(path, "x"); err == nil {
|
||||
t.Fatalf("expected error on bad version, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_AtomicWriteDurability(t *testing.T) {
|
||||
// We can't simulate a real power-loss in unit tests, but we can verify
|
||||
// that no .tmp file is left behind after a successful flush, and that
|
||||
// the rename target is intact.
|
||||
s, path := newStore(t)
|
||||
if err := s.Upsert(Allocation{ContainerID: "x", State: StateCommitted}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) {
|
||||
t.Fatalf(".tmp leaked: err=%v", err)
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil || len(b) == 0 {
|
||||
t.Fatalf("final file unreadable: err=%v len=%d", err, len(b))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user