Skip to content

Commit a08c2c8

Browse files
authored
feat: add systemd mode for full VM experience (#50)
* plan * feat: add systemd mode for EC2-like VMs Replace shell-based init script with Go binary that supports two modes: ## Exec Mode (existing behavior) - Go init runs as PID 1 - Starts guest-agent in background - Runs container entrypoint as child process - Used for standard Docker images (nginx, python, etc.) ## Systemd Mode (new) - Auto-detected when image CMD is /sbin/init or /lib/systemd/systemd - Go init sets up rootfs, then chroots and execs systemd - Systemd becomes PID 1 and manages the full system - guest-agent runs as a systemd service (hypeman-agent.service) - Enables EC2-like experience: ssh, systemctl, journalctl all work ## Key changes: - lib/system/init/: New Go-based init binary with modular boot phases - lib/images/systemd.go: IsSystemdImage() auto-detection from CMD - lib/instances/configdisk.go: Passes INIT_MODE to guest - lib/system/init/init.sh: Shell wrapper to mount /proc /sys /dev before Go runtime (Go requires these during initialization) - integration/systemd_test.go: Full E2E test verifying: - systemd is PID 1 - hypeman-agent.service is active - journalctl works for viewing logs ## Boot flow: 1. Kernel loads initrd with busybox + Go init + guest-agent 2. init.sh mounts /proc, /sys, /dev (Go runtime needs these) 3. init.sh execs Go init binary 4. Go init mounts overlay rootfs, configures network, copies agent 5. Based on INIT_MODE: exec mode (run entrypoint) or systemd mode (chroot + exec /sbin/init) * tweaks * tweaks * Fix test * fix test * rename AgentConnectionError -> AgentVsockDialError * feat: add shared vmconfig package for host-to-guest config schema Defines Config and VolumeMount types in a shared package that both the host (configdisk.go) and guest init binary can import, eliminating duplication. * refactor: use shared vmconfig package in host and guest code - configdisk.go now uses vmconfig.Config instead of local GuestConfig - init binary now imports vmconfig instead of duplicating types - Also adds logging to dropToShell() for better debugging - mode_systemd.go now runs user's CMD instead of hardcoding /sbin/init * fix: remove overly-broad /init suffix detection in IsSystemdImage Per review feedback, matching any path ending in /init is too aggressive since many entrypoint scripts are named 'init'. Now only matches explicit systemd paths: /sbin/init, /lib/systemd/systemd, /usr/lib/systemd/systemd * chore: remove plan file before merge * fix: use %d format for GuestCIDR int field * fix: address Bugbot feedback on env vars and volume mount collisions - buildEnv: user's env vars now take precedence over defaults (PATH, HOME) - systemd mode: pass user's env vars via buildEnv instead of os.Environ() - volumes: use device name for overlay mount points to avoid basename collisions
1 parent 73750f6 commit a08c2c8

28 files changed

Lines changed: 1752 additions & 593 deletions

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ cloud-hypervisor
2121
cloud-hypervisor/**
2222
lib/system/exec_agent/exec-agent
2323
lib/system/guest_agent/guest-agent
24-
lib/system/guest_agent/guest_agent
24+
lib/system/init/init
2525

2626
# Envoy binaries
2727
lib/ingress/binaries/**

Makefile

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
SHELL := /bin/bash
2-
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries build-preview-cli release-prep clean
2+
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded
33

44
# Directory where local binaries will be installed
55
BIN_DIR ?= $(CURDIR)/bin
@@ -165,26 +165,33 @@ ensure-caddy-binaries:
165165
fi
166166

167167
# Build guest-agent (guest binary) into its own directory for embedding
168-
lib/system/guest_agent/guest-agent: lib/system/guest_agent/main.go
168+
lib/system/guest_agent/guest-agent: lib/system/guest_agent/*.go
169169
@echo "Building guest-agent..."
170170
cd lib/system/guest_agent && CGO_ENABLED=0 go build -ldflags="-s -w" -o guest-agent .
171171

172+
# Build init binary (runs as PID 1 in guest VM) for embedding
173+
lib/system/init/init: lib/system/init/*.go
174+
@echo "Building init binary..."
175+
cd lib/system/init && CGO_ENABLED=0 go build -ldflags="-s -w" -o init .
176+
177+
build-embedded: lib/system/guest_agent/guest-agent lib/system/init/init
178+
172179
# Build the binary
173-
build: ensure-ch-binaries ensure-caddy-binaries lib/system/guest_agent/guest-agent | $(BIN_DIR)
180+
build: ensure-ch-binaries ensure-caddy-binaries build-embedded | $(BIN_DIR)
174181
go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api
175182

176183
# Build all binaries
177184
build-all: build
178185

179186
# Run in development mode with hot reload
180-
dev: ensure-ch-binaries ensure-caddy-binaries lib/system/guest_agent/guest-agent $(AIR)
187+
dev: ensure-ch-binaries ensure-caddy-binaries build-embedded $(AIR)
181188
@rm -f ./tmp/main
182189
$(AIR) -c .air.toml
183190

184191
# Run tests (as root for network capabilities, enables caching and parallelism)
185192
# Usage: make test - runs all tests
186193
# make test TEST=TestCreateInstanceWithNetwork - runs specific test
187-
test: ensure-ch-binaries ensure-caddy-binaries lib/system/guest_agent/guest-agent
194+
test: ensure-ch-binaries ensure-caddy-binaries build-embedded
188195
@if [ -n "$(TEST)" ]; then \
189196
echo "Running specific test: $(TEST)"; \
190197
sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) -v -timeout=180s ./...; \
@@ -203,8 +210,9 @@ clean:
203210
rm -rf lib/vmm/binaries/cloud-hypervisor/
204211
rm -rf lib/ingress/binaries/
205212
rm -f lib/system/guest_agent/guest-agent
213+
rm -f lib/system/init/init
206214

207215
# Prepare for release build (called by GoReleaser)
208216
# Downloads all embedded binaries and builds embedded components
209-
release-prep: download-ch-binaries build-caddy-binaries lib/system/guest_agent/guest-agent
217+
release-prep: download-ch-binaries build-caddy-binaries build-embedded
210218
go mod tidy

cmd/api/api/exec.go

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,12 @@ var upgrader = websocket.Upgrader{
2929

3030
// ExecRequest represents the JSON body for exec requests
3131
type ExecRequest struct {
32-
Command []string `json:"command"`
33-
TTY bool `json:"tty"`
34-
Env map[string]string `json:"env,omitempty"`
35-
Cwd string `json:"cwd,omitempty"`
36-
Timeout int32 `json:"timeout,omitempty"` // seconds
32+
Command []string `json:"command"`
33+
TTY bool `json:"tty"`
34+
Env map[string]string `json:"env,omitempty"`
35+
Cwd string `json:"cwd,omitempty"`
36+
Timeout int32 `json:"timeout,omitempty"` // seconds
37+
WaitForAgent int32 `json:"wait_for_agent,omitempty"` // seconds to wait for guest agent to be ready
3738
}
3839

3940
// ExecHandler handles exec requests via WebSocket for bidirectional streaming
@@ -106,6 +107,7 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) {
106107
"tty", execReq.TTY,
107108
"cwd", execReq.Cwd,
108109
"timeout", execReq.Timeout,
110+
"wait_for_agent", execReq.WaitForAgent,
109111
)
110112

111113
// Create WebSocket read/writer wrapper
@@ -122,14 +124,15 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) {
122124

123125
// Execute via vsock
124126
exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
125-
Command: execReq.Command,
126-
Stdin: wsConn,
127-
Stdout: wsConn,
128-
Stderr: wsConn,
129-
TTY: execReq.TTY,
130-
Env: execReq.Env,
131-
Cwd: execReq.Cwd,
132-
Timeout: execReq.Timeout,
127+
Command: execReq.Command,
128+
Stdin: wsConn,
129+
Stdout: wsConn,
130+
Stderr: wsConn,
131+
TTY: execReq.TTY,
132+
Env: execReq.Env,
133+
Cwd: execReq.Cwd,
134+
Timeout: execReq.Timeout,
135+
WaitForAgent: time.Duration(execReq.WaitForAgent) * time.Second,
133136
})
134137

135138
duration := time.Since(startTime)

cmd/api/api/exec_test.go

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -115,38 +115,23 @@ func TestExecInstanceNonTTY(t *testing.T) {
115115
t.Logf("vsock socket exists: %s", actualInst.VsockSocket)
116116
}
117117

118-
// Wait for exec agent to be ready (retry a few times)
119-
var exit *guest.ExitStatus
120118
var stdout, stderr outputBuffer
121-
var execErr error
122119

123120
dialer, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
124121
require.NoError(t, err)
125122

126123
t.Log("Testing exec command: whoami")
127-
maxRetries := 10
128-
for i := 0; i < maxRetries; i++ {
129-
stdout = outputBuffer{}
130-
stderr = outputBuffer{}
131-
132-
exit, execErr = guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
133-
Command: []string{"/bin/sh", "-c", "whoami"},
134-
Stdin: nil,
135-
Stdout: &stdout,
136-
Stderr: &stderr,
137-
TTY: false,
138-
})
139-
140-
if execErr == nil {
141-
break
142-
}
143-
144-
t.Logf("Exec attempt %d/%d failed, retrying: %v", i+1, maxRetries, execErr)
145-
time.Sleep(1 * time.Second)
146-
}
124+
exit, execErr := guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
125+
Command: []string{"/bin/sh", "-c", "whoami"},
126+
Stdin: nil,
127+
Stdout: &stdout,
128+
Stderr: &stderr,
129+
TTY: false,
130+
WaitForAgent: 10 * time.Second, // Wait up to 10s for guest agent to be ready
131+
})
147132

148133
// Assert exec worked
149-
require.NoError(t, execErr, "exec should succeed after retries")
134+
require.NoError(t, execErr, "exec should succeed")
150135
require.NotNil(t, exit, "exit status should be returned")
151136
require.Equal(t, 0, exit.Code, "whoami should exit with code 0")
152137

@@ -251,7 +236,7 @@ func TestExecWithDebianMinimal(t *testing.T) {
251236

252237
// Verify the app exited but VM is still usable (key behavior this test validates)
253238
logs = collectTestLogs(t, svc, inst.Id, 200)
254-
assert.Contains(t, logs, "overlay-init: app exited with code", "App should have exited")
239+
assert.Contains(t, logs, "[exec] app exited with code", "App should have exited")
255240

256241
// Test exec commands work even though the main app (bash) has exited
257242
dialer2, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)

integration/systemd_test.go

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
package integration
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"os"
7+
"strings"
8+
"testing"
9+
"time"
10+
11+
"github.com/onkernel/hypeman/cmd/api/config"
12+
"github.com/onkernel/hypeman/lib/devices"
13+
"github.com/onkernel/hypeman/lib/guest"
14+
"github.com/onkernel/hypeman/lib/hypervisor"
15+
"github.com/onkernel/hypeman/lib/images"
16+
"github.com/onkernel/hypeman/lib/instances"
17+
"github.com/onkernel/hypeman/lib/network"
18+
"github.com/onkernel/hypeman/lib/paths"
19+
"github.com/onkernel/hypeman/lib/system"
20+
"github.com/onkernel/hypeman/lib/volumes"
21+
"github.com/stretchr/testify/assert"
22+
"github.com/stretchr/testify/require"
23+
)
24+
25+
// TestSystemdMode verifies that hypeman correctly detects and runs
26+
// systemd-based images with systemd as PID 1.
27+
//
28+
// This test uses the jrei/systemd-ubuntu image from Docker Hub which runs
29+
// systemd as its CMD. The test verifies that hypeman auto-detects this and:
30+
// - Uses systemd mode (chroot to container rootfs)
31+
// - Starts systemd as PID 1
32+
// - Injects and starts the hypeman-agent.service
33+
func TestSystemdMode(t *testing.T) {
34+
if testing.Short() {
35+
t.Skip("skipping integration test in short mode")
36+
}
37+
38+
// Skip if KVM is not available
39+
if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
40+
t.Skip("/dev/kvm not available")
41+
}
42+
43+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
44+
defer cancel()
45+
46+
// Set up test environment
47+
tmpDir := t.TempDir()
48+
p := paths.New(tmpDir)
49+
50+
cfg := &config.Config{
51+
DataDir: tmpDir,
52+
BridgeName: "vmbr0",
53+
SubnetCIDR: "10.100.0.0/16",
54+
DNSServer: "1.1.1.1",
55+
}
56+
57+
// Create managers
58+
imageManager, err := images.NewManager(p, 1, nil)
59+
require.NoError(t, err)
60+
61+
systemManager := system.NewManager(p)
62+
networkManager := network.NewManager(p, cfg, nil)
63+
deviceManager := devices.NewManager(p)
64+
volumeManager := volumes.NewManager(p, 0, nil)
65+
66+
limits := instances.ResourceLimits{
67+
MaxOverlaySize: 100 * 1024 * 1024 * 1024,
68+
MaxVcpusPerInstance: 0,
69+
MaxMemoryPerInstance: 0,
70+
MaxTotalVcpus: 0,
71+
MaxTotalMemory: 0,
72+
}
73+
74+
instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil)
75+
76+
// Cleanup any orphaned instances
77+
t.Cleanup(func() {
78+
instanceManager.DeleteInstance(ctx, "systemd-test")
79+
})
80+
81+
imageName := "docker.io/jrei/systemd-ubuntu:22.04"
82+
83+
// Pull the systemd image
84+
t.Log("Pulling systemd image:", imageName)
85+
_, err = imageManager.CreateImage(ctx, images.CreateImageRequest{
86+
Name: imageName,
87+
})
88+
require.NoError(t, err)
89+
90+
// Wait for image to be ready
91+
t.Log("Waiting for image build...")
92+
var img *images.Image
93+
for i := 0; i < 120; i++ {
94+
img, err = imageManager.GetImage(ctx, imageName)
95+
if err == nil && img.Status == images.StatusReady {
96+
break
97+
}
98+
time.Sleep(1 * time.Second)
99+
}
100+
require.Equal(t, images.StatusReady, img.Status, "image should be ready")
101+
102+
// Verify systemd detection
103+
t.Run("IsSystemdImage", func(t *testing.T) {
104+
isSystemd := images.IsSystemdImage(img.Entrypoint, img.Cmd)
105+
assert.True(t, isSystemd, "image should be detected as systemd, entrypoint=%v cmd=%v", img.Entrypoint, img.Cmd)
106+
})
107+
108+
// Ensure system files (kernel, initrd)
109+
t.Log("Ensuring system files...")
110+
err = systemManager.EnsureSystemFiles(ctx)
111+
require.NoError(t, err)
112+
113+
// Create the systemd instance
114+
t.Log("Creating systemd instance...")
115+
inst, err := instanceManager.CreateInstance(ctx, instances.CreateInstanceRequest{
116+
Name: "systemd-test",
117+
Image: imageName,
118+
Size: 2 * 1024 * 1024 * 1024, // 2GB
119+
HotplugSize: 512 * 1024 * 1024,
120+
OverlaySize: 1024 * 1024 * 1024,
121+
Vcpus: 2,
122+
NetworkEnabled: false, // No network needed for this test
123+
})
124+
require.NoError(t, err)
125+
t.Logf("Instance created: %s", inst.Id)
126+
127+
// Wait for guest agent to be ready
128+
t.Log("Waiting for guest agent...")
129+
err = waitForGuestAgent(ctx, instanceManager, inst.Id, 60*time.Second)
130+
require.NoError(t, err, "guest agent should be ready")
131+
132+
// Test: Verify systemd is PID 1
133+
t.Run("SystemdIsPID1", func(t *testing.T) {
134+
output, exitCode, err := execInInstance(ctx, inst, "cat", "/proc/1/comm")
135+
require.NoError(t, err, "exec should work")
136+
require.Equal(t, 0, exitCode, "command should succeed")
137+
138+
pid1Name := strings.TrimSpace(output)
139+
assert.Equal(t, "systemd", pid1Name, "PID 1 should be systemd")
140+
t.Logf("PID 1 is: %s", pid1Name)
141+
})
142+
143+
// Test: Verify guest-agent binary exists
144+
t.Run("GuestAgentExists", func(t *testing.T) {
145+
output, exitCode, err := execInInstance(ctx, inst, "test", "-x", "/opt/hypeman/guest-agent")
146+
require.NoError(t, err, "exec should work")
147+
assert.Equal(t, 0, exitCode, "guest-agent binary should exist at /opt/hypeman/guest-agent, output: %s", output)
148+
})
149+
150+
// Test: Verify hypeman-agent.service is active
151+
t.Run("AgentServiceActive", func(t *testing.T) {
152+
output, exitCode, err := execInInstance(ctx, inst, "systemctl", "is-active", "hypeman-agent")
153+
require.NoError(t, err, "exec should work")
154+
status := strings.TrimSpace(output)
155+
assert.Equal(t, 0, exitCode, "hypeman-agent service should be active, status: %s", status)
156+
assert.Equal(t, "active", status, "service status should be 'active'")
157+
t.Logf("hypeman-agent service status: %s", status)
158+
})
159+
160+
// Test: Verify we can view agent logs via journalctl
161+
t.Run("AgentLogsAccessible", func(t *testing.T) {
162+
output, exitCode, err := execInInstance(ctx, inst, "journalctl", "-u", "hypeman-agent", "--no-pager", "-n", "5")
163+
require.NoError(t, err, "exec should work")
164+
assert.Equal(t, 0, exitCode, "journalctl should succeed")
165+
t.Logf("Agent logs (last 5 lines):\n%s", output)
166+
})
167+
168+
t.Log("All systemd mode tests passed!")
169+
}
170+
171+
// waitForGuestAgent polls until the guest agent is ready
172+
func waitForGuestAgent(ctx context.Context, mgr instances.Manager, instanceID string, timeout time.Duration) error {
173+
inst, err := mgr.GetInstance(ctx, instanceID)
174+
if err != nil {
175+
return err
176+
}
177+
178+
dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
179+
if err != nil {
180+
return err
181+
}
182+
183+
// Use WaitForAgent to wait for the agent to be ready
184+
var stdout bytes.Buffer
185+
_, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
186+
Command: []string{"echo", "ready"},
187+
Stdout: &stdout,
188+
TTY: false,
189+
WaitForAgent: timeout,
190+
})
191+
return err
192+
}
193+
194+
// execInInstance executes a command in the instance
195+
func execInInstance(ctx context.Context, inst *instances.Instance, command ...string) (string, int, error) {
196+
dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
197+
if err != nil {
198+
return "", -1, err
199+
}
200+
201+
var stdout, stderr bytes.Buffer
202+
exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
203+
Command: command,
204+
Stdout: &stdout,
205+
Stderr: &stderr,
206+
TTY: false,
207+
})
208+
if err != nil {
209+
return stderr.String(), -1, err
210+
}
211+
212+
return stdout.String(), exit.Code, nil
213+
}

0 commit comments

Comments
 (0)