diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index cac8c7ec..2d84ac08 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -104,10 +104,27 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if vol.Readonly != nil { readonly = *vol.Readonly } + overlay := false + if vol.Overlay != nil { + overlay = *vol.Overlay + } + var overlaySize int64 + if vol.OverlaySize != nil && *vol.OverlaySize != "" { + var overlaySizeBytes datasize.ByteSize + if err := overlaySizeBytes.UnmarshalText([]byte(*vol.OverlaySize)); err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_overlay_size", + Message: fmt.Sprintf("invalid overlay_size for volume %s: %v", vol.VolumeId, err), + }, nil + } + overlaySize = int64(overlaySizeBytes) + } volumes[i] = instances.VolumeAttachment{ - VolumeID: vol.VolumeId, - MountPath: vol.MountPath, - Readonly: readonly, + VolumeID: vol.VolumeId, + MountPath: vol.MountPath, + Readonly: readonly, + Overlay: overlay, + OverlaySize: overlaySize, } } } @@ -461,11 +478,17 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if len(inst.Volumes) > 0 { oapiVolumes := make([]oapi.VolumeAttachment, len(inst.Volumes)) for i, vol := range inst.Volumes { - oapiVolumes[i] = oapi.VolumeAttachment{ + oapiVol := oapi.VolumeAttachment{ VolumeId: vol.VolumeID, MountPath: vol.MountPath, Readonly: lo.ToPtr(vol.Readonly), } + if vol.Overlay { + oapiVol.Overlay = lo.ToPtr(true) + overlaySizeStr := datasize.ByteSize(vol.OverlaySize).HR() + oapiVol.OverlaySize = lo.ToPtr(overlaySizeStr) + } + oapiVolumes[i] = oapiVol } oapiInst.Volumes = &oapiVolumes } diff --git a/cmd/api/api/volumes.go b/cmd/api/api/volumes.go index ff931f15..63d9f134 100644 --- a/cmd/api/api/volumes.go +++ b/cmd/api/api/volumes.go @@ -140,12 +140,20 @@ func volumeToOAPI(vol volumes.Volume) oapi.Volume { SizeGb: vol.SizeGb, CreatedAt: vol.CreatedAt, } - if vol.AttachedTo != nil { - oapiVol.AttachedTo = vol.AttachedTo - } - if vol.MountPath != nil { - oapiVol.MountPath = vol.MountPath + + // Convert attachments + if len(vol.Attachments) > 0 { + attachments := make([]oapi.VolumeAttachmentInfo, len(vol.Attachments)) + for i, att := range vol.Attachments { + attachments[i] = oapi.VolumeAttachmentInfo{ + InstanceId: att.InstanceID, + MountPath: att.MountPath, + Readonly: att.Readonly, + } + } + oapiVol.Attachments = &attachments } + return oapiVol } diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index 60a9aac8..dfc6bdb5 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -107,22 +107,32 @@ GUEST_DNS="%s" // Build volume mounts section // Volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config) + // For overlay volumes, two devices are used: base + overlay disk + // Format: device:path:mode[:overlay_device] volumeSection := "" if len(inst.Volumes) > 0 { var volumeLines strings.Builder - volumeLines.WriteString("\n# Volume mounts (device:path:readonly)\n") + volumeLines.WriteString("\n# Volume mounts (device:path:mode[:overlay_device])\n") volumeLines.WriteString("VOLUME_MOUNTS=\"") + deviceIdx := 0 // Track device index (starts at 'd' = vdd) for i, vol := range inst.Volumes { - // Device naming: vdd, vde, vdf, ... - device := fmt.Sprintf("/dev/vd%c", 'd'+i) - readonly := "rw" - if vol.Readonly { - readonly = "ro" - } + device := fmt.Sprintf("/dev/vd%c", 'd'+deviceIdx) if i > 0 { volumeLines.WriteString(" ") } - volumeLines.WriteString(fmt.Sprintf("%s:%s:%s", device, vol.MountPath, readonly)) + if vol.Overlay { + // Overlay mode: base device + overlay device + overlayDevice := fmt.Sprintf("/dev/vd%c", 'd'+deviceIdx+1) + volumeLines.WriteString(fmt.Sprintf("%s:%s:overlay:%s", device, vol.MountPath, overlayDevice)) + deviceIdx += 2 // Overlay uses 2 devices + } else { + mode := "rw" + if vol.Readonly { + mode = "ro" + } + volumeLines.WriteString(fmt.Sprintf("%s:%s:%s", device, vol.MountPath, mode)) + deviceIdx++ // Regular volume uses 1 device + } } volumeLines.WriteString("\"\n") volumeSection = volumeLines.String() diff --git a/lib/instances/create.go b/lib/instances/create.go index 4811cb50..faa32279 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -263,18 +263,14 @@ func (m *manager) createInstance( if len(req.Volumes) > 0 { log.DebugContext(ctx, "validating volumes", "id", id, "count", len(req.Volumes)) for _, volAttach := range req.Volumes { - // Check volume exists and is not attached - vol, err := m.volumeManager.GetVolume(ctx, volAttach.VolumeID) + // Check volume exists + _, err := m.volumeManager.GetVolume(ctx, volAttach.VolumeID) if err != nil { log.ErrorContext(ctx, "volume not found", "id", id, "volume_id", volAttach.VolumeID, "error", err) return nil, fmt.Errorf("volume %s: %w", volAttach.VolumeID, err) } - if vol.AttachedTo != nil { - log.ErrorContext(ctx, "volume already attached", "id", id, "volume_id", volAttach.VolumeID, "attached_to", *vol.AttachedTo) - return nil, fmt.Errorf("volume %s is already attached to instance %s", volAttach.VolumeID, *vol.AttachedTo) - } - // Mark volume as attached + // Mark volume as attached (AttachVolume handles multi-attach validation) if err := m.volumeManager.AttachVolume(ctx, volAttach.VolumeID, volumes.AttachVolumeRequest{ InstanceID: id, MountPath: volAttach.MountPath, @@ -287,8 +283,17 @@ func (m *manager) createInstance( // Add volume cleanup to stack volumeID := volAttach.VolumeID // capture for closure cu.Add(func() { - m.volumeManager.DetachVolume(ctx, volumeID) + m.volumeManager.DetachVolume(ctx, volumeID, id) }) + + // Create overlay disk for volumes with overlay enabled + if volAttach.Overlay { + log.DebugContext(ctx, "creating volume overlay disk", "id", id, "volume_id", volAttach.VolumeID, "size", volAttach.OverlaySize) + if err := m.createVolumeOverlayDisk(id, volAttach.VolumeID, volAttach.OverlaySize); err != nil { + log.ErrorContext(ctx, "failed to create volume overlay disk", "id", id, "volume_id", volAttach.VolumeID, "error", err) + return nil, fmt.Errorf("create volume overlay disk %s: %w", volAttach.VolumeID, err) + } + } } // Store volume attachments in metadata stored.Volumes = req.Volumes @@ -377,8 +382,16 @@ func validateCreateRequest(req CreateInstanceRequest) error { // validateVolumeAttachments validates volume attachment requests func validateVolumeAttachments(volumes []VolumeAttachment) error { - if len(volumes) > MaxVolumesPerInstance { - return fmt.Errorf("cannot attach more than %d volumes per instance", MaxVolumesPerInstance) + // Count total devices needed (each overlay volume needs 2 devices: base + overlay) + totalDevices := 0 + for _, vol := range volumes { + totalDevices++ + if vol.Overlay { + totalDevices++ // Overlay needs an additional device + } + } + if totalDevices > MaxVolumesPerInstance { + return fmt.Errorf("cannot attach more than %d volume devices per instance (overlay volumes count as 2)", MaxVolumesPerInstance) } seenPaths := make(map[string]bool) @@ -401,6 +414,16 @@ func validateVolumeAttachments(volumes []VolumeAttachment) error { return fmt.Errorf("duplicate mount path %q", cleanPath) } seenPaths[cleanPath] = true + + // Validate overlay mode requirements + if vol.Overlay { + if !vol.Readonly { + return fmt.Errorf("volume %s: overlay mode requires readonly=true", vol.VolumeID) + } + if vol.OverlaySize <= 0 { + return fmt.Errorf("volume %s: overlay_size is required when overlay=true", vol.VolumeID) + } + } } return nil @@ -556,12 +579,26 @@ func (m *manager) buildVMConfig(inst *Instance, imageInfo *images.Image, netConf } // Add attached volumes as additional disks + // For overlay volumes, add both base (readonly) and overlay disk for _, volAttach := range inst.Volumes { volumePath := m.volumeManager.GetVolumePath(volAttach.VolumeID) - disks = append(disks, vmm.DiskConfig{ - Path: &volumePath, - Readonly: ptr(volAttach.Readonly), - }) + if volAttach.Overlay { + // Base volume is always read-only when overlay is enabled + disks = append(disks, vmm.DiskConfig{ + Path: &volumePath, + Readonly: ptr(true), + }) + // Overlay disk is writable + overlayPath := m.paths.InstanceVolumeOverlay(inst.Id, volAttach.VolumeID) + disks = append(disks, vmm.DiskConfig{ + Path: &overlayPath, + }) + } else { + disks = append(disks, vmm.DiskConfig{ + Path: &volumePath, + Readonly: ptr(volAttach.Readonly), + }) + } } // Serial console configuration diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 32630aef..62ed44a1 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -62,7 +62,7 @@ func (m *manager) deleteInstance( if len(inst.Volumes) > 0 { log.DebugContext(ctx, "detaching volumes", "id", id, "count", len(inst.Volumes)) for _, volAttach := range inst.Volumes { - if err := m.volumeManager.DetachVolume(ctx, volAttach.VolumeID); err != nil { + if err := m.volumeManager.DetachVolume(ctx, volAttach.VolumeID, id); err != nil { // Log error but continue with cleanup log.WarnContext(ctx, "failed to detach volume, continuing with cleanup", "id", id, "volume_id", volAttach.VolumeID, "error", err) } diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index dde54d5e..4b533e0a 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -222,7 +222,7 @@ func TestCreateAndDeleteInstance(t *testing.T) { // Verify volume file exists and is not attached assert.FileExists(t, p.VolumeData(vol.Id)) - assert.Nil(t, vol.AttachedTo, "Volume should not be attached yet") + assert.Empty(t, vol.Attachments, "Volume should not be attached yet") // Create instance with real nginx image and attached volume req := CreateInstanceRequest{ @@ -267,9 +267,9 @@ func TestCreateAndDeleteInstance(t *testing.T) { // Verify volume shows as attached vol, err = volumeManager.GetVolume(ctx, vol.Id) require.NoError(t, err) - require.NotNil(t, vol.AttachedTo, "Volume should be attached") - assert.Equal(t, inst.Id, *vol.AttachedTo) - assert.Equal(t, "/mnt/data", *vol.MountPath) + require.Len(t, vol.Attachments, 1, "Volume should be attached") + assert.Equal(t, inst.Id, vol.Attachments[0].InstanceID) + assert.Equal(t, "/mnt/data", vol.Attachments[0].MountPath) // Verify directories exist assert.DirExists(t, p.InstanceDir(inst.Id)) @@ -444,7 +444,7 @@ func TestCreateAndDeleteInstance(t *testing.T) { // Verify volume is detached but still exists vol, err = volumeManager.GetVolume(ctx, vol.Id) require.NoError(t, err) - assert.Nil(t, vol.AttachedTo, "Volume should be detached after instance deletion") + assert.Empty(t, vol.Attachments, "Volume should be detached after instance deletion") assert.FileExists(t, p.VolumeData(vol.Id), "Volume file should still exist") // Delete volume diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index fce71951..ac27b561 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -29,7 +29,7 @@ func TestValidateVolumeAttachments_MaxVolumes(t *testing.T) { err := validateVolumeAttachments(volumes) assert.Error(t, err) - assert.Contains(t, err.Error(), "cannot attach more than 23 volumes") + assert.Contains(t, err.Error(), "cannot attach more than 23") } func TestValidateVolumeAttachments_SystemDirectory(t *testing.T) { @@ -83,6 +83,70 @@ func TestValidateVolumeAttachments_Empty(t *testing.T) { assert.NoError(t, err) } +func TestValidateVolumeAttachments_OverlayRequiresReadonly(t *testing.T) { + // Overlay=true with Readonly=false should fail + volumes := []VolumeAttachment{{ + VolumeID: "vol-1", + MountPath: "/mnt/data", + Readonly: false, // Invalid: overlay requires readonly=true + Overlay: true, + OverlaySize: 100 * 1024 * 1024, + }} + + err := validateVolumeAttachments(volumes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "overlay mode requires readonly=true") +} + +func TestValidateVolumeAttachments_OverlayRequiresSize(t *testing.T) { + // Overlay=true without OverlaySize should fail + volumes := []VolumeAttachment{{ + VolumeID: "vol-1", + MountPath: "/mnt/data", + Readonly: true, + Overlay: true, + OverlaySize: 0, // Invalid: overlay requires size + }} + + err := validateVolumeAttachments(volumes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "overlay_size is required") +} + +func TestValidateVolumeAttachments_OverlayValid(t *testing.T) { + // Valid overlay configuration + volumes := []VolumeAttachment{{ + VolumeID: "vol-1", + MountPath: "/mnt/data", + Readonly: true, + Overlay: true, + OverlaySize: 100 * 1024 * 1024, // 100MB + }} + + err := validateVolumeAttachments(volumes) + assert.NoError(t, err) +} + +func TestValidateVolumeAttachments_OverlayCountsAsTwoDevices(t *testing.T) { + // 12 regular volumes + 12 overlay volumes = 12 + 24 = 36 devices (exceeds 23) + // But let's be more precise: 11 overlay volumes = 22 devices, + 1 regular = 23 (at limit) + // 12 overlay volumes = 24 devices (exceeds limit) + volumes := make([]VolumeAttachment, 12) + for i := range volumes { + volumes[i] = VolumeAttachment{ + VolumeID: "vol-" + string(rune('a'+i)), + MountPath: "/mnt/vol" + string(rune('a'+i)), + Readonly: true, + Overlay: true, + OverlaySize: 100 * 1024 * 1024, + } + } + + err := validateVolumeAttachments(volumes) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot attach more than 23") +} + // createTestManager creates a manager with specified limits for testing func createTestManager(t *testing.T, limits ResourceLimits) *manager { t.Helper() diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 788c7666..35efee99 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -87,6 +87,21 @@ func (m *manager) createOverlayDisk(id string, sizeBytes int64) error { return images.CreateEmptyExt4Disk(overlayPath, sizeBytes) } +// createVolumeOverlayDisk creates a sparse overlay disk for a volume attachment. +// Cleanup note: If instance creation fails after this point, the overlay disk is +// cleaned up automatically by deleteInstanceData() which removes the entire instance +// directory (including vol-overlays/) via the cleanup stack in createInstance(). +func (m *manager) createVolumeOverlayDisk(instanceID, volumeID string, sizeBytes int64) error { + // Ensure vol-overlays directory exists + overlaysDir := m.paths.InstanceVolumeOverlaysDir(instanceID) + if err := os.MkdirAll(overlaysDir, 0755); err != nil { + return fmt.Errorf("create vol-overlays directory: %w", err) + } + + overlayPath := m.paths.InstanceVolumeOverlay(instanceID, volumeID) + return images.CreateEmptyExt4Disk(overlayPath, sizeBytes) +} + // deleteInstanceData removes all instance data from disk func (m *manager) deleteInstanceData(id string) error { instDir := m.paths.InstanceDir(id) diff --git a/lib/instances/types.go b/lib/instances/types.go index 38ad4c4f..82355f1e 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -20,9 +20,11 @@ const ( // VolumeAttachment represents a volume attached to an instance type VolumeAttachment struct { - VolumeID string // Volume ID - MountPath string // Mount path in guest - Readonly bool // Whether mounted read-only + VolumeID string // Volume ID + MountPath string // Mount path in guest + Readonly bool // Whether mounted read-only + Overlay bool // If true, create per-instance overlay for writes (requires Readonly=true) + OverlaySize int64 // Size of overlay disk in bytes (max diff from base) } // StoredMetadata represents instance metadata that is persisted to disk diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go new file mode 100644 index 00000000..9c877352 --- /dev/null +++ b/lib/instances/volumes_test.go @@ -0,0 +1,315 @@ +package instances + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/onkernel/hypeman/lib/images" + "github.com/onkernel/hypeman/lib/paths" + "github.com/onkernel/hypeman/lib/system" + "github.com/onkernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// waitForExecAgent polls until exec-agent is ready +func waitForExecAgent(ctx context.Context, mgr *manager, instanceID string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + logs, err := collectLogs(ctx, mgr, instanceID, 100) + if err == nil && strings.Contains(logs, "[exec-agent] listening on vsock port 2222") { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return context.DeadlineExceeded +} + +// execWithRetry runs a command with retries until exec-agent is ready +func execWithRetry(ctx context.Context, vsockSocket string, command []string) (string, int, error) { + var output string + var code int + var err error + + for i := 0; i < 10; i++ { + output, code, err = execCommand(ctx, vsockSocket, command...) + if err == nil { + return output, code, nil + } + time.Sleep(500 * time.Millisecond) + } + return output, code, err +} + +// TestVolumeMultiAttachReadOnly tests that a volume can be: +// 1. Attached read-write to one instance, written to +// 2. Detached (by deleting the instance) +// 3. Attached read-only to multiple instances simultaneously +// 4. Data persists and is readable from all instances +func TestVolumeMultiAttachReadOnly(t *testing.T) { + // Require KVM + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group") + } + + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + p := paths.New(tmpDir) + + // Setup: prepare image and system files + imageManager, err := images.NewManager(p, 1) + require.NoError(t, err) + + t.Log("Pulling alpine image...") + _, err = imageManager.CreateImage(ctx, images.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, "docker.io/library/alpine:latest") + if err == nil && img.Status == images.StatusReady { + break + } + time.Sleep(1 * time.Second) + } + t.Log("Image ready") + + systemManager := system.NewManager(p) + t.Log("Ensuring system files...") + err = systemManager.EnsureSystemFiles(ctx) + require.NoError(t, err) + t.Log("System files ready") + + // Create volume + volumeManager := volumes.NewManager(p, 0) + t.Log("Creating volume...") + vol, err := volumeManager.CreateVolume(ctx, volumes.CreateVolumeRequest{ + Name: "shared-data", + SizeGb: 1, + }) + require.NoError(t, err) + t.Logf("Volume created: %s", vol.Id) + + // Phase 1: Create instance with volume attached read-write + t.Log("Phase 1: Creating writer instance with read-write volume...") + writerInst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "writer", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: false}, + }, + }) + require.NoError(t, err) + t.Logf("Writer instance created: %s", writerInst.Id) + + // Wait for exec-agent + err = waitForExecAgent(ctx, manager, writerInst.Id, 15*time.Second) + require.NoError(t, err, "exec-agent should be ready") + + // Write test file, sync, and verify in one command to ensure data persistence + t.Log("Writing test file to volume...") + output, code, err := execWithRetry(ctx, writerInst.VsockSocket, []string{ + "/bin/sh", "-c", "echo 'Hello from writer' > /data/test.txt && sync && cat /data/test.txt", + }) + require.NoError(t, err) + require.Equal(t, 0, code, "Write+verify command should succeed") + require.Contains(t, output, "Hello from writer", "File should contain test data") + t.Log("Test file written successfully") + + // Delete writer instance (detaches volume) + t.Log("Deleting writer instance...") + err = manager.DeleteInstance(ctx, writerInst.Id) + require.NoError(t, err) + + // Verify volume is detached + vol, err = volumeManager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Empty(t, vol.Attachments, "Volume should be detached") + + // Phase 2: Create two instances with the volume attached read-only + t.Log("Phase 2: Creating two reader instances with read-only volume...") + + reader1, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "reader-1", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: true}, + }, + }) + require.NoError(t, err) + t.Logf("Reader 1 created: %s", reader1.Id) + + // Reader 2 uses overlay mode: can read base data AND write to its own overlay + reader2, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "reader-2-overlay", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: true, Overlay: true, OverlaySize: 100 * 1024 * 1024}, // 100MB overlay + }, + }) + require.NoError(t, err) + t.Logf("Reader 2 (overlay) created: %s", reader2.Id) + + // Verify volume has two attachments + vol, err = volumeManager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 2, "Volume should have 2 attachments") + + // Wait for exec-agent on both readers + err = waitForExecAgent(ctx, manager, reader1.Id, 15*time.Second) + require.NoError(t, err, "reader-1 exec-agent should be ready") + + err = waitForExecAgent(ctx, manager, reader2.Id, 15*time.Second) + require.NoError(t, err, "reader-2 exec-agent should be ready") + + // Verify data is readable from reader-1 + t.Log("Verifying data from reader-1...") + output1, code, err := execWithRetry(ctx, reader1.VsockSocket, []string{"cat", "/data/test.txt"}) + require.NoError(t, err) + require.Equal(t, 0, code) + require.Contains(t, output1, "Hello from writer", "Reader 1 should see the file") + + // Verify data is readable from reader-2 (overlay mode) + t.Log("Verifying data from reader-2 (overlay)...") + output2, code, err := execWithRetry(ctx, reader2.VsockSocket, []string{"cat", "/data/test.txt"}) + require.NoError(t, err) + require.Equal(t, 0, code) + assert.Contains(t, output2, "Hello from writer", "Reader 2 should see the file from base volume") + + // Verify overlay allows writes: append to the file and verify in one command + t.Log("Verifying overlay allows writes (append to file)...") + output2, code, err = execWithRetry(ctx, reader2.VsockSocket, []string{ + "/bin/sh", "-c", "echo 'Appended by overlay' >> /data/test.txt && sync && cat /data/test.txt", + }) + require.NoError(t, err) + require.Equal(t, 0, code, "Append to file should succeed with overlay") + assert.Contains(t, output2, "Hello from writer", "Reader 2 should still see original data") + assert.Contains(t, output2, "Appended by overlay", "Reader 2 should see appended data") + + // Verify reader-1 does NOT see the appended data AND write fails (all in one command) + t.Log("Verifying read-only enforcement and isolation on reader-1...") + output1, code, err = execWithRetry(ctx, reader1.VsockSocket, []string{ + "/bin/sh", "-c", "cat /data/test.txt && echo 'illegal' > /data/illegal.txt", + }) + require.NoError(t, err, "Exec should succeed even if write command fails") + // Code should be non-zero because the write fails + assert.NotEqual(t, 0, code, "Write to read-only volume should fail with non-zero exit") + assert.Contains(t, output1, "Hello from writer", "Reader 1 should see original data") + assert.NotContains(t, output1, "Appended by overlay", "Reader 1 should NOT see overlay data (isolated)") + + t.Log("Multi-attach with overlay test passed!") + + // Cleanup + t.Log("Cleaning up...") + manager.DeleteInstance(ctx, reader1.Id) + manager.DeleteInstance(ctx, reader2.Id) + volumeManager.DeleteVolume(ctx, vol.Id) +} + +// TestOverlayDiskCleanupOnDelete verifies that vol-overlays/ directory is removed +// when an instance with overlay volumes is deleted. +func TestOverlayDiskCleanupOnDelete(t *testing.T) { + // Skip in short mode - this is an integration test + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + // Require KVM access + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available - skipping VM test") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + p := paths.New(tmpDir) + + // Setup: prepare image and system files + imageManager, err := images.NewManager(p, 1) + require.NoError(t, err) + + t.Log("Pulling alpine image...") + _, err = imageManager.CreateImage(ctx, images.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, "docker.io/library/alpine:latest") + if err == nil && img.Status == images.StatusReady { + break + } + time.Sleep(1 * time.Second) + } + + systemManager := system.NewManager(p) + err = systemManager.EnsureSystemFiles(ctx) + require.NoError(t, err) + + // Create volume + volumeManager := volumes.NewManager(p, 0) + vol, err := volumeManager.CreateVolume(ctx, volumes.CreateVolumeRequest{ + Name: "cleanup-test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Create instance with overlay volume + inst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "overlay-cleanup-test", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Volumes: []VolumeAttachment{ + {VolumeID: vol.Id, MountPath: "/data", Readonly: true, Overlay: true, OverlaySize: 100 * 1024 * 1024}, + }, + }) + require.NoError(t, err) + + // Verify vol-overlays directory exists + overlaysDir := p.InstanceVolumeOverlaysDir(inst.Id) + _, err = os.Stat(overlaysDir) + require.NoError(t, err, "vol-overlays directory should exist after instance creation") + + // Verify overlay disk file exists + overlayDisk := p.InstanceVolumeOverlay(inst.Id, vol.Id) + _, err = os.Stat(overlayDisk) + require.NoError(t, err, "overlay disk file should exist after instance creation") + + // Delete the instance + err = manager.DeleteInstance(ctx, inst.Id) + require.NoError(t, err) + + // Verify instance directory is removed (which includes vol-overlays/) + instanceDir := p.InstanceDir(inst.Id) + _, err = os.Stat(instanceDir) + assert.True(t, os.IsNotExist(err), "instance directory should be removed after deletion") + + // Cleanup + volumeManager.DeleteVolume(ctx, vol.Id) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e5e3eee5..9629cf2a 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -257,8 +257,8 @@ type InstanceState string // Volume defines model for Volume. type Volume struct { - // AttachedTo Instance ID if attached - AttachedTo *string `json:"attached_to"` + // Attachments List of current attachments (empty if not attached) + Attachments *[]VolumeAttachmentInfo `json:"attachments,omitempty"` // CreatedAt Creation timestamp (RFC3339) CreatedAt time.Time `json:"created_at"` @@ -266,9 +266,6 @@ type Volume struct { // Id Unique identifier Id string `json:"id"` - // MountPath Mount path if attached - MountPath *string `json:"mount_path"` - // Name Volume name Name string `json:"name"` @@ -281,6 +278,12 @@ type VolumeAttachment struct { // MountPath Path where volume is mounted in the guest MountPath string `json:"mount_path"` + // Overlay Create per-instance overlay for writes (requires readonly=true) + Overlay *bool `json:"overlay,omitempty"` + + // OverlaySize Max overlay size as human-readable string (e.g., "1GB"). Required if overlay=true. + OverlaySize *string `json:"overlay_size,omitempty"` + // Readonly Whether volume is mounted read-only Readonly *bool `json:"readonly,omitempty"` @@ -288,6 +291,18 @@ type VolumeAttachment struct { VolumeId string `json:"volume_id"` } +// VolumeAttachmentInfo defines model for VolumeAttachmentInfo. +type VolumeAttachmentInfo struct { + // InstanceId ID of the instance this volume is attached to + InstanceId string `json:"instance_id"` + + // MountPath Mount path in the guest + MountPath string `json:"mount_path"` + + // Readonly Whether the attachment is read-only + Readonly bool `json:"readonly"` +} + // GetInstanceLogsParams defines parameters for GetInstanceLogs. type GetInstanceLogsParams struct { // Tail Number of lines to return from end @@ -4972,64 +4987,67 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xcC28Tu5f/Kpb3f6V2NXm2ZWn+Wq1KeVWiULUXrrSULc74JPHFYw+2JyWgfveVHzOZ", - "yUwehTaXCiQkknn4vH7naaffcCyTVAoQRuPBN6zjCSTEfTwyhsSTd5JnCZzD5wy0sZdTJVNQhoF7KJGZ", - "MFcpMRP7jYKOFUsNkwIP8BkxE3Q9AQVo6lZBeiIzTtEQkHsPKI4wfCFJygEPcCcRpkOJITjCZpbaS9oo", - "Jsb4JsIKCJWCzzyZEcm4wYMR4RqiBbKndmlENLKvtNw7xXpDKTkQgW/cip8zpoDiwfuyGB+Kh+Xwb4iN", - "JX6sgBg4Sch4uSYESaCugzfHJ4jZ95CCESgQMaAdaI/bEaIy/gSqzWSHs6EiatYRYya+DDgxoM1uRTWr", - "n63ra0E8x9sKwYQ2RMTLZQMxtf8RSpmVi/Czyu2asao6eCamTEmRgDBoShQjQw66LN43/PrN02dXz16/", - "wwNLmWaxezXCZ2/O/8QDvNftdu26Nf4n0qQ8G19p9hUqyMB7L57gRUaOCv5RAolUMzSSCoU10M4kS4ho", - "WdRYDu29hBjE2SdAl3a9SxyhS9x7cYmrxuk7UjUlOLNvhIg1piY8ZQKW2jpaAr2XVXHsQ2iHy2tQMdGA", - "OBgDSkeIsjEzOkJEUESJnoBG1mn+jWIihDRIG6IMkgqBoOiamQki7rmqEpJZ61qqT1wS2urhCCfkyysQ", - "YxsXHu1FOCWWmmXr/96T1tdu6/DDTvjQ+vCf+aXd//lXo3xg7Np1EV/7GyiWYsTGmSL2ujOqmQBiAdY4", - "qsHZaoRWAGNUVoskf03ATEAhIxFxwbBY0l6yJMLrKOewpBG/YEPcqYFYTkFxMmsAca/bgOK/FDPOouE9", - "RJn+hOzLayBsV/MYPujWQdxtRnEDUw08PbGICj61CScFI73+afjY39SvpnGa6QpL/UV2XmfJEBSSIzRl", - "ymSEo+Ozt5WQ0y8WZsLAGJRb2WUpXceZT4K6BIRg/wIPxKDYxlKLP8MSizlmIHFr/UvBCA/wf3TmqbYT", - "8mzHr+xTrQ2QpShHlCKz5lCeB5flIX1N2ma0ITClITbGmTYyQYyCMGzEQKEdkhnZGoMARQxQxEbIRoZU", - "ySmjQKtmm0reslnchYENY5VnFwXhKlHFLeUtswyfV+NhfckLC0Mm0JiNyXBmqhmn163bv1nR+fpNqn6m", - "lFR15caSNoh4lKacxQ4hLZ1CzEYsRmBXQPYFtJOQeMIEFD5T1eqQ0CsVzBk1ZVxDGG+AbinneWLhSbRj", - "w2SSccNSDv6e3t0Utk7yp26lOmIjzIQAdQW5em6xUgJaN6bNhWyWy1I84qI+hWE2HluVlFV3yrRmYoxy", - "66IRA04HPguvrZ2cNeeMLcVBkGFDNLyyebjFYQq8DALvUZbZRCpABU680SpSMTElnNErJtKsERJLVfk8", - "Uy6t+UURGcrMuGjmDVYm4kpe5+sjmQnaqKyaOl4C4b4fqGpCG2KykICzxOpWfrL6nJOTn9aaIyzSZIaT", - "vOBaMEDSEOyOT5+ikZKJLR0MYQIUSsCQ0H0UHL3Hrs7GEW5ZTFECiRRIjkb/thwUrlKPchnnFqcLZUDh", - "IC5XAL0ipoG1ch7RhiQp2jl/fry3t3e4mLL7B61ur9U7+LPXHXTtv//FEfap1laSxEArJKN6wGDjkBmq", - "1M9BSz4FihIi2Ai0QeHJMmU9If2DRwMyjHv9PQqj/YNH7Xa7iQwIo2apZKKB1LPi3mam6PjSuDVfs60n", - "P2aHe2hsNpHlGz47+vOlbXkzrTpcxoR39JCJQel78XV+w33wX4dMNDZERcxd4NSFmBARbPr2boSYRiPC", - "+EIjnmach+sDK4mAuACkdMFmiV7XpfnXFpqcfQWKGhtjQ8a20fCI+7EOOMKfM8jgKpWaeeq18US4Y4uE", - "YcY4Re4NtGOFy0scd6la4PSXil8qJV3Z4MuOGuGnRb1uKdtnAs1MGMbd2GJWoXiw9+jxf3UPe/2SczNh", - "Hu3jjVgpwu5Cze5kDnejIianIKjPoBYG/lMsxdR6hfvi+LNxxgOnEsDzezVj2O6IifEVZQ3o/MvfRJQp", - "iI3ry9f7EO6QNF0PxeaqrohphfiliNyYW/Jesp5e7j6U790ulN/PdKY+ayH6SguS6olsEDXvlQnKn0Hw", - "hWmjQzvOdLkfLyQPE7zFNrlpslOpBsPMZkXLudmMpqE0OKr2OplgnzOodEPHb0+e9kNLWyVjvu6Tw8df", - "vhBz+Ihd68OvyVCN/94jD2Q+tHKi86NjGTm6xVSmCVpFs810aMOBfvcgJsIsbbC91mwsgKKTM0QoVaB1", - "OR/ky1eN3jvst3uPHrd73W67190kOyYkXkH79Oh4c+Ldvq/8BmQ4iOkARj+QnYPZ/KSQ8Gsy0+gyH7Nc", - "YnQ9AYGCmRaycxjFbNQf1Odd3zfeWrDC2gHWbQZWG0UPNxldEvov3NT09nH/YGncX2tVm8tgXb+dJ7IL", - "97B7S6bpUiFkeisZ+mty11oZSsO9bQz0FsNIKTjdz/iO2Uq7MsPL7bZxCXKRm7kqUn7bVXQwuBQt5EeB", - "dIDenZ6isDoaZgYVY32gaOeYy4yil7MU1JRpqZAghk1h165wngnBxNiu4KJubO/wGVL++uqXz0imPXX7", - "buq+rX7jYpIZKq+Fe0dPMoPsN8eyFSEUFKuX8HAeoNfSvRM4jWwAXahM/ONE0OGs/vhiFbMTE4GGNilr", - "IxXQ3UtRKpqDpnGEg8ZwhL34OMK5VPaj5859coRLlp47gUdVvdTM0Xpl5Ar7nzy1+SJ/dmFmpE3Ld+2b", - "OONdl7bdw9tOKZrqs7eLBdktps+rtqv9vrG9t1R/5R3q70yyP+GkuxyTciJro1Et8v3g0QCm8zMBVhQb", - "i8fZ4tjpjg4I5PVknfKqEwN5NrlqwmSw6gpMFl63uj+d04hWH0qwgIA4U8zMLmwy8jofAlGgjjKvc5el", - "nBDu8pz4xJgU39y4Sf2oIZa8sN0Pi9HR2Ylr3xIiyNjG+3eniLMRxLOYA8rcVL0WjN0W8Zvjk9aQ2ICf", - "9xCup2TGKcQ+nRBh18cRnoLSnm633W+7jX6ZgiApwwO81+61bU1t1eBE7EyK8fIYHOws6FwsOqGOdxMG", - "0FazOpVCe930u10/jxcm4JXMt2Q6f2s/I/KZfV3eDxScChec0arB9yieUV8D6CxJiJpZ2d1VFE8g/uRu", - "dVwdoJcK9Ippc+If+UGJNipp/BS9XsfUJLV82YossH8T4f1u78407PfWGsi+FSQzE6nYV6CW6MEdmnUp", - "0RNhQAnCkQY1BRV2SspOiAfvq+73/sPNh7LdnbrmukqlbrB16aAR9oEBtHki6ezORGw4ynRTDUI2nd3U", - "kNa/Mw4CwBqU7KYew3ws6atTomci3vXo2oKhnxCK8m3WfwrR+939LSB6YWfvAXnSWca5O60UxtLzvYRy", - "PO18syXNjU9uHHzLVPW2p+567m0pUSQBA0o7DhZsdP6qBSKW1NYnXnWhibN3XY7OK768lKp6VFRS3GIJ", - "8KHmbfsN9b2j6kX5DZMNYOKtmwMjWlot/ID9/VHO+UnOP/rPw1D2j/5zP5b9Y+9ofqDzfsDS3VZozg+Z", - "/AbfWvC9gJDs50pzoSk06WuqveKprRR8+czrNjVfweHvsm+Tsq+srpWV33z+eI/F38Jx743qv7sz8Rxv", - "TQoPQ6wwffil6r6HAmmPIleB+bPhbG7RcozrfGN0k/qrtCG4KgWXB5xu5rus/nLDk7uuvnLiWy/AcsIP", - "Mg26DST3w4JQjJVyzdJ67KfDQ3e7sW/rZdaDhpirtGqqqweiDpfjctm1uNOpgCTzMyq2t9SSA7JvIaLR", - "hWOwdQHCoGdTK137UpyDyZTQblzNiTboNeJMgEY7Vm1Kcg4UDWfoo+XqIyrgvBvZVwSS4cg9n10K+wYT", - "GWikHS9MjJGA67AgG6GPI8m5vP5vi9+Pbbf9tNR3XllZ/yH/iZZv3npZjETKKc4f8wJ3rtjR/ZyBms0J", - "hzPPc1LFXL/XbdzWqG1SBZ02qpSMjDtUwgwjHMnM+HPUTYx4zTezsuQE0wZhxMAX0wGLpZbnr+pQi3qt", - "F+NyHARDOxcXz3Z/B4wNc5JTWeHpzsODAhvCRtjwdftbjZX7uX/gl09b+c74PwzD/e7h/ZM+lmLEWWxQ", - "a44jywUTtiQWdDhztp0fOXhIDhIAPZfMhekgV6OP5PeW+kg47fDL+8gcH7+4l8RSKYiNP6z0sDYfSuVm", - "yd133Pmm+bmhKG953p2eNieWcDit881/OFnXK8//GsNPU9mFYw/ryOQCPghfDTJRCOd/tu6nsjiZ8kC3", - "XdwPpIMILnWUu/7m/FD+WyO/DrrvfsDb9DdbNhrvbtW3irN1P4tvbTsbBh4Id7+Xqujjobi5R1ouiZEL", - "Q+DSyeul21zhEPZWNrlCaLnFFlcuwe/dgA02uErKWrW9VQT4+9vc+o7Yd3fGzVG2NPL93tb66be1prkN", - "51Fsw42szcqXDauK+9jEKkrb7W5hvft5Mm7pl4MP8CjTtEhiy/bOfi4IdrcXWLe9Z/buAXdoLyBP2KX9", - "MreAXbEJMK9kTDiiMAUuU/dLcf8sjnCmePhBwaDj/3LFRGozeNx93MU3H27+PwAA//8Mq1KNYFIAAA==", + "H4sIAAAAAAAC/+xciW4bOZN+FYL7DyAvWqftbKzBYuGxcxiIE8OeZICNsw7VXZI4YZMdki1bCfzuCx7d", + "6kuHE1sT/wkQIFIfZN31VRXlrzgUcSI4cK3w8CtW4RRiYj8eak3C6TvB0hjO4XMKSpvLiRQJSE3BPhSL", + "lOurhOip+RaBCiVNNBUcD/EZ0VN0PQUJaGZXQWoqUhahESD7HkQ4wHBD4oQBHuJuzHU3IprgAOt5Yi4p", + "LSmf4NsASyCR4GzuthmTlGk8HBOmIKhse2qWRkQh80rbvpOvNxKCAeH41q74OaUSIjx8X2TjQ/6wGP0N", + "oTabH0kgGk5iMlkuCU5iqMvgzdEJouY9JGEMEngIqAWdSSdAkQg/gexQ0WV0JImcd/mE8pshIxqU3imJ", + "ZvWzdXlV2LO0rWCMK014uJw34DPzH4kiavgi7Kx0u6assgye8RmVgsfANZoRScmIgSqy9xW/fnP87OrZ", + "63d4aHaO0tC+GuCzN+d/4iHe7fV6Zt0a/VOhE5ZOrhT9AiXLwLsv/sBVQg5z+lEMsZBzNBYS+TVQa5rG", + "hLeN1RgKzb2YaMToJ0CXZr1LHKBL3H9xicvKGditakKwat/IItaomrCEcliq62CJ6b0ss2MeQi0mrkGG", + "RAFioDVIFaCITqhWASI8QhFRU1DIOM3vKCScC42UJlIjIRHwCF1TPUXEPlcWQjxvXwv5iQkStfs4wDG5", + "eQV8YuLCk90AJ8TsZsj6v/ek/aXXPvjQ8h/aH/4zu7TzP/9q5A+0WbvO4mt3A4WCj+kklcRct0rVU0DU", + "mzUOauZsJBKVDEbLtBZJ/pqCnoJEWiBig2G+pLlktvCvo4zCgkTcgg1xp2bEYgaSkXmDEfd7DVb8l6Ta", + "atS/hyKqPiHz8hoTNqs5G97v1Y2412zFDUQ10PSHsSjvU5tQkhPSH5z6j4NN/WoWJqkqkTSokvM6jUcg", + "kRijGZU6JQwdnb0thZxBvjDlGiYg7co2S6m6nbkkqAqG4PWf2wPRKDSx1NifprGxOaohtmv9S8IYD/F/", + "dBeptuvzbNet7FKtCZCFKEekJPPmUJ4Fl+UhfU3aplFDYEp8bAxTpUWMaARc0zEFiVok1aI9AQ6SaIgQ", + "HSMTGRIpZjSCqKy2mWBtk8VtGNgwVjlykWeuFFXsUk4zy+zzajKqL3lhzJByNKETMprrcsbp9+r6bxZ0", + "tn6TqJ9JKWRduKGIGlg8TBJGQ2shbZVASMc0RGBWQOYF1IpJOKUccp8pS3VEoivp1Rk0ZVxNKGsw3ULO", + "c5v5J1HLhMk4ZZomDNw9tbOp2VrOj+1KdYsNMOUc5BVk4rnDSjEo1Zg2K9ks4yV/xEb9CEbpZGJEUhTd", + "KVWK8gnKtIvGFFg0dFl4LXay2lwQttQOPA8bWsMrk4fbDGbAikbgPMoQGwsJKLcTp7QSV5TPCKPRFeVJ", + "2mgSS0X5PJU2rblFERmJVNto5hRW3MRCXuvrY5HyqFFYNXG8BMJcPVCWhNJEpz4Bp7GRrfhk5LnYTnxa", + "qw6/SJMaTjLAVVFA3BDsjk6P0ViK2EAHTSgHiWLQxFcfOUXvscXZOMBtY1MRgVhwJMbj3w0FuavUo1zK", + "mLHTCgzIHcTmCoiuiG4grZhHlCZxglrnz492d3cPqil7sN/u9dv9/T/7vWHP/PtfHGCXag2SJBraPhnV", + "Awad+MxQ3v0clGAziFBMOB2D0sg/WdxZTclg/8mQjML+YDeC8d7+k06n07QNcC3niaC8Yatn+b3NVNF1", + "0Li9WLOjpt+nhwcobDbh5Ss+O/zzpSl5UyW7TISEddWI8mHhe/51ccN+cF9HlDcWRHnMrVBqQ4yPCCZ9", + "OzdCVKExoaxSiCcpY/760HDCIcwNUthgs0Su69L8a2OajH6BCDUWxppMTKHhLO77KuAAf04hhatEKOp2", + "r7Un/B0DEkYpZRGyb6CWYS6DOPZSGeAMlrJfgJIWNjjYUdv4OMfrZmfzjN8z5Zoy27aYl3bc333y9L96", + "B/1Bwbkp10/28Eak5GG3gtktz/5ukMfkBHjkMqgxA/cpFHxmvMJ+sfSZOOMMpxTAs3s1ZZjqiPLJVUQb", + "rPMvdxNFVEKobV2+3odwlyTJelNsRnV5TMvZL0TkxtyS1ZL19HL/oXz3bqH8Yboz9V4LUVeKk0RNRQOr", + "Wa1MUPYMghuqtPLlOFXFejzn3HfwqmVyU2enhAZ9z2ZFyblZj6YBGhyWa52U088plKqho7cnxwNf0pa3", + "0V/2yMHTmxuiD57Qa3XwJR7Jyd+75JH0h1Z2dL63LSPGd+jKNJlWXmxT5ctwiL65ERNgmjToXik64RCh", + "kzNEokiCUsV8kC1fVnr/YNDpP3na6fd6nX5vk+wYk3DF3qeHR5tv3hs45Dcko2EYDWH8HdnZq811Cgm7", + "JnOFLrM2yyVG11PgyKupkp19K2aj+qDe7/q29lZFC2sbWHdpWG0UPWxndEnov7Bd07vH/f2lcX+tVk0u", + "g3X1dpbILuzD9i2RJEuZEMmdeBisyV1reSg097bR0KuGkUJwepj2HTVIu9TDy/S2MQS5yNRcZim7bREd", + "DC95G7lWYDRE705PkV8djVKN8rY+RKh1xEQaoZfzBOSMKiERJ5rOYMescJ5yTvnErGCjbmjusDmS7vrq", + "l89Iqtzu5t3Eflv9xsU01ZG45vYdNU01Mt8syYYFDyhWL+HMeYheC/uOpzQwAbSCTNzjhEejef3xKopp", + "hYSjkUnKSgsJ0c4lL4BmL2kcYC8xHGDHPg5wxpX56Kizn+zGBU0vnMBZVR1qktzOGkz6FVXaOEiYSmmw", + "XOFh1II40fOspsmMfudbrfyEj0VT2+++oXDv4K5djSY897YK4P4du9XFuJJtsjai1KLXd473qcrm+oYV", + "E08nabV1tHLI71P++hm/8zeUgGznqDDDC6beuJbUltVeRO4sgOBs/t8m7+zgJjy4Gpackpt8BwsYiEKV", + "GZfjIxvv+ynXTgedZ31nOs6WsGR0yvilGWNsfu4hg8l1Zaw6CJElyasm1/GGvsJ1XAtwbdd2sUew7qxF", + "Y7CpD7C81hvJPjmuFhuuAF1IppD3K011pZfyFKx0B3fsxNz7RuMva7pxDj2FQlg3fBQ1u66mrkaMggRL", + "nBUoqevHxDAIU0n1/MJkB6eNERAJ8jB1crFpw25tLy94nWqd4NtbOyBySi3z+cIU3TREh2cn1otjwsnE", + "uNS7U8ToGMJ5yACldphTwwD2ZMKbo5P2iBickZWutpVBtRW/eTom3KyPAzwDqdy+vc6gY8+XiAQ4SSge", + "4t1Ov2NKOSMRy2J3mk81JmAjpTFHm9JOIku79nMPIz+VCK6cbAa9nhsDce1DLFlMArt/K9eadKl2XSL2", + "O1gRVvKHEYMrjR2hDnqqNI6JnBve7VUUTiH8ZG91LfxUSxkyeOLEPfKdHG2EMdzwpg6fa5xmOMeTfxvg", + "vV7/3iTsRroN277lJNVTIekXiMym+/eo1qWbnnANkhOGFMgZSD+gKzohHr4vu9/7D7cfinq34lrIKhGq", + "QdeF823YRQlQ+g8Rze+NxYYTdLfliGQy4m3N0gb3RoE3sAYh22bbKOuGu6KIqDkPd5x1bUHRf5AIZdP9", + "f8qi93p7W7DoykD5EXnSWcqYPSTnpyGLEVYxnna/GhR+65IbA1epl73t2F7PvC0hksSgQSpLQUVH56/a", + "wEMRGfToROd7B+auT9euSMnQf9mjgoLgqhDtQ83b9hqwlN3VsfLLTDYwE6fdzDCCpWjhO/TvSozFAeLf", + "Bs/9LOC3wXM3Dfht93BxjvhhjKW3rdCcnW36ZXxrje8F+GS/EJoNTR7rr0F7+VNbAXxZq/UumC+n8Bfs", + "2wT2FcW1Evkt2t4PCP4qvzLYCP/dn4oX9tYkcN8m8A2znwr3PRaT9j0/g8DcTxLoQqPFGNf9SqNN8Fdh", + "Dr0qBee2cXKM7KhhGf6yfZT7Rl/Z5lsHYNnGjzIN2rml/T2LB2OFXLMUj/1w9tDbbuzbOsx61CZmkVZN", + "dPVA1GViUoRd1QG7BBIvjkaZ2lIJBsi8hYhCF5bA9gVwjZ7NDHedS34OOpVc2X4wI0qj14hRDgq1jNik", + "YAwiNJqjj4aqjyg3553AvMKR8L/0YPNLbt6gPAWFlKWF8gnicO0XpGP0cSwYE9d2YvGxY6eeS33nleH1", + "H/KfYPmZAceLFkhawbnThWCPs9t9P6cg54uN/VH7xVb53KXfa5zE1WadXqaNIiVjbc8yUU0JQyLV7vh+", + "EyFO8s2kLGvyrw8jGm50F4wttR19ZYeqyrUOxsXEM4ZaFxfPdn4FjA1zkhVZ7unWw70AG8KGP2dgJ1+N", + "yP3cPfDTp63sQMY/bIZ7vYOH3/pI8DGjoUbthR0ZKig3kJhHo7nV7eKky2NyEG/QC85smPZ8NfpIdm+p", + "j/hDNj+9jyzs4yf3klBICaF2Z+Qe1/ChADcL7t6yx+oWx9WCrOR5d3ranFj8mcjuV/fhZF2tvPgjID8M", + "svPHUtZtkzH4KHzV8xSBO5iyfT8V+cmhRzp2sb/L9yzY1FGs+pvzQ/FP3Pw81n3/Dd6mPxW0UXt3q76V", + "Hfr6YXxr29nQ00CY/ZleSR6Pxc2dpWWcaFFpAhcO/C8dc/mz/1sZcvnQcocRV8bBr2nABgOugrBWjbfy", + "AP9ww61viH33p9zMypZGvl9jrR9+rDXLdLiIYhsOsjaDLxuiiocYYuXQdrsjrHc/TsYtnDh/hEeZZnkS", + "WzY7+7FMsLe9wLrtmdm7R1yhvYAsYRfmZXYBs2KTwbwSIWEoghkwkdifPLhncYBTyfwPCoZd9wdTpkLp", + "4dPe0x6+/XD7/wEAAP//oKJwbNdUAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 4dedc604..05a566d8 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -27,6 +27,8 @@ // ch.sock // vsock.sock // logs/ +// vol-overlays/ +// {volumeID}.raw // snapshots/ // snapshot-latest/ // config.json @@ -140,6 +142,16 @@ func (p *Paths) InstanceConfigDisk(id string) string { return filepath.Join(p.InstanceDir(id), "config.ext4") } +// InstanceVolumeOverlay returns the path to a volume's overlay disk for an instance. +func (p *Paths) InstanceVolumeOverlay(instanceID, volumeID string) string { + return filepath.Join(p.InstanceDir(instanceID), "vol-overlays", volumeID+".raw") +} + +// InstanceVolumeOverlaysDir returns the directory for volume overlays. +func (p *Paths) InstanceVolumeOverlaysDir(instanceID string) string { + return filepath.Join(p.InstanceDir(instanceID), "vol-overlays") +} + // InstanceSocket returns the path to instance API socket. func (p *Paths) InstanceSocket(id string) string { return filepath.Join(p.InstanceDir(id), "ch.sock") diff --git a/lib/system/init_script.go b/lib/system/init_script.go index fdfcb5fe..2c6a5e97 100644 --- a/lib/system/init_script.go +++ b/lib/system/init_script.go @@ -71,7 +71,8 @@ else exit 1 fi -# Mount attached volumes (from config: VOLUME_MOUNTS="device:path:mode device:path:mode ...") +# Mount attached volumes (from config: VOLUME_MOUNTS="device:path:mode[:overlay_device] ...") +# Modes: ro (read-only), rw (read-write), overlay (base ro + per-instance overlay) if [ -n "${VOLUME_MOUNTS:-}" ]; then echo "overlay-init: mounting volumes" for vol in $VOLUME_MOUNTS; do @@ -82,13 +83,40 @@ if [ -n "${VOLUME_MOUNTS:-}" ]; then # Create mount point in overlay mkdir -p "/overlay/newroot${path}" - # Mount with appropriate options - if [ "$mode" = "ro" ]; then - mount -t ext4 -o ro "$device" "/overlay/newroot${path}" + if [ "$mode" = "overlay" ]; then + # Overlay mode: mount base read-only, create overlayfs with per-instance writable layer + overlay_device=$(echo "$vol" | cut -d: -f4) + + # Create temp mount points for base and overlay disk. + # These persist for the lifetime of the VM but are NOT leaked - they exist inside + # the ephemeral guest rootfs (which is itself an overlayfs) and are destroyed + # when the VM terminates along with all guest state. + base_mount="/mnt/vol-base-$(basename "$path")" + overlay_mount="/mnt/vol-overlay-$(basename "$path")" + mkdir -p "$base_mount" "$overlay_mount" + + # Mount base volume read-only (noload to skip journal recovery) + mount -t ext4 -o ro,noload "$device" "$base_mount" + + # Mount overlay disk (writable) + mount -t ext4 "$overlay_device" "$overlay_mount" + mkdir -p "$overlay_mount/upper" "$overlay_mount/work" + + # Create overlayfs combining base (lower) and instance overlay (upper) + mount -t overlay \ + -o "lowerdir=$base_mount,upperdir=$overlay_mount/upper,workdir=$overlay_mount/work" \ + overlay "/overlay/newroot${path}" + + echo "overlay-init: mounted volume $device at $path (overlay via $overlay_device)" + elif [ "$mode" = "ro" ]; then + # Read-only mount (noload to skip journal recovery for multi-attach safety) + mount -t ext4 -o ro,noload "$device" "/overlay/newroot${path}" + echo "overlay-init: mounted volume $device at $path (ro)" else + # Read-write mount mount -t ext4 "$device" "/overlay/newroot${path}" + echo "overlay-init: mounted volume $device at $path (rw)" fi - echo "overlay-init: mounted volume $device at $path ($mode)" done fi diff --git a/lib/volumes/README.md b/lib/volumes/README.md index 665ab931..10bbb9b9 100644 --- a/lib/volumes/README.md +++ b/lib/volumes/README.md @@ -21,11 +21,31 @@ When an instance with volumes is created, each volume's raw disk file is passed The init process inside the guest reads the requested mount paths from the config disk and mounts each volume at its specified path. +## Multi-Attach (Read-Only Sharing) + +A single volume can be attached to multiple instances simultaneously if **all** attachments are read-only. This enables sharing static content (libraries, datasets, config files) across many VMs without duplication. + +**Rules:** +- First attachment can be read-write or read-only +- If any attachment is read-write, no other attachments are allowed +- If all existing attachments are read-only, additional read-only attachments are allowed +- Cannot add read-write attachment to a volume with existing attachments + +## Overlay Mode + +When attaching a volume with `overlay: true`, the instance gets copy-on-write semantics: +- Base volume is mounted read-only (shared) +- A per-instance overlay disk captures all writes +- Instance sees combined view: base data + its local changes +- Other instances don't see each other's overlay writes (isolated) + +This allows multiple instances to share a common base (e.g., dataset, model weights) while each can make local modifications without affecting others. Requires `readonly: true` and `overlay_size` specifying the max size of per-instance writes. + ## Constraints - Volumes can only be attached at instance creation time (no hot-attach) -- A volume can only be attached to one instance at a time - Deleting an instance detaches its volumes but does not delete them +- Cannot delete a volume while it has any attachments ## Storage diff --git a/lib/volumes/manager.go b/lib/volumes/manager.go index ff9b5fe7..b82297d6 100644 --- a/lib/volumes/manager.go +++ b/lib/volumes/manager.go @@ -19,8 +19,12 @@ type Manager interface { DeleteVolume(ctx context.Context, id string) error // Attachment operations (called by instance manager) + // Multi-attach rules: + // - If no attachments: allow any mode (rw or ro) + // - If existing attachment is rw: reject all new attachments + // - If existing attachments are ro: only allow new ro attachments AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error - DetachVolume(ctx context.Context, id string) error + DetachVolume(ctx context.Context, volumeID string, instanceID string) error // GetVolumePath returns the path to the volume data file GetVolumePath(id string) string @@ -191,8 +195,8 @@ func (m *manager) DeleteVolume(ctx context.Context, id string) error { return err } - // Check if volume is attached - if meta.AttachedTo != nil { + // Check if volume has any attachments + if len(meta.Attachments) > 0 { return ErrInUse } @@ -208,6 +212,10 @@ func (m *manager) DeleteVolume(ctx context.Context, id string) error { } // AttachVolume marks a volume as attached to an instance +// Multi-attach rules (dynamic based on current state): +// - If no attachments: allow any mode (rw or ro) +// - If existing attachment is rw: reject all new attachments +// - If existing attachments are ro: only allow new ro attachments func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error { lock := m.getVolumeLock(id) lock.Lock() @@ -218,35 +226,64 @@ func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeR return err } - // Check if already attached - if meta.AttachedTo != nil { - return ErrInUse + // Check if this instance is already attached + for _, att := range meta.Attachments { + if att.InstanceID == req.InstanceID { + return fmt.Errorf("volume already attached to instance %s", req.InstanceID) + } + } + + // Apply multi-attach rules + if len(meta.Attachments) > 0 { + // Check if any existing attachment is read-write + for _, att := range meta.Attachments { + if !att.Readonly { + return fmt.Errorf("volume has exclusive read-write attachment to instance %s", att.InstanceID) + } + } + // Existing attachments are all read-only, new attachment must also be read-only + if !req.Readonly { + return fmt.Errorf("cannot attach read-write: volume has existing read-only attachments") + } } - // Update attachment info - meta.AttachedTo = &req.InstanceID - meta.MountPath = &req.MountPath - meta.Readonly = req.Readonly + // Add new attachment + meta.Attachments = append(meta.Attachments, storedAttachment{ + InstanceID: req.InstanceID, + MountPath: req.MountPath, + Readonly: req.Readonly, + }) return saveMetadata(m.paths, meta) } -// DetachVolume marks a volume as detached from an instance -func (m *manager) DetachVolume(ctx context.Context, id string) error { - lock := m.getVolumeLock(id) +// DetachVolume removes the attachment for a specific instance +func (m *manager) DetachVolume(ctx context.Context, volumeID string, instanceID string) error { + lock := m.getVolumeLock(volumeID) lock.Lock() defer lock.Unlock() - meta, err := loadMetadata(m.paths, id) + meta, err := loadMetadata(m.paths, volumeID) if err != nil { return err } - // Clear attachment info - meta.AttachedTo = nil - meta.MountPath = nil - meta.Readonly = false + // Find and remove the attachment for this instance + found := false + newAttachments := make([]storedAttachment, 0, len(meta.Attachments)) + for _, att := range meta.Attachments { + if att.InstanceID == instanceID { + found = true + continue // Skip this attachment (remove it) + } + newAttachments = append(newAttachments, att) + } + + if !found { + return fmt.Errorf("volume not attached to instance %s", instanceID) + } + meta.Attachments = newAttachments return saveMetadata(m.paths, meta) } @@ -259,13 +296,21 @@ func (m *manager) GetVolumePath(id string) string { func (m *manager) metadataToVolume(meta *storedMetadata) *Volume { createdAt, _ := time.Parse(time.RFC3339, meta.CreatedAt) + // Convert stored attachments to domain attachments + attachments := make([]Attachment, len(meta.Attachments)) + for i, att := range meta.Attachments { + attachments[i] = Attachment{ + InstanceID: att.InstanceID, + MountPath: att.MountPath, + Readonly: att.Readonly, + } + } + return &Volume{ - Id: meta.Id, - Name: meta.Name, - SizeGb: meta.SizeGb, - CreatedAt: createdAt, - AttachedTo: meta.AttachedTo, - MountPath: meta.MountPath, - Readonly: meta.Readonly, + Id: meta.Id, + Name: meta.Name, + SizeGb: meta.SizeGb, + CreatedAt: createdAt, + Attachments: attachments, } } diff --git a/lib/volumes/manager_test.go b/lib/volumes/manager_test.go new file mode 100644 index 00000000..3752e230 --- /dev/null +++ b/lib/volumes/manager_test.go @@ -0,0 +1,392 @@ +package volumes + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/onkernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupTestManager(t *testing.T) (Manager, *paths.Paths, func()) { + t.Helper() + + // Create temp dir + tmpDir, err := os.MkdirTemp("", "volume-test-*") + require.NoError(t, err) + + p := paths.New(tmpDir) + + // Create required directories + require.NoError(t, os.MkdirAll(p.VolumesDir(), 0755)) + + manager := NewManager(p, 0) // 0 = unlimited storage + + cleanup := func() { + os.RemoveAll(tmpDir) + } + + return manager, p, cleanup +} + +func TestMultiAttach_FirstAttachmentRW(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-write should succeed + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: false, + }) + assert.NoError(t, err) + + // Verify attachment + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + require.Len(t, vol.Attachments, 1) + assert.Equal(t, "instance-1", vol.Attachments[0].InstanceID) + assert.Equal(t, "/data", vol.Attachments[0].MountPath) + assert.False(t, vol.Attachments[0].Readonly) +} + +func TestMultiAttach_FirstAttachmentRO(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-only should succeed + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + assert.NoError(t, err) + + // Verify attachment + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + require.Len(t, vol.Attachments, 1) + assert.True(t, vol.Attachments[0].Readonly) +} + +func TestMultiAttach_RejectSecondAttachWhenRW(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-write + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: false, + }) + require.NoError(t, err) + + // Second attachment (either RO or RW) should fail + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: true, // Even RO should fail when existing is RW + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "exclusive read-write attachment") +} + +func TestMultiAttach_AllowMultipleRO(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-only + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Second attachment as read-only should succeed + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: true, + }) + assert.NoError(t, err) + + // Verify both attachments + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 2) +} + +func TestMultiAttach_RejectRWWhenExistingRO(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment as read-only + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Second attachment as read-write should fail + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: false, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot attach read-write") +} + +func TestMultiAttach_RejectDuplicateInstance(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First attachment + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Same instance trying to attach again should fail + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/other", + Readonly: true, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "already attached") +} + +func TestDetach_RemovesSpecificAttachment(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Attach to two instances + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Detach instance-1 + err = manager.DetachVolume(ctx, vol.Id, "instance-1") + assert.NoError(t, err) + + // Verify only instance-2 remains + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + require.Len(t, vol.Attachments, 1) + assert.Equal(t, "instance-2", vol.Attachments[0].InstanceID) +} + +func TestDetach_ErrorIfNotAttached(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Detach from instance that's not attached + err = manager.DetachVolume(ctx, vol.Id, "instance-1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not attached") +} + +func TestDeleteVolume_RejectIfAttached(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "test-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Attach it + err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: true, + }) + require.NoError(t, err) + + // Try to delete - should fail + err = manager.DeleteVolume(ctx, vol.Id) + assert.ErrorIs(t, err, ErrInUse) +} + +func TestMultiAttach_ConcurrentAttachments(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "concurrent-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Launch multiple goroutines trying to attach simultaneously + const numGoroutines = 10 + results := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + go func(instanceNum int) { + err := manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: fmt.Sprintf("instance-%d", instanceNum), + MountPath: "/data", + Readonly: true, + }) + results <- err + }(i) + } + + // Collect results + var successCount, errorCount int + for i := 0; i < numGoroutines; i++ { + err := <-results + if err == nil { + successCount++ + } else { + errorCount++ + } + } + + // All should succeed since all are read-only + assert.Equal(t, numGoroutines, successCount, "All read-only attachments should succeed") + assert.Equal(t, 0, errorCount, "No errors expected for concurrent read-only attachments") + + // Verify final state has all attachments + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, numGoroutines, "Should have all attachments") +} + +func TestMultiAttach_ConcurrentRWConflict(t *testing.T) { + manager, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + // Create a volume + vol, err := manager.CreateVolume(ctx, CreateVolumeRequest{ + Name: "rw-conflict-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Launch multiple goroutines trying to attach read-write simultaneously + const numGoroutines = 5 + results := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + go func(instanceNum int) { + err := manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: fmt.Sprintf("instance-%d", instanceNum), + MountPath: "/data", + Readonly: false, // All trying read-write + }) + results <- err + }(i) + } + + // Collect results + var successCount, errorCount int + for i := 0; i < numGoroutines; i++ { + err := <-results + if err == nil { + successCount++ + } else { + errorCount++ + } + } + + // Only ONE should succeed (first one gets exclusive lock) + assert.Equal(t, 1, successCount, "Exactly one read-write attachment should succeed") + assert.Equal(t, numGoroutines-1, errorCount, "Others should fail due to exclusive lock") + + // Verify final state has exactly one attachment + vol, err = manager.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 1, "Should have exactly one attachment") + assert.False(t, vol.Attachments[0].Readonly, "Attachment should be read-write") +} + diff --git a/lib/volumes/storage.go b/lib/volumes/storage.go index 9ac7cd2f..57387bee 100644 --- a/lib/volumes/storage.go +++ b/lib/volumes/storage.go @@ -15,15 +15,20 @@ import ( // data.raw # ext4-formatted sparse disk // metadata.json # Volume metadata +// storedAttachment represents an attachment in stored metadata +type storedAttachment struct { + InstanceID string `json:"instance_id"` + MountPath string `json:"mount_path"` + Readonly bool `json:"readonly"` +} + // storedMetadata represents volume metadata that is persisted to disk type storedMetadata struct { - Id string `json:"id"` - Name string `json:"name"` - SizeGb int `json:"size_gb"` - CreatedAt string `json:"created_at"` // RFC3339 format - AttachedTo *string `json:"attached_to,omitempty"` - MountPath *string `json:"mount_path,omitempty"` - Readonly bool `json:"readonly,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + SizeGb int `json:"size_gb"` + CreatedAt string `json:"created_at"` // RFC3339 format + Attachments []storedAttachment `json:"attachments,omitempty"` } // ensureVolumeDir creates the volume directory diff --git a/lib/volumes/types.go b/lib/volumes/types.go index b2505b9e..d0810b60 100644 --- a/lib/volumes/types.go +++ b/lib/volumes/types.go @@ -2,17 +2,20 @@ package volumes import "time" +// Attachment represents a volume attached to an instance +type Attachment struct { + InstanceID string + MountPath string + Readonly bool +} + // Volume represents a persistent block storage volume type Volume struct { - Id string - Name string - SizeGb int - CreatedAt time.Time - - // Attachment state (nil/empty if not attached) - AttachedTo *string // Instance ID if attached - MountPath *string // Mount path in guest if attached - Readonly bool // Whether mounted read-only + Id string + Name string + SizeGb int + CreatedAt time.Time + Attachments []Attachment // List of current attachments (empty if not attached) } // CreateVolumeRequest is the domain request for creating a volume diff --git a/openapi.yaml b/openapi.yaml index 975e3ab7..bd6ac20e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -71,6 +71,14 @@ components: type: boolean description: Whether volume is mounted read-only default: false + overlay: + type: boolean + description: Create per-instance overlay for writes (requires readonly=true) + default: false + overlay_size: + type: string + description: Max overlay size as human-readable string (e.g., "1GB"). Required if overlay=true. + example: "1GB" PortMapping: type: object @@ -326,6 +334,23 @@ components: description: Size in gigabytes example: 10 + VolumeAttachmentInfo: + type: object + required: [instance_id, mount_path, readonly] + properties: + instance_id: + type: string + description: ID of the instance this volume is attached to + example: inst-abc123 + mount_path: + type: string + description: Mount path in the guest + example: /mnt/data + readonly: + type: boolean + description: Whether the attachment is read-only + example: false + Volume: type: object required: [id, name, size_gb, created_at] @@ -342,16 +367,11 @@ components: type: integer description: Size in gigabytes example: 10 - attached_to: - type: string - description: Instance ID if attached - example: inst-abc123 - nullable: true - mount_path: - type: string - description: Mount path if attached - example: /mnt/data - nullable: true + attachments: + type: array + description: List of current attachments (empty if not attached) + items: + $ref: "#/components/schemas/VolumeAttachmentInfo" created_at: type: string format: date-time