Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
85ce1ab
Ingress feature
sjmiller609 Dec 4, 2025
1aa82fb
fixes
sjmiller609 Dec 4, 2025
2b58ccf
Host port configurable by ingress resource
sjmiller609 Dec 4, 2025
b384a5d
Auto download envoy when missing
sjmiller609 Dec 4, 2025
2abcdd5
envoy starting
sjmiller609 Dec 4, 2025
088b41f
Avoid shared memory conflicts between envoys on same host
sjmiller609 Dec 4, 2025
4b661a8
Use hostname for routing
sjmiller609 Dec 4, 2025
8b58c56
Update default config
sjmiller609 Dec 4, 2025
ce516fd
review tweaks
sjmiller609 Dec 4, 2025
c20e84f
context logger
sjmiller609 Dec 4, 2025
3b7477f
Config validation and OTEL
sjmiller609 Dec 4, 2025
4a6dcc2
Fix otel
sjmiller609 Dec 4, 2025
425ebc9
Envoy provides stats to otel, not per-request tracing
sjmiller609 Dec 4, 2025
256f209
Update stainless
sjmiller609 Dec 4, 2025
9c18d3d
Update stainless again
sjmiller609 Dec 4, 2025
9178103
naming things is hard
rgarcia Dec 5, 2025
45ca7ce
Address pr review comments
sjmiller609 Dec 5, 2025
1608b11
Merge remote-tracking branch 'origin/ingress' into ingress
sjmiller609 Dec 5, 2025
cc16755
Use file-based simple xDS instead of static config+reload
sjmiller609 Dec 5, 2025
29080d4
Fix config validation for dynamic resources, add tests, fix import cycle
sjmiller609 Dec 5, 2025
e174b6f
Otel cluster should be static
sjmiller609 Dec 5, 2025
4ec66f9
e2e test should actually do a http request
sjmiller609 Dec 5, 2025
425d1f7
Give ingress 5s to be ready
sjmiller609 Dec 5, 2025
ea914cb
Up to 30s
sjmiller609 Dec 5, 2025
f51ccb1
Delete comment
sjmiller609 Dec 5, 2025
b3e2680
Fix dynamic config update race condition
sjmiller609 Dec 5, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,6 @@ lib/vmm/binaries/cloud-hypervisor/*/*/cloud-hypervisor
cloud-hypervisor
cloud-hypervisor/**
lib/system/exec_agent/exec-agent

# Envoy binaries
lib/ingress/binaries/envoy/*/*/envoy
29 changes: 25 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
SHELL := /bin/bash
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries download-envoy-binaries ensure-envoy-binaries

# Directory where local binaries will be installed
BIN_DIR ?= $(CURDIR)/bin
Expand Down Expand Up @@ -49,6 +49,19 @@ download-ch-binaries:
@chmod +x lib/vmm/binaries/cloud-hypervisor/v*/*/cloud-hypervisor
@echo "Binaries downloaded successfully"

# Download Envoy binaries
download-envoy-binaries:
@echo "Downloading Envoy binaries..."
@mkdir -p lib/ingress/binaries/envoy/v1.36/{x86_64,aarch64}
@echo "Downloading Envoy v1.36.3 for x86_64..."
@curl -L -o lib/ingress/binaries/envoy/v1.36/x86_64/envoy \
https://github.com/envoyproxy/envoy/releases/download/v1.36.3/envoy-1.36.3-linux-x86_64
@echo "Downloading Envoy v1.36.3 for aarch64..."
@curl -L -o lib/ingress/binaries/envoy/v1.36/aarch64/envoy \
https://github.com/envoyproxy/envoy/releases/download/v1.36.3/envoy-1.36.3-linux-aarch_64
@chmod +x lib/ingress/binaries/envoy/v1.36/*/envoy
@echo "Envoy binaries downloaded successfully"

# Download Cloud Hypervisor API spec
download-ch-spec:
@echo "Downloading Cloud Hypervisor API spec..."
Expand Down Expand Up @@ -86,21 +99,29 @@ generate-grpc:
# Generate all code
generate-all: oapi-generate generate-vmm-client generate-wire generate-grpc

# Check if binaries exist, download if missing
# Check if CH binaries exist, download if missing
.PHONY: ensure-ch-binaries
ensure-ch-binaries:
@if [ ! -f lib/vmm/binaries/cloud-hypervisor/v48.0/x86_64/cloud-hypervisor ]; then \
echo "Cloud Hypervisor binaries not found, downloading..."; \
$(MAKE) download-ch-binaries; \
fi

# Check if Envoy binaries exist, download if missing
.PHONY: ensure-envoy-binaries
ensure-envoy-binaries:
@if [ ! -f lib/ingress/binaries/envoy/v1.36/x86_64/envoy ]; then \
echo "Envoy binaries not found, downloading..."; \
$(MAKE) download-envoy-binaries; \
fi

# Build exec-agent (guest binary) into its own directory for embedding
lib/system/exec_agent/exec-agent: lib/system/exec_agent/main.go
@echo "Building exec-agent..."
cd lib/system/exec_agent && CGO_ENABLED=0 go build -ldflags="-s -w" -o exec-agent .

# Build the binary
build: ensure-ch-binaries lib/system/exec_agent/exec-agent | $(BIN_DIR)
build: ensure-ch-binaries ensure-envoy-binaries lib/system/exec_agent/exec-agent | $(BIN_DIR)
go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api

# Build exec CLI
Expand All @@ -118,7 +139,7 @@ dev: $(AIR)
# Compile test binaries and grant network capabilities (runs as user, not root)
# Usage: make test - runs all tests
# make test TEST=TestCreateInstanceWithNetwork - runs specific test
test: ensure-ch-binaries lib/system/exec_agent/exec-agent
test: ensure-ch-binaries ensure-envoy-binaries lib/system/exec_agent/exec-agent
@echo "Building test binaries..."
@mkdir -p $(BIN_DIR)/tests
@for pkg in $$(go list -tags containers_image_openpgp ./...); do \
Expand Down
22 changes: 22 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,24 @@ getcap ./bin/hypeman

**Note:** These capabilities must be reapplied after each rebuild. For production deployments, set capabilities on the installed binary. For local testing, this is handled automatically in `make test`.

**File Descriptor Limits:**

Envoy (used for ingress) requires a higher file descriptor limit than the default on some systems (root defaults to 1024 on many systems). If you see "Too many open files" errors, increase the limit:

```bash
# Check current limit (also check with: sudo bash -c 'ulimit -n')
ulimit -n

# Increase temporarily (current session)
ulimit -n 65536

# For persistent changes, add to /etc/security/limits.conf:
* soft nofile 65536
* hard nofile 65536
root soft nofile 65536
root hard nofile 65536
```

### Configuration

#### Environment variables
Expand All @@ -81,6 +99,10 @@ Hypeman can be configured using the following environment variables:
| `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers) | hostname |
| `LOG_LEVEL` | Default log level (debug, info, warn, error) | `info` |
| `LOG_LEVEL_<SUBSYSTEM>` | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC) | inherits default |
| `ENVOY_LISTEN_ADDRESS` | Address for Envoy ingress listeners | `0.0.0.0` |
| `ENVOY_ADMIN_ADDRESS` | Address for Envoy admin API | `127.0.0.1` |
| `ENVOY_ADMIN_PORT` | Port for Envoy admin API | `9901` |
| `ENVOY_STOP_ON_SHUTDOWN` | Stop Envoy when hypeman shuts down (if false, Envoy continues running) | `false` |

**Important: Subnet Configuration**

Expand Down
5 changes: 4 additions & 1 deletion cmd/api/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package api
import (
"github.com/onkernel/hypeman/cmd/api/config"
"github.com/onkernel/hypeman/lib/images"
"github.com/onkernel/hypeman/lib/ingress"
"github.com/onkernel/hypeman/lib/instances"
"github.com/onkernel/hypeman/lib/network"
"github.com/onkernel/hypeman/lib/oapi"
Expand All @@ -16,6 +17,7 @@ type ApiService struct {
InstanceManager instances.Manager
VolumeManager volumes.Manager
NetworkManager network.Manager
IngressManager ingress.Manager
}

var _ oapi.StrictServerInterface = (*ApiService)(nil)
Expand All @@ -27,13 +29,14 @@ func New(
instanceManager instances.Manager,
volumeManager volumes.Manager,
networkManager network.Manager,
ingressManager ingress.Manager,
) *ApiService {
return &ApiService{
Config: config,
ImageManager: imageManager,
InstanceManager: instanceManager,
VolumeManager: volumeManager,
NetworkManager: networkManager,
IngressManager: ingressManager,
}
}

162 changes: 162 additions & 0 deletions cmd/api/api/ingress.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package api

import (
"context"
"errors"

"github.com/onkernel/hypeman/lib/ingress"
"github.com/onkernel/hypeman/lib/logger"
"github.com/onkernel/hypeman/lib/oapi"
)

// ListIngresses lists all ingress resources
func (s *ApiService) ListIngresses(ctx context.Context, request oapi.ListIngressesRequestObject) (oapi.ListIngressesResponseObject, error) {
log := logger.FromContext(ctx)

ingresses, err := s.IngressManager.List(ctx)
if err != nil {
log.ErrorContext(ctx, "failed to list ingresses", "error", err)
return oapi.ListIngresses500JSONResponse{
Code: "internal_error",
Message: "failed to list ingresses",
}, nil
}

oapiIngresses := make([]oapi.Ingress, len(ingresses))
for i, ing := range ingresses {
oapiIngresses[i] = ingressToOAPI(ing)
}

return oapi.ListIngresses200JSONResponse(oapiIngresses), nil
}

// CreateIngress creates a new ingress resource
func (s *ApiService) CreateIngress(ctx context.Context, request oapi.CreateIngressRequestObject) (oapi.CreateIngressResponseObject, error) {
log := logger.FromContext(ctx)

// Convert OAPI request to domain request
domainReq := ingress.CreateIngressRequest{
Name: request.Body.Name,
Rules: make([]ingress.IngressRule, len(request.Body.Rules)),
}

for i, rule := range request.Body.Rules {
matchPort := 80
if rule.Match.Port != nil {
matchPort = *rule.Match.Port
}
domainReq.Rules[i] = ingress.IngressRule{
Match: ingress.IngressMatch{
Hostname: rule.Match.Hostname,
Port: matchPort,
},
Target: ingress.IngressTarget{
Instance: rule.Target.Instance,
Port: rule.Target.Port,
},
}
}

ing, err := s.IngressManager.Create(ctx, domainReq)
if err != nil {
switch {
case errors.Is(err, ingress.ErrInvalidRequest):
return oapi.CreateIngress400JSONResponse{
Code: "bad_request",
Message: err.Error(),
}, nil
case errors.Is(err, ingress.ErrAlreadyExists):
return oapi.CreateIngress409JSONResponse{
Code: "already_exists",
Message: err.Error(),
}, nil
case errors.Is(err, ingress.ErrHostnameInUse):
return oapi.CreateIngress409JSONResponse{
Code: "hostname_in_use",
Message: err.Error(),
}, nil
case errors.Is(err, ingress.ErrInstanceNotFound):
return oapi.CreateIngress400JSONResponse{
Code: "instance_not_found",
Message: err.Error(),
}, nil
default:
log.ErrorContext(ctx, "failed to create ingress", "error", err, "name", request.Body.Name)
return oapi.CreateIngress500JSONResponse{
Code: "internal_error",
Message: "failed to create ingress",
}, nil
}
}

return oapi.CreateIngress201JSONResponse(ingressToOAPI(*ing)), nil
}

// GetIngress gets ingress details by ID or name
func (s *ApiService) GetIngress(ctx context.Context, request oapi.GetIngressRequestObject) (oapi.GetIngressResponseObject, error) {
log := logger.FromContext(ctx)

ing, err := s.IngressManager.Get(ctx, request.Id)
if err != nil {
if errors.Is(err, ingress.ErrNotFound) {
return oapi.GetIngress404JSONResponse{
Code: "not_found",
Message: "ingress not found",
}, nil
}
log.ErrorContext(ctx, "failed to get ingress", "error", err, "id", request.Id)
return oapi.GetIngress500JSONResponse{
Code: "internal_error",
Message: "failed to get ingress",
}, nil
}

return oapi.GetIngress200JSONResponse(ingressToOAPI(*ing)), nil
}

// DeleteIngress deletes an ingress by ID or name
func (s *ApiService) DeleteIngress(ctx context.Context, request oapi.DeleteIngressRequestObject) (oapi.DeleteIngressResponseObject, error) {
log := logger.FromContext(ctx)

err := s.IngressManager.Delete(ctx, request.Id)
if err != nil {
if errors.Is(err, ingress.ErrNotFound) {
return oapi.DeleteIngress404JSONResponse{
Code: "not_found",
Message: "ingress not found",
}, nil
}
log.ErrorContext(ctx, "failed to delete ingress", "error", err, "id", request.Id)
return oapi.DeleteIngress500JSONResponse{
Code: "internal_error",
Message: "failed to delete ingress",
}, nil
}

return oapi.DeleteIngress204Response{}, nil
}

// ingressToOAPI converts a domain Ingress to the OAPI type
func ingressToOAPI(ing ingress.Ingress) oapi.Ingress {
rules := make([]oapi.IngressRule, len(ing.Rules))
for i, rule := range ing.Rules {
port := rule.Match.GetPort()
rules[i] = oapi.IngressRule{
Match: oapi.IngressMatch{
Hostname: rule.Match.Hostname,
Port: &port,
},
Target: oapi.IngressTarget{
Instance: rule.Target.Instance,
Port: rule.Target.Port,
},
}
}

return oapi.Ingress{
Id: ing.ID,
Name: ing.Name,
Rules: rules,
CreatedAt: ing.CreatedAt,
}
}
Loading
Loading