From 4da68d922092bf16453a15b8409e82f8bf0729ea Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 15:53:58 -0700 Subject: [PATCH 1/4] fix(envd): make /init lock ctx-aware to prevent retry pile-up If any /init is stuck inside SetData, the orchestrator's infinite retry loop pushes more callers behind sync.Mutex.Lock(), each holding a copy of the request body and a goroutine. Switch initLock to a buffered channel (capacity 1) and acquire via select on ctx.Done(); on cancellation return 503 so the orchestrator just retries with a stable Timestamp (PostInit is idempotent per #2687) instead of accumulating goroutines server-side. --- packages/envd/internal/api/init.go | 11 +++++++++-- packages/envd/internal/api/store.go | 5 ++++- packages/envd/pkg/version.go | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 4d0081b0a2..512a5154de 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -128,8 +128,15 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { // Safe because Destroy() is nil-safe and TakeFrom clears the source. defer initRequest.AccessToken.Destroy() - a.initLock.Lock() - defer a.initLock.Unlock() + select { + case a.initLock <- struct{}{}: + case <-ctx.Done(): + logger.Warn().Err(ctx.Err()).Msg("Gave up waiting for initLock") + w.WriteHeader(http.StatusServiceUnavailable) + + return + } + defer func() { <-a.initLock }() // Update data only if the request is newer or if there's no timestamp at all if initRequest.Timestamp == nil || a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) { diff --git a/packages/envd/internal/api/store.go b/packages/envd/internal/api/store.go index 7564d1e62b..b778bf366d 100644 --- a/packages/envd/internal/api/store.go +++ b/packages/envd/internal/api/store.go @@ -37,7 +37,9 @@ type API struct { mmdsClient MMDSClient lastSetTime *utils.AtomicMax - initLock sync.Mutex + // initLock serializes /init handlers but, unlike sync.Mutex, respects the + // caller's ctx so a stuck predecessor cannot pile up retry goroutines. + initLock chan struct{} caCertInstaller *host.CACertInstaller isMountingNFS atomic.Bool @@ -54,6 +56,7 @@ func New(l *zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host. lastSetTime: utils.NewAtomicMax(), accessToken: &SecureToken{}, caCertInstaller: host.NewCACertInstaller(l), + initLock: make(chan struct{}, 1), } } diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index 88bc32c769..4cfbb08b29 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.5.23" +const Version = "0.5.24" From c91a0d584b6b8b9899dd812b88b133d9d91f96e9 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 22:07:40 -0700 Subject: [PATCH 2/4] drop warn log on initLock ctx cancel Orchestrator already logs the failure; doubling up here is noise. --- packages/envd/internal/api/init.go | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 512a5154de..f5bfd2e0b3 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -131,7 +131,6 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { select { case a.initLock <- struct{}{}: case <-ctx.Done(): - logger.Warn().Err(ctx.Err()).Msg("Gave up waiting for initLock") w.WriteHeader(http.StatusServiceUnavailable) return From a317ce76a903300519506936bcba6a234ac8f829 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 22:45:32 -0700 Subject: [PATCH 3/4] use semaphore.Weighted for initLock --- packages/envd/internal/api/init.go | 6 ++---- packages/envd/internal/api/store.go | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index f5bfd2e0b3..f49f317647 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -128,14 +128,12 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { // Safe because Destroy() is nil-safe and TakeFrom clears the source. defer initRequest.AccessToken.Destroy() - select { - case a.initLock <- struct{}{}: - case <-ctx.Done(): + if err := a.initLock.Acquire(ctx, 1); err != nil { w.WriteHeader(http.StatusServiceUnavailable) return } - defer func() { <-a.initLock }() + defer a.initLock.Release(1) // Update data only if the request is newer or if there's no timestamp at all if initRequest.Timestamp == nil || a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) { diff --git a/packages/envd/internal/api/store.go b/packages/envd/internal/api/store.go index b778bf366d..7971a8ca52 100644 --- a/packages/envd/internal/api/store.go +++ b/packages/envd/internal/api/store.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "github.com/rs/zerolog" + "golang.org/x/sync/semaphore" "github.com/e2b-dev/infra/packages/envd/internal/execcontext" "github.com/e2b-dev/infra/packages/envd/internal/host" @@ -39,7 +40,7 @@ type API struct { lastSetTime *utils.AtomicMax // initLock serializes /init handlers but, unlike sync.Mutex, respects the // caller's ctx so a stuck predecessor cannot pile up retry goroutines. - initLock chan struct{} + initLock *semaphore.Weighted caCertInstaller *host.CACertInstaller isMountingNFS atomic.Bool @@ -56,7 +57,7 @@ func New(l *zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host. lastSetTime: utils.NewAtomicMax(), accessToken: &SecureToken{}, caCertInstaller: host.NewCACertInstaller(l), - initLock: make(chan struct{}, 1), + initLock: semaphore.NewWeighted(1), } } From b3c2214e75a881b9a8e6557175c6c80ce1a8da82 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 22:46:56 -0700 Subject: [PATCH 4/4] drop redundant initLock comment --- packages/envd/internal/api/store.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/envd/internal/api/store.go b/packages/envd/internal/api/store.go index 7971a8ca52..e5c536fead 100644 --- a/packages/envd/internal/api/store.go +++ b/packages/envd/internal/api/store.go @@ -38,9 +38,7 @@ type API struct { mmdsClient MMDSClient lastSetTime *utils.AtomicMax - // initLock serializes /init handlers but, unlike sync.Mutex, respects the - // caller's ctx so a stuck predecessor cannot pile up retry goroutines. - initLock *semaphore.Weighted + initLock *semaphore.Weighted caCertInstaller *host.CACertInstaller isMountingNFS atomic.Bool