From 3c19cb6d6e871ac21c5b9cb4d76e46c3cefc7cab Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Tue, 10 Feb 2026 18:46:03 -0500 Subject: [PATCH 1/3] feat: pre-cache base images and serve via BuildKit mirror Instead of rewriting Dockerfile FROM instructions at build time, use BuildKit's native registry mirrors directive to serve pre-cached base images from the local registry. The server pulls base images from Docker Hub before launching the builder VM and pushes them to the local registry. BuildKit is configured with our registry as a mirror for docker.io, so pre-cached images are served locally. Key changes: - Add mirrorBaseImagesForBuild for all builds (not just admin) - Add BuildKit mirrors directive in builder agent buildkitd.toml - Preserve library/ prefix in image refs to match BuildKit mirror paths - Return 502 for out-of-scope mirror reads so BuildKit can fall back - Pass Docker Hub credentials through sudo in .air.toml for local dev Co-Authored-By: Claude Opus 4.6 --- .air.toml | 2 +- go.mod | 7 +- go.sum | 33 ++--- lib/builds/builder_agent/main.go | 13 +- lib/builds/dockerfile.go | 166 ++++++++++++++++++++++ lib/builds/dockerfile_test.go | 230 +++++++++++++++++++++++++++++++ lib/builds/manager.go | 55 ++++++++ lib/builds/mirror.go | 86 ++++++++++++ lib/images/mirror.go | 131 ++++++++++++++++++ lib/images/mirror_test.go | 91 ++++++++++++ lib/images/oci_test.go | 190 ------------------------- lib/middleware/oapi_auth.go | 20 ++- lib/registry/token.go | 13 +- 13 files changed, 809 insertions(+), 228 deletions(-) create mode 100644 lib/builds/dockerfile.go create mode 100644 lib/builds/dockerfile_test.go create mode 100644 lib/builds/mirror.go create mode 100644 lib/images/mirror.go create mode 100644 lib/images/mirror_test.go delete mode 100644 lib/images/oci_test.go diff --git a/.air.toml b/.air.toml index 5ee2db21..8db54031 100644 --- a/.air.toml +++ b/.air.toml @@ -12,7 +12,7 @@ tmp_dir = "tmp" exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false - full_bin = "sudo ./tmp/main" + full_bin = "sudo env DOCKER_CONFIG=$HOME/.docker ./tmp/main" include_dir = [] include_ext = ["go", "tpl", "tmpl", "html", "yaml"] include_file = [] diff --git a/go.mod b/go.mod index 16a40d39..d6691b8a 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -67,9 +68,9 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/docker/cli v28.2.2+incompatible // indirect - github.com/docker/docker v28.2.2+incompatible // indirect + github.com/docker/docker v28.5.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -78,7 +79,6 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-test/deep v1.1.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect @@ -90,6 +90,7 @@ require ( github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect diff --git a/go.sum b/go.sum index 6fd5278f..a369c838 100644 --- a/go.sum +++ b/go.sum @@ -21,9 +21,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -52,12 +51,12 @@ github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsy github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -84,8 +83,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -119,8 +116,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= @@ -158,8 +153,8 @@ github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -239,8 +234,6 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -283,26 +276,19 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= @@ -310,7 +296,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -330,8 +315,6 @@ golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index 045b3005..d2d22ceb 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -50,8 +50,8 @@ type BuildConfig struct { Secrets []SecretRef `json:"secrets,omitempty"` TimeoutSeconds int `json:"timeout_seconds"` NetworkMode string `json:"network_mode"` - IsAdminBuild bool `json:"is_admin_build,omitempty"` - GlobalCacheKey string `json:"global_cache_key,omitempty"` + IsAdminBuild bool `json:"is_admin_build,omitempty"` + GlobalCacheKey string `json:"global_cache_key,omitempty"` } // SecretRef references a secret to inject during build @@ -644,6 +644,14 @@ func setupBuildkitdConfig(config *BuildConfig) error { } // If HTTPS without insecure and without CA, use system CA (no config needed) + // Configure docker.io to use the local registry as a mirror. + // BuildKit will try the mirror first for FROM pulls. Since base images + // are pre-cached server-side via mirrorBaseImagesForBuild(), the mirror + // will have them and serve them directly without pulling from Docker Hub. + tomlContent.WriteString("\n") + tomlContent.WriteString("[registry.\"docker.io\"]\n") + tomlContent.WriteString(fmt.Sprintf(" mirrors = [\"%s\"]\n", registryHost)) + // Ensure config directory exists buildkitDir := "/home/builder/.config/buildkit" if err := os.MkdirAll(buildkitDir, 0755); err != nil { @@ -894,3 +902,4 @@ func getBuildkitVersion() string { out, _ := cmd.Output() return strings.TrimSpace(string(out)) } + diff --git a/lib/builds/dockerfile.go b/lib/builds/dockerfile.go new file mode 100644 index 00000000..2df03358 --- /dev/null +++ b/lib/builds/dockerfile.go @@ -0,0 +1,166 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/name" +) + +// ParseDockerfileFROMs extracts and deduplicates base image references from +// Dockerfile content. It reuses the same parsing logic as the builder agent's +// rewriteDockerfileFROMs: split lines, find FROM, skip flags/comments/scratch, +// normalize refs. Inter-stage references (FROM builder) and variable references +// (${VAR}) are skipped since they can't be resolved at parse time. +func ParseDockerfileFROMs(content string) []string { + lines := strings.Split(content, "\n") + + // Track stage names so we can skip inter-stage FROM references + stageNames := make(map[string]bool) + seen := make(map[string]bool) + var refs []string + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip empty lines and comments + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + // Check for FROM instruction (case insensitive) + upper := strings.ToUpper(trimmed) + if !strings.HasPrefix(upper, "FROM ") { + continue + } + + parts := strings.Fields(trimmed) + if len(parts) < 2 { + continue + } + + // Find the image reference (skip FROM and any flags like --platform) + imageIdx := 1 + for imageIdx < len(parts) && strings.HasPrefix(parts[imageIdx], "--") { + imageIdx++ + } + if imageIdx >= len(parts) { + continue + } + + imageRef := parts[imageIdx] + + // Record AS alias if present + for j := imageIdx + 1; j < len(parts)-1; j++ { + if strings.EqualFold(parts[j], "AS") { + stageNames[strings.ToLower(parts[j+1])] = true + break + } + } + + // Skip scratch + if imageRef == "scratch" { + continue + } + + // Skip inter-stage references (e.g. FROM builder) + if stageNames[strings.ToLower(imageRef)] { + continue + } + + // Skip variable references that can't be resolved + if strings.Contains(imageRef, "${") { + continue + } + + // Normalize the image reference (same logic as builder agent) + normalized := normalizeImageRef(imageRef) + + if !seen[normalized] { + seen[normalized] = true + refs = append(refs, normalized) + } + } + + return refs +} + +// normalizeImageRef normalizes a Docker image reference to match the local +// registry path that BuildKit mirror requests will use. Official Docker Hub +// images keep the library/ prefix (e.g. "node:20-alpine" → "library/node:20-alpine") +// because BuildKit requests them as /v2/library/node/manifests/.... +// Non-Docker Hub images keep the full registry path. +// +// This is consistent with normalizeToLocalRef in lib/images/mirror.go, which +// controls where mirrored images are pushed. +func normalizeImageRef(ref string) string { + parsed, err := name.ParseReference(ref) + if err != nil { + // Fall back to basic normalization if parsing fails + return strings.TrimPrefix(ref, "docker.io/") + } + + // Get canonicalized repository (e.g. "index.docker.io/library/node") + repo := parsed.Context().String() + + // Strip index.docker.io/ prefix (canonical form of docker.io) + repo = strings.TrimPrefix(repo, "index.docker.io/") + repo = strings.TrimPrefix(repo, "docker.io/") + + // Keep library/ prefix — BuildKit mirror requests use it for official images + + // Build the tag or digest suffix + var suffix string + if tag, ok := parsed.(name.Tag); ok { + suffix = ":" + tag.TagStr() + } else if dig, ok := parsed.(name.Digest); ok { + suffix = "@" + dig.DigestStr() + } + + return repo + suffix +} + +// ExtractDockerfileFromTarball reads just the Dockerfile entry from a .tar.gz +// archive and returns its content as a string. It looks for entries named +// "Dockerfile" or "./Dockerfile" at the root of the archive. +func ExtractDockerfileFromTarball(tarballPath string) (string, error) { + f, err := os.Open(tarballPath) + if err != nil { + return "", fmt.Errorf("open tarball: %w", err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return "", fmt.Errorf("create gzip reader: %w", err) + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("read tar entry: %w", err) + } + + // Match Dockerfile at root (with or without ./ prefix) + name := filepath.Clean(hdr.Name) + if name == "Dockerfile" { + data, err := io.ReadAll(tr) + if err != nil { + return "", fmt.Errorf("read Dockerfile from tarball: %w", err) + } + return string(data), nil + } + } + + return "", fmt.Errorf("Dockerfile not found in tarball") +} diff --git a/lib/builds/dockerfile_test.go b/lib/builds/dockerfile_test.go new file mode 100644 index 00000000..39674c84 --- /dev/null +++ b/lib/builds/dockerfile_test.go @@ -0,0 +1,230 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "testing" +) + +func TestParseDockerfileFROMs_SingleFROM(t *testing.T) { + content := `FROM onkernel/nodejs22-base:0.1.1 +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "onkernel/nodejs22-base:0.1.1" { + t.Errorf("expected onkernel/nodejs22-base:0.1.1, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_MultiStage(t *testing.T) { + content := `FROM golang:1.21 AS builder +RUN go build -o /app . + +FROM alpine:3.21 +COPY --from=builder /app /app +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 2 { + t.Fatalf("expected 2 refs, got %d: %v", len(refs), refs) + } + if refs[0] != "library/golang:1.21" { + t.Errorf("expected library/golang:1.21, got %s", refs[0]) + } + if refs[1] != "library/alpine:3.21" { + t.Errorf("expected library/alpine:3.21, got %s", refs[1]) + } +} + +func TestParseDockerfileFROMs_DockerIONormalization(t *testing.T) { + content := `FROM docker.io/library/alpine:3.21 +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/alpine:3.21" { + t.Errorf("expected library/alpine:3.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_PlatformFlag(t *testing.T) { + content := `FROM --platform=linux/amd64 node:20-alpine +RUN npm install +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/node:20-alpine" { + t.Errorf("expected library/node:20-alpine, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipScratch(t *testing.T) { + content := `FROM golang:1.21 AS builder +RUN go build -o /app . + +FROM scratch +COPY --from=builder /app /app +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/golang:1.21" { + t.Errorf("expected library/golang:1.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipStageReferences(t *testing.T) { + content := `FROM node:20 AS deps +RUN npm ci + +FROM node:20 AS builder +COPY --from=deps /app/node_modules ./node_modules +RUN npm run build + +FROM builder +CMD ["node", "dist/index.js"] +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref (deduplicated), got %d: %v", len(refs), refs) + } + if refs[0] != "library/node:20" { + t.Errorf("expected library/node:20, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipVariableReferences(t *testing.T) { + content := `ARG BASE_IMAGE=node:20 +FROM ${BASE_IMAGE} +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 0 { + t.Fatalf("expected 0 refs (variable), got %d: %v", len(refs), refs) + } +} + +func TestParseDockerfileFROMs_Deduplication(t *testing.T) { + content := `FROM alpine:3.21 AS stage1 +RUN echo one + +FROM alpine:3.21 AS stage2 +RUN echo two +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref (deduplicated), got %d: %v", len(refs), refs) + } + if refs[0] != "library/alpine:3.21" { + t.Errorf("expected library/alpine:3.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_CommentsAndEmptyLines(t *testing.T) { + content := `# Build stage +FROM golang:1.21 + +# This is a comment +# FROM fake:image + +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/golang:1.21" { + t.Errorf("expected library/golang:1.21, got %s", refs[0]) + } +} + +func TestExtractDockerfileFromTarball(t *testing.T) { + // Create a temp tarball with a Dockerfile + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + dockerfileContent := "FROM alpine:3.21\nRUN echo hello\n" + createTarball(t, tarballPath, map[string]string{ + "Dockerfile": dockerfileContent, + "main.go": "package main\n", + }) + + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != dockerfileContent { + t.Errorf("expected %q, got %q", dockerfileContent, content) + } +} + +func TestExtractDockerfileFromTarball_NotFound(t *testing.T) { + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + createTarball(t, tarballPath, map[string]string{ + "main.go": "package main\n", + }) + + _, err := ExtractDockerfileFromTarball(tarballPath) + if err == nil { + t.Fatal("expected error for missing Dockerfile") + } +} + +func TestExtractDockerfileFromTarball_DotSlashPrefix(t *testing.T) { + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + dockerfileContent := "FROM node:20\nRUN npm install\n" + createTarball(t, tarballPath, map[string]string{ + "./Dockerfile": dockerfileContent, + }) + + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != dockerfileContent { + t.Errorf("expected %q, got %q", dockerfileContent, content) + } +} + +// createTarball creates a .tar.gz file with the given files (name -> content). +func createTarball(t *testing.T, path string, files map[string]string) { + t.Helper() + + f, err := os.Create(path) + if err != nil { + t.Fatalf("create tarball file: %v", err) + } + defer f.Close() + + gw := gzip.NewWriter(f) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write tar header for %s: %v", name, err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatalf("write tar content for %s: %v", name, err) + } + } +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 3a612baa..d919df37 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -248,6 +248,30 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc } } + // Add pull access for base image repos so the builder agent can + // detect mirrored images via checkImageExistsInRegistry + dockerfileContent := req.Dockerfile + if dockerfileContent == "" { + tarballPath := m.paths.BuildSourceDir(id) + "/source.tar.gz" + if content, err := ExtractDockerfileFromTarball(tarballPath); err == nil { + dockerfileContent = content + } + } + if dockerfileContent != "" { + refs := ParseDockerfileFROMs(dockerfileContent) + seen := make(map[string]bool) + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoAccess = append(repoAccess, RepoPermission{Repo: repo, Scope: "pull"}) + } + } + } + registryToken, err := m.tokenGenerator.GenerateToken(id, repoAccess, tokenTTL) if err != nil { deleteBuild(m.paths, id) @@ -315,6 +339,13 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + // Mirror base images to the local registry before launching the VM. + // BuildKit is configured with our registry as a mirror for docker.io, + // so pre-cached images will be served locally without pulling from Docker Hub. + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } + // Run the build in a builder VM result, err := m.executeBuild(buildCtx, id, req, policy) @@ -1129,6 +1160,30 @@ func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) err } } + // Add pull access for base image repos so the builder agent can + // detect mirrored images via checkImageExistsInRegistry + dockerfileContent := req.Dockerfile + if dockerfileContent == "" { + tarballPath := m.paths.BuildSourceDir(buildID) + "/source.tar.gz" + if content, err := ExtractDockerfileFromTarball(tarballPath); err == nil { + dockerfileContent = content + } + } + if dockerfileContent != "" { + refs := ParseDockerfileFROMs(dockerfileContent) + seen := make(map[string]bool) + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoAccess = append(repoAccess, RepoPermission{Repo: repo, Scope: "pull"}) + } + } + } + // Generate fresh registry token registryToken, err := m.tokenGenerator.GenerateToken(buildID, repoAccess, tokenTTL) if err != nil { diff --git a/lib/builds/mirror.go b/lib/builds/mirror.go new file mode 100644 index 00000000..b787634b --- /dev/null +++ b/lib/builds/mirror.go @@ -0,0 +1,86 @@ +package builds + +import ( + "context" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/kernel/hypeman/lib/images" +) + +// mirrorBaseImagesForBuild extracts base image references from the build's +// Dockerfile and mirrors each one to the local registry. BuildKit is configured +// with our registry as a mirror for docker.io, so pre-cached images will be +// served locally without pulling from Docker Hub. +// +// Individual mirror failures are logged but do not fail the build (graceful +// degradation — BuildKit will pull from Docker Hub as before). +func (m *manager) mirrorBaseImagesForBuild(ctx context.Context, id string, req CreateBuildRequest) error { + // Get Dockerfile content: prefer inline Dockerfile, fall back to tarball + var dockerfileContent string + if req.Dockerfile != "" { + dockerfileContent = req.Dockerfile + } else { + tarballPath := m.paths.BuildSourceDir(id) + "/source.tar.gz" + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + m.logger.Warn("could not extract Dockerfile from tarball for mirroring", + "id", id, "error", err) + return nil + } + dockerfileContent = content + } + + // Parse FROM references + refs := ParseDockerfileFROMs(dockerfileContent) + if len(refs) == 0 { + return nil + } + + m.logger.Info("mirroring base images to local registry", "id", id, "images", refs) + + // Generate a scoped registry token that grants push access to the base + // image repos. The local registry requires JWT auth for all operations; + // go-containerregistry uses this via the Docker token auth flow (Basic + // auth username = JWT → /v2/token validates and returns bearer token). + // Build repo permissions. The Docker token scope uses the repo name without + // the tag (e.g. "onkernel/nodejs22-base", not "onkernel/nodejs22-base:0.1.1"). + seen := make(map[string]bool) + var repoPerms []RepoPermission + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoPerms = append(repoPerms, RepoPermission{Repo: repo, Scope: "push"}) + } + } + registryToken, err := m.tokenGenerator.GenerateToken(id, repoPerms, 10*time.Minute) + if err != nil { + m.logger.Warn("failed to generate registry token for mirroring", + "id", id, "error", err) + return nil + } + // go-containerregistry's basicTransport only sends Basic auth when BOTH + // Username and Password are non-empty. The password value doesn't matter — + // our token handler extracts the JWT from the username field only. + authConfig := &authn.AuthConfig{Username: registryToken, Password: "x"} + + for _, ref := range refs { + result, err := images.MirrorBaseImage(ctx, m.config.RegistryURL, images.MirrorRequest{ + SourceImage: ref, + }, authConfig) + if err != nil { + m.logger.Warn("failed to mirror base image", + "id", id, "image", ref, "error", err) + continue + } + m.logger.Info("mirrored base image", + "id", id, "image", ref, "local_ref", result.LocalRef, "digest", result.Digest) + } + + return nil +} diff --git a/lib/images/mirror.go b/lib/images/mirror.go new file mode 100644 index 00000000..a675176e --- /dev/null +++ b/lib/images/mirror.go @@ -0,0 +1,131 @@ +package images + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// MirrorRequest contains the parameters for mirroring a base image +type MirrorRequest struct { + // SourceImage is the full image reference to pull from (e.g., "docker.io/onkernel/nodejs22-base:0.1.1") + SourceImage string +} + +// MirrorResult contains the result of a mirror operation +type MirrorResult struct { + // SourceImage is the original image reference + SourceImage string `json:"source_image"` + // LocalRef is the local registry reference (e.g., "onkernel/nodejs22-base:0.1.1") + LocalRef string `json:"local_ref"` + // Digest is the image digest + Digest string `json:"digest"` +} + +// MirrorBaseImage pulls an image from an external registry and pushes it to the +// local registry with the same normalized name. This enables Dockerfile FROM rewriting +// to use locally mirrored base images instead of pulling from Docker Hub. +// +// For example, mirroring "docker.io/onkernel/nodejs22-base:0.1.1" will create +// "onkernel/nodejs22-base:0.1.1" in the local registry. +func MirrorBaseImage(ctx context.Context, registryURL string, req MirrorRequest, authConfig *authn.AuthConfig) (*MirrorResult, error) { + // Parse source reference + srcRef, err := name.ParseReference(req.SourceImage) + if err != nil { + return nil, fmt.Errorf("parse source image reference: %w", err) + } + + // Pull the image from source + img, err := remote.Image(srcRef, + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + remote.WithPlatform(currentPlatform())) + if err != nil { + return nil, fmt.Errorf("pull source image: %w", wrapRegistryError(err)) + } + + // Get the digest + digest, err := img.Digest() + if err != nil { + return nil, fmt.Errorf("get image digest: %w", err) + } + + // Build the local reference under bases/ namespace + // Normalize the source to strip docker.io/ prefix for cleaner local refs + localRef := normalizeToLocalRef(srcRef) + + // Strip any scheme from registry URL + registryHost := stripScheme(registryURL) + + // Build full destination reference + dstRefStr := fmt.Sprintf("%s/%s", registryHost, localRef) + dstRef, err := name.ParseReference(dstRefStr) + if err != nil { + return nil, fmt.Errorf("parse destination reference: %w", err) + } + + // Push to local registry + // For insecure registries, we need to use the insecure transport + opts := []remote.Option{ + remote.WithContext(ctx), + } + + // If authConfig is provided, use it + if authConfig != nil { + opts = append(opts, remote.WithAuth(authn.FromConfig(*authConfig))) + } + + if err := remote.Write(dstRef, img, opts...); err != nil { + return nil, fmt.Errorf("push to local registry: %w", wrapRegistryError(err)) + } + + return &MirrorResult{ + SourceImage: req.SourceImage, + LocalRef: localRef, + Digest: digest.String(), + }, nil +} + +// normalizeToLocalRef converts a source image reference to a normalized local reference. +// It strips the docker.io/ prefix but preserves the library/ prefix for official images. +// The library/ prefix is kept because BuildKit's mirror protocol requests official images +// as library/ (e.g., /v2/library/node/manifests/...). +// +// Examples: +// - "docker.io/onkernel/nodejs22-base:0.1.1" -> "onkernel/nodejs22-base:0.1.1" +// - "docker.io/library/alpine:3.21" -> "library/alpine:3.21" +// - "node:20-alpine" -> "library/node:20-alpine" (go-containerregistry canonicalizes to library/) +// - "gcr.io/google-containers/pause:3.2" -> "gcr.io/google-containers/pause:3.2" +func normalizeToLocalRef(ref name.Reference) string { + // Get the repository name (includes registry for non-Docker Hub images) + repo := ref.Context().String() + + // Strip index.docker.io/ prefix (canonical form of docker.io) + repo = strings.TrimPrefix(repo, "index.docker.io/") + + // Strip docker.io/ prefix + repo = strings.TrimPrefix(repo, "docker.io/") + + // Keep library/ prefix — BuildKit mirror requests use it for official images + + // Build the tag or digest suffix + var suffix string + if tag, ok := ref.(name.Tag); ok { + suffix = ":" + tag.TagStr() + } else if dig, ok := ref.(name.Digest); ok { + suffix = "@" + dig.DigestStr() + } + + return repo + suffix +} + +// stripScheme removes http:// or https:// prefix from a URL +func stripScheme(url string) string { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + return url +} diff --git a/lib/images/mirror_test.go b/lib/images/mirror_test.go new file mode 100644 index 00000000..1fc26136 --- /dev/null +++ b/lib/images/mirror_test.go @@ -0,0 +1,91 @@ +package images + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeToLocalRef(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "docker hub user image with tag", + input: "docker.io/onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "docker hub user image without registry prefix", + input: "onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "docker hub official image with tag", + input: "docker.io/library/alpine:3.21", + expected: "alpine:3.21", + }, + { + name: "docker hub official image short form", + input: "alpine:3.21", + expected: "alpine:3.21", + }, + { + name: "docker hub image with index.docker.io", + input: "index.docker.io/onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "gcr.io image", + input: "gcr.io/google-containers/pause:3.2", + expected: "gcr.io/google-containers/pause:3.2", + }, + { + name: "ghcr.io image", + input: "ghcr.io/some-org/some-image:v1.0", + expected: "ghcr.io/some-org/some-image:v1.0", + }, + { + name: "image with latest tag", + input: "nginx:latest", + expected: "nginx:latest", + }, + { + name: "image without tag uses latest", + input: "nginx", + expected: "nginx:latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := name.ParseReference(tt.input) + require.NoError(t, err) + result := normalizeToLocalRef(ref) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestStripScheme(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"https://localhost:8080", "localhost:8080"}, + {"http://localhost:8080", "localhost:8080"}, + {"localhost:8080", "localhost:8080"}, + {"https://registry.example.com", "registry.example.com"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := stripScheme(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/lib/images/oci_test.go b/lib/images/oci_test.go deleted file mode 100644 index 592da9ac..00000000 --- a/lib/images/oci_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package images - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// BuildKit cache config mediatype - this is what BuildKit uses when exporting -// cache with image-manifest=true -const buildKitCacheConfigMediaType = "application/vnd.buildkit.cacheconfig.v0" - -// TestUnpackLayersFailsOnBuildKitCacheMediatype verifies that hypeman's image -// unpacker fails when encountering BuildKit cache images. This reproduces the -// production issue where global cache images exported by BuildKit cannot be -// pre-pulled by hypeman because they use a non-standard config mediatype. -// -// The error occurs because: -// 1. BuildKit exports cache with --export-cache type=registry,image-manifest=true -// 2. The exported manifest uses "application/vnd.buildkit.cacheconfig.v0" as config mediatype -// 3. hypeman's unpackLayers expects "application/vnd.oci.image.config.v1+json" -// 4. umoci.UnpackRootfs fails with "config blob is not correct mediatype" -func TestUnpackLayersFailsOnBuildKitCacheMediatype(t *testing.T) { - // Create a temp directory for the OCI layout - cacheDir := t.TempDir() - - // Create OCI layout structure with BuildKit cache mediatype - err := createBuildKitCacheLayout(cacheDir, "test-cache") - require.NoError(t, err, "failed to create mock BuildKit cache layout") - - // Create OCI client and try to unpack - client, err := newOCIClient(cacheDir) - require.NoError(t, err) - - targetDir := t.TempDir() - err = client.unpackLayers(context.Background(), "test-cache", targetDir) - - // This should fail with a mediatype error - require.Error(t, err, "unpackLayers should fail on BuildKit cache mediatype") - assert.Contains(t, err.Error(), "config", "error should mention config") - - t.Logf("Got expected error: %v", err) -} - -// TestExtractMetadataSucceedsOnBuildKitCache verifies that extractOCIMetadata -// does NOT fail on BuildKit cache images - it's go-containerregistry which is -// lenient about mediatypes. The failure only happens during unpackLayers when -// umoci tries to unpack the rootfs. -func TestExtractMetadataSucceedsOnBuildKitCache(t *testing.T) { - cacheDir := t.TempDir() - - err := createBuildKitCacheLayout(cacheDir, "test-cache") - require.NoError(t, err) - - client, err := newOCIClient(cacheDir) - require.NoError(t, err) - - // This succeeds because go-containerregistry doesn't validate config mediatype - // The failure only happens in unpackLayers when umoci validates the config - meta, err := client.extractOCIMetadata("test-cache") - require.NoError(t, err, "extractOCIMetadata succeeds - go-containerregistry is lenient") - - // But the metadata will be empty/invalid since it's not a real OCI config - t.Logf("Got metadata (likely empty): %+v", meta) -} - -// createBuildKitCacheLayout creates an OCI layout that mimics what BuildKit -// exports when using --export-cache type=registry,image-manifest=true -// -// Layout structure: -// cacheDir/ -// ├── oci-layout (OCI layout version marker) -// ├── index.json (points to manifest) -// └── blobs/sha256/ -// ├── (image manifest with buildkit config mediatype) -// ├── (buildkit cache config blob) -// └── (dummy layer) -func createBuildKitCacheLayout(cacheDir, layoutTag string) error { - // Create directory structure - blobsDir := filepath.Join(cacheDir, "blobs", "sha256") - if err := os.MkdirAll(blobsDir, 0755); err != nil { - return err - } - - // 1. Create oci-layout file - ociLayout := map[string]string{"imageLayoutVersion": "1.0.0"} - ociLayoutBytes, _ := json.Marshal(ociLayout) - if err := os.WriteFile(filepath.Join(cacheDir, "oci-layout"), ociLayoutBytes, 0644); err != nil { - return err - } - - // 2. Create a dummy layer blob (gzipped tar with a single file) - // This is a minimal valid gzipped tar - layerContent := []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // empty tar - } - layerDigest := sha256Hash(layerContent) - if err := os.WriteFile(filepath.Join(blobsDir, layerDigest), layerContent, 0644); err != nil { - return err - } - - // 3. Create BuildKit cache config blob - // This is what BuildKit puts in the config - NOT a standard OCI config - cacheConfig := map[string]interface{}{ - "layers": []map[string]interface{}{ - { - "blob": "sha256:" + layerDigest, - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - }, - }, - } - configBytes, _ := json.Marshal(cacheConfig) - configDigest := sha256Hash(configBytes) - if err := os.WriteFile(filepath.Join(blobsDir, configDigest), configBytes, 0644); err != nil { - return err - } - - // 4. Create image manifest with BuildKit's cache config mediatype - manifest := map[string]interface{}{ - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": map[string]interface{}{ - "mediaType": buildKitCacheConfigMediaType, // This is the problem! - "digest": "sha256:" + configDigest, - "size": len(configBytes), - }, - "layers": []map[string]interface{}{ - { - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "digest": "sha256:" + layerDigest, - "size": len(layerContent), - }, - }, - } - manifestBytes, _ := json.Marshal(manifest) - manifestDigest := sha256Hash(manifestBytes) - if err := os.WriteFile(filepath.Join(blobsDir, manifestDigest), manifestBytes, 0644); err != nil { - return err - } - - // 5. Create index.json pointing to the manifest with our layout tag - index := map[string]interface{}{ - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": []map[string]interface{}{ - { - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": "sha256:" + manifestDigest, - "size": len(manifestBytes), - "annotations": map[string]string{ - "org.opencontainers.image.ref.name": layoutTag, - }, - }, - }, - } - indexBytes, _ := json.Marshal(index) - if err := os.WriteFile(filepath.Join(cacheDir, "index.json"), indexBytes, 0644); err != nil { - return err - } - - return nil -} - -// sha256Hash computes the SHA256 hash of data and returns the hex string -func sha256Hash(data []byte) string { - h := sha256.Sum256(data) - return hex.EncodeToString(h[:]) -} - -// TestConvertToOCIMediaTypePassesThroughBuildKitType verifies that the -// mediatype conversion function doesn't handle BuildKit's cache config type, -// which is the root cause of the unpack failure. -func TestConvertToOCIMediaTypePassesThroughBuildKitType(t *testing.T) { - // Verify that BuildKit's mediatype passes through unchanged - result := convertToOCIMediaType(buildKitCacheConfigMediaType) - assert.Equal(t, buildKitCacheConfigMediaType, result, - "BuildKit cache config mediatype should pass through unchanged (this is the bug)") - - // Standard Docker types should be converted - assert.Equal(t, "application/vnd.oci.image.config.v1+json", - convertToOCIMediaType("application/vnd.docker.container.image.v1+json")) -} diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 6f8a8254..2f877fe6 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -3,6 +3,7 @@ package middleware import ( "context" "encoding/base64" + "errors" "fmt" "net/http" "strings" @@ -14,6 +15,9 @@ import ( "github.com/kernel/hypeman/lib/logger" ) +// errRepoNotAllowed is returned when a valid token doesn't have access to the requested repository. +var errRepoNotAllowed = errors.New("repository not allowed by token") + type contextKey string const userIDKey contextKey = "user_id" @@ -322,7 +326,7 @@ func validateRegistryToken(tokenString, jwtSecret, requestPath, method string) ( } if !allowed { - return nil, fmt.Errorf("repository %s not allowed by token", repo) + return nil, fmt.Errorf("%w: %s", errRepoNotAllowed, repo) } // Check scope for write operations @@ -377,6 +381,20 @@ func JwtAuth(jwtSecret string) func(http.Handler) http.Handler { return } log.DebugContext(r.Context(), "registry token validation failed", "error", err) + + // For read operations (GET/HEAD), if the token is valid but the + // repo isn't in the allowed list, return 502 Bad Gateway. + // BuildKit treats 5xx from a mirror as "mirror unavailable" and + // falls back to the upstream registry (Docker Hub). A 404 would + // be treated as "image doesn't exist" with no fallback. + if errors.Is(err, errRepoNotAllowed) && !isWriteOperation(r.Method) { + log.DebugContext(r.Context(), "returning 502 for mirror fallback", + "path", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadGateway) + fmt.Fprintf(w, `{"errors":[{"code":"UNAVAILABLE","message":"image not mirrored"}]}`) + return + } } else { log.DebugContext(r.Context(), "failed to extract token", "error", err) } diff --git a/lib/registry/token.go b/lib/registry/token.go index 029aedf1..2c0a6185 100644 --- a/lib/registry/token.go +++ b/lib/registry/token.go @@ -80,16 +80,17 @@ func (h *TokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Check if requested scope is allowed by the token + // Check if requested scope is allowed by the token. + // If not, still return a valid token — the subsequent manifest request + // will get a 404 (not found) instead of 403. This is critical for BuildKit + // mirror fallback: a 403 on the token endpoint is treated as a hard auth + // failure and prevents fallback to upstream registries like Docker Hub. if scope != "" { repo, actions := parseScope(scope) if repo != "" && !h.isScopeAllowed(claims, repo, actions) { - log.DebugContext(r.Context(), "scope not allowed by token", + log.DebugContext(r.Context(), "scope not in token, returning token anyway for mirror fallback", "requested_repo", repo, - "requested_actions", actions, - "allowed_repos", claims["repos"]) - h.writeError(w, http.StatusForbidden, "DENIED", "requested scope not allowed") - return + "requested_actions", actions) } } From 4a010a59d86dcd505974c6bbeb33ff30c635d9d9 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Tue, 10 Feb 2026 18:56:27 -0500 Subject: [PATCH 2/3] fix: update tests for mirror fallback behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update normalizeToLocalRef tests to expect library/ prefix for official Docker Hub images (nginx → library/nginx) - Update token endpoint tests: out-of-scope requests now return 200 instead of 403, since access control moved to the middleware layer to support BuildKit mirror fallback Co-Authored-By: Claude Opus 4.6 --- lib/images/mirror_test.go | 8 ++++---- lib/registry/auth_integration_test.go | 13 +++++++------ lib/registry/token_test.go | 16 ++++++++++------ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/images/mirror_test.go b/lib/images/mirror_test.go index 1fc26136..30fe0d3c 100644 --- a/lib/images/mirror_test.go +++ b/lib/images/mirror_test.go @@ -27,12 +27,12 @@ func TestNormalizeToLocalRef(t *testing.T) { { name: "docker hub official image with tag", input: "docker.io/library/alpine:3.21", - expected: "alpine:3.21", + expected: "library/alpine:3.21", }, { name: "docker hub official image short form", input: "alpine:3.21", - expected: "alpine:3.21", + expected: "library/alpine:3.21", }, { name: "docker hub image with index.docker.io", @@ -52,12 +52,12 @@ func TestNormalizeToLocalRef(t *testing.T) { { name: "image with latest tag", input: "nginx:latest", - expected: "nginx:latest", + expected: "library/nginx:latest", }, { name: "image without tag uses latest", input: "nginx", - expected: "nginx:latest", + expected: "library/nginx:latest", }, } diff --git a/lib/registry/auth_integration_test.go b/lib/registry/auth_integration_test.go index 62d6eabb..4fb2827e 100644 --- a/lib/registry/auth_integration_test.go +++ b/lib/registry/auth_integration_test.go @@ -105,9 +105,10 @@ func TestBuildKitAuthFlow(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) }) - t.Run("authenticated request for unauthorized repo returns 403", func(t *testing.T) { - // Token only allows access to builds/build-123 and cache/org-test - // Request for a different repo should fail with 403 + t.Run("authenticated request for unauthorized repo returns token for mirror fallback", func(t *testing.T) { + // Token only allows access to builds/build-123 and cache/org-test. + // Token endpoint returns 200 anyway — access control is enforced + // at the middleware layer. This enables BuildKit mirror fallback. req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/token?scope=repository:builds/other-build:push&service=hypeman", nil) require.NoError(t, err) @@ -118,7 +119,7 @@ func TestBuildKitAuthFlow(t *testing.T) { require.NoError(t, err) defer resp.Body.Close() - assert.Equal(t, http.StatusForbidden, resp.StatusCode) + assert.Equal(t, http.StatusOK, resp.StatusCode) }) } @@ -156,9 +157,9 @@ func TestDockerConfigCredentialLookup(t *testing.T) { expectedStatus: http.StatusOK, }, { - name: "unauthorized repo", + name: "unauthorized repo returns token for mirror fallback", scope: "repository:builds/other:push", - expectedStatus: http.StatusForbidden, + expectedStatus: http.StatusOK, }, } diff --git a/lib/registry/token_test.go b/lib/registry/token_test.go index 311f410b..7d1e6011 100644 --- a/lib/registry/token_test.go +++ b/lib/registry/token_test.go @@ -91,8 +91,10 @@ func TestTokenHandler_BearerAuth(t *testing.T) { func TestTokenHandler_ScopeValidation(t *testing.T) { handler := NewTokenHandler(testJWTSecret) - t.Run("scope not in token is rejected", func(t *testing.T) { - // Token allows builds/build-123, but request is for builds/other + t.Run("scope not in token still returns token for mirror fallback", func(t *testing.T) { + // Token allows builds/build-123, but request is for builds/other. + // Token endpoint returns 200 anyway — access control is enforced + // at the middleware layer. This enables BuildKit mirror fallback. registryToken := generateRegistryToken(t, "build-123", []string{"builds/build-123"}, "push", time.Hour) basicAuth := base64.StdEncoding.EncodeToString([]byte(registryToken + ":")) @@ -102,11 +104,13 @@ func TestTokenHandler_ScopeValidation(t *testing.T) { rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusForbidden, rr.Code) + assert.Equal(t, http.StatusOK, rr.Code) }) - t.Run("push action with pull-only token is rejected", func(t *testing.T) { - // Token only has pull scope + t.Run("push action with pull-only token still returns token for mirror fallback", func(t *testing.T) { + // Token only has pull scope but requests push. + // Token endpoint returns 200 anyway — access control is enforced + // at the middleware layer. registryToken := generateRegistryToken(t, "build-123", []string{"builds/build-123"}, "pull", time.Hour) basicAuth := base64.StdEncoding.EncodeToString([]byte(registryToken + ":")) @@ -116,7 +120,7 @@ func TestTokenHandler_ScopeValidation(t *testing.T) { rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusForbidden, rr.Code) + assert.Equal(t, http.StatusOK, rr.Code) }) t.Run("pull action with push token is allowed", func(t *testing.T) { From ba5ca188a1b407e2569402a8390fa154286b6e7a Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Wed, 11 Feb 2026 15:06:13 -0500 Subject: [PATCH 3/3] fix: strip digest before tag when extracting repo name for mirror token Digest-based image refs (e.g., library/alpine@sha256:abc123) would match the colon inside the digest during tag stripping, producing an incorrect repo name for the token scope. Strip @digest first, matching the pattern in extractInternalBaseImageRepos. Co-Authored-By: Claude Opus 4.6 --- lib/builds/mirror.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/builds/mirror.go b/lib/builds/mirror.go index b787634b..57797537 100644 --- a/lib/builds/mirror.go +++ b/lib/builds/mirror.go @@ -50,6 +50,9 @@ func (m *manager) mirrorBaseImagesForBuild(ctx context.Context, id string, req C var repoPerms []RepoPermission for _, ref := range refs { repo := ref + if idx := strings.LastIndex(repo, "@"); idx != -1 { + repo = repo[:idx] + } if idx := strings.LastIndex(repo, ":"); idx > 0 { repo = repo[:idx] }