add support for non-mainnet presets (#226)

* add support for non-mainnet presets

* fix for block root calculation

* fix linter issues

* fix linter issues

* fix linter issues

* fix linter issues

* add setting to enable custom preset support

* bump `github.com/ethpandaops/beacon`

* add `custom_preset` to README

* increase itegration test timeout

* do not pass `WithCustomSpecSupport` to go-eth2-client with custon_preset disabled

Author: pk910

.github/workflows/integration.yaml

@@ -26,7 +26,7 @@ jobs:
           - {consensus: nimbus, network: hoodi, image: 'statusim/nimbus-eth2:amd64-v25.11.0'}
           - {consensus: lodestar, network: hoodi, image: 'chainsafe/lodestar:v1.36.0'}
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Print details

README.md

@@ -149,6 +149,7 @@ global:
 
 checkpointz:
   mode: light
+  custom_preset: false # Enable this for non-mainnet presets
   caches:
     blocks:
       # Controls the amount of "block" items that can be stored by Checkpointz (minimum 3)

go.mod

@@ -6,11 +6,12 @@ require (
 	github.com/attestantio/go-eth2-client v0.27.2
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
 	github.com/creasty/defaults v1.6.0
-	github.com/ethpandaops/beacon v0.65.0
+	github.com/ethpandaops/beacon v0.66.0
 	github.com/ethpandaops/ethwallclock v0.2.0
 	github.com/go-co-op/gocron v1.18.0
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/nanmu42/gzip v1.2.0
+	github.com/pk910/dynamic-ssz v1.1.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.23.2
 	github.com/sirupsen/logrus v1.9.3
@@ -20,7 +21,9 @@ require (
 )
 
 require (
+	github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/casbin/govaluate v1.8.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
@@ -52,7 +55,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pk910/dynamic-ssz v0.0.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect

go.sum

@@ -1,7 +1,11 @@
+github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7 h1:0r1HjExe/tyypkt380UTpjvILd5kLw51Xzl6a+hknQ8=
+github.com/OffchainLabs/hashtree v0.2.1-0.20250530191054-577f0b75c7f7/go.mod h1:b07+cRZs+eAR8TR57CB9TQlt5Gnl/06Xs76xt/1wq0M=
 github.com/attestantio/go-eth2-client v0.27.2 h1:VjA9R39ovy8ryb7IpFfD5eLYBg/20biztxh6fKZ7/K0=
 github.com/attestantio/go-eth2-client v0.27.2/go.mod h1:i56XBegxVt7wXupnLBOj9IyGwy5cqaoTsCSKlwTubEU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/casbin/govaluate v1.8.0 h1:1dUaV/I0LFP2tcY1uNQEb6wBCbp8GMTcC/zhwQDWvZo=
+github.com/casbin/govaluate v1.8.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9 h1:xz6Nv3zcwO2Lila35hcb0QloCQsc38Al13RNEzWRpX4=
@@ -24,6 +28,10 @@ github.com/ethereum/go-ethereum v1.16.4 h1:H6dU0r2p/amA7cYg6zyG9Nt2JrKKH6oX2utfc
 github.com/ethereum/go-ethereum v1.16.4/go.mod h1:P7551slMFbjn2zOQaKrJShZVN/d8bGxp4/I6yZVlb5w=
 github.com/ethpandaops/beacon v0.65.0 h1:ssnab73uiuzhmhtU56q9D3ecVrEYv5n+tuqrXRK4YAg=
 github.com/ethpandaops/beacon v0.65.0/go.mod h1:lgzrJjQVV77wZ+PJymsY3bQbAK4jrtP8n3WOwMf1Pcs=
+github.com/ethpandaops/beacon v0.65.1-0.20251204024030-639105ffe6c2 h1:WU40omAqHrPF/Mfovc9+eMCrTAl8Km/KOanyYqr3SBg=
+github.com/ethpandaops/beacon v0.65.1-0.20251204024030-639105ffe6c2/go.mod h1:lgzrJjQVV77wZ+PJymsY3bQbAK4jrtP8n3WOwMf1Pcs=
+github.com/ethpandaops/beacon v0.66.0 h1:BRnf4yTEzkZwHW6sTp1x+mBoO5pwbQOX6wtLt3Nh1Y4=
+github.com/ethpandaops/beacon v0.66.0/go.mod h1:lgzrJjQVV77wZ+PJymsY3bQbAK4jrtP8n3WOwMf1Pcs=
 github.com/ethpandaops/ethwallclock v0.2.0 h1:EeFKtZ7v6TAdn/oAh0xaPujD7N4amjBxrWIByraUfLM=
 github.com/ethpandaops/ethwallclock v0.2.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
@@ -154,6 +162,8 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/pk910/dynamic-ssz v0.0.4 h1:DT29+1055tCEPCaR4V/ez+MOKW7BzBsmjyFvBRqx0ME=
 github.com/pk910/dynamic-ssz v0.0.4/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c=
+github.com/pk910/dynamic-ssz v1.1.1 h1:b8sPR8fyhBvz8SHa2RH20SNtt5VDzAEY6fKsPCUcYX4=
+github.com/pk910/dynamic-ssz v1.1.1/go.mod h1:3zyemisUysY2PWACZ8LeZS2tAw8AkuTb2GaLmqYsg1I=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

pkg/api/handler.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/attestantio/go-eth2-client/spec"
 	"github.com/ethpandaops/checkpointz/pkg/beacon"
+	"github.com/ethpandaops/checkpointz/pkg/beacon/ssz"
 	"github.com/ethpandaops/checkpointz/pkg/service/checkpointz"
 	"github.com/ethpandaops/checkpointz/pkg/service/eth"
 	"github.com/julienschmidt/httprouter"
@@ -26,6 +26,7 @@ type Handler struct {
 
 	eth           *eth.Handler
 	checkpointz   *checkpointz.Handler
+	sszEncoder    *ssz.Encoder
 	publicURL     string
 	brandName     string
 	brandImageURL string
@@ -39,6 +40,7 @@ func NewHandler(log logrus.FieldLogger, beac beacon.FinalityProvider, config *be
 
 		eth:           eth.NewHandler(log, beac, "checkpointz"),
 		checkpointz:   checkpointz.NewHandler(log, beac),
+		sszEncoder:    beac.SSZEncoder(),
 		publicURL:     config.Frontend.PublicURL,
 		brandName:     config.Frontend.BrandName,
 		brandImageURL: config.Frontend.BrandImageURL,
@@ -171,47 +173,14 @@ func (h *Handler) handleEthV2BeaconBlocks(ctx context.Context, r *http.Request,
 		return NewInternalServerErrorResponse(nil), err
 	}
 
-	var rsp = &HTTPResponse{}
-
-	switch strings.ToLower(block.Version.String()) {
-	case spec.DataVersionPhase0.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Phase0.MarshalJSON,
-			ContentTypeSSZ:  block.Phase0.MarshalSSZ,
-		})
-	case spec.DataVersionAltair.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Altair.MarshalJSON,
-			ContentTypeSSZ:  block.Altair.MarshalSSZ,
-		})
-	case spec.DataVersionBellatrix.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Bellatrix.MarshalJSON,
-			ContentTypeSSZ:  block.Bellatrix.MarshalSSZ,
-		})
-	case spec.DataVersionCapella.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Capella.MarshalJSON,
-			ContentTypeSSZ:  block.Capella.MarshalSSZ,
-		})
-	case spec.DataVersionDeneb.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Deneb.MarshalJSON,
-			ContentTypeSSZ:  block.Deneb.MarshalSSZ,
-		})
-	case spec.DataVersionElectra.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Electra.MarshalJSON,
-			ContentTypeSSZ:  block.Electra.MarshalSSZ,
-		})
-	case spec.DataVersionFulu.String():
-		rsp = NewSuccessResponse(ContentTypeResolvers{
-			ContentTypeJSON: block.Fulu.MarshalJSON,
-			ContentTypeSSZ:  block.Fulu.MarshalSSZ,
-		})
-	default:
-		return NewInternalServerErrorResponse(nil), errors.New("unknown block version")
-	}
+	rsp := NewSuccessResponse(ContentTypeResolvers{
+		ContentTypeJSON: func() ([]byte, error) {
+			return h.sszEncoder.EncodeBlockJSON(block)
+		},
+		ContentTypeSSZ: func() ([]byte, error) {
+			return h.sszEncoder.EncodeBlockSSZ(block)
+		},
+	})
 
 	rsp.AddExtraData("version", block.Version.String())
 	rsp.AddExtraData("execution_optimistic", false)
@@ -251,24 +220,7 @@ func (h *Handler) handleEthV2DebugBeaconStates(ctx context.Context, r *http.Requ
 
 	rsp := NewSuccessResponse(ContentTypeResolvers{
 		ContentTypeSSZ: func() ([]byte, error) {
-			switch strings.ToLower(state.Version.String()) {
-			case spec.DataVersionPhase0.String():
-				return state.Phase0.MarshalSSZ()
-			case spec.DataVersionAltair.String():
-				return state.Altair.MarshalSSZ()
-			case spec.DataVersionBellatrix.String():
-				return state.Bellatrix.MarshalSSZ()
-			case spec.DataVersionCapella.String():
-				return state.Capella.MarshalSSZ()
-			case spec.DataVersionDeneb.String():
-				return state.Deneb.MarshalSSZ()
-			case spec.DataVersionElectra.String():
-				return state.Electra.MarshalSSZ()
-			case "fulu":
-				return state.Fulu.MarshalSSZ()
-			default:
-				return nil, fmt.Errorf("unknown state version: %s", state.Version.String())
-			}
+			return h.sszEncoder.EncodeStateSSZ(state)
 		},
 	})
 

pkg/beacon/config.go

@@ -11,6 +11,8 @@ import (
 type Config struct {
 	// Mode sets the operational mode of the provider.
 	Mode OperatingMode `yaml:"mode" default:"light"`
+	// CustomPreset enables the use of a custom preset for the provider.
+	CustomPreset bool `yaml:"custom_preset" default:"false"`
 	// Cache holds configuration for the caches.
 	Caches CacheConfig `yaml:"caches"`
 

pkg/beacon/default.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ethpandaops/beacon/pkg/beacon/state"
 	"github.com/ethpandaops/checkpointz/pkg/beacon/checkpoints"
 	"github.com/ethpandaops/checkpointz/pkg/beacon/node"
+	"github.com/ethpandaops/checkpointz/pkg/beacon/ssz"
 	"github.com/ethpandaops/checkpointz/pkg/beacon/store"
 	"github.com/ethpandaops/checkpointz/pkg/eth"
 	"github.com/ethpandaops/ethwallclock"
@@ -32,6 +33,7 @@ type Default struct {
 	nodeConfigs []node.Config
 	nodes       Nodes
 	broker      *emission.Emitter
+	sszEncoder  *ssz.Encoder
 
 	head          *v1.Finality
 	servingBundle *v1.Finality
@@ -70,7 +72,7 @@ func NewDefaultProvider(namespace string, log logrus.FieldLogger, nodes []node.C
 	return &Default{
 		nodeConfigs: nodes,
 		log:         log.WithField("module", "beacon/default"),
-		nodes:       NewNodesFromConfig(log, nodes, namespace),
+		nodes:       NewNodesFromConfig(log, nodes, namespace, config.CustomPreset),
 		config:      config,
 
 		head:          &v1.Finality{},
@@ -79,6 +81,7 @@ func NewDefaultProvider(namespace string, log logrus.FieldLogger, nodes []node.C
 		historicalSlotFailures: make(map[phase0.Slot]int),
 
 		broker:           emission.NewEmitter(),
+		sszEncoder:       ssz.NewEncoder(config.CustomPreset),
 		blocks:           store.NewBlock(log, config.Caches.Blocks, namespace),
 		states:           store.NewBeaconState(log, config.Caches.States, namespace),
 		depositSnapshots: store.NewDepositSnapshot(log, config.Caches.DepositSnapshots, namespace),
@@ -439,6 +442,7 @@ func (d *Default) setSpec(s *state.Spec) {
 	defer d.specMutex.Unlock()
 
 	d.spec = s
+	d.sszEncoder.SetSpec(s)
 }
 
 func (d *Default) Spec() (*state.Spec, error) {
@@ -454,6 +458,10 @@ func (d *Default) Spec() (*state.Spec, error) {
 	return &copied, nil
 }
 
+func (d *Default) SSZEncoder() *ssz.Encoder {
+	return d.sszEncoder
+}
+
 func (d *Default) OperatingMode() OperatingMode {
 	return d.config.Mode
 }
@@ -659,7 +667,7 @@ func (d *Default) storeBlock(_ context.Context, block *spec.VersionedSignedBeaco
 		return errors.New("block is nil")
 	}
 
-	root, err := block.Root()
+	root, err := d.sszEncoder.GetBlockRoot(block)
 	if err != nil {
 		return err
 	}
@@ -680,7 +688,7 @@ func (d *Default) storeBlock(_ context.Context, block *spec.VersionedSignedBeaco
 		expiresAt = time.Now().Add(999999 * time.Hour)
 	}
 
-	if err := d.blocks.Add(block, expiresAt); err != nil {
+	if err := d.blocks.Add(root, block, expiresAt); err != nil {
 		return err
 	}
 
@@ -742,7 +750,6 @@ func (d *Default) ListFinalizedSlots(ctx context.Context) ([]phase0.Slot, error)
 
 	latestSlot := phase0.Slot(uint64(finality.Finalized.Epoch) * uint64(sp.SlotsPerEpoch))
 
-	//nolint:gosec // This is not a security issue
 	for i, val := uint64(latestSlot), uint64(latestSlot)-uint64(sp.SlotsPerEpoch)*uint64(d.config.HistoricalEpochCount); i > val; i -= uint64(sp.SlotsPerEpoch) {
 		slots = append(slots, phase0.Slot(i))
 	}

pkg/beacon/download.go

@@ -115,7 +115,7 @@ func (d *Default) checkGenesis(ctx context.Context) error {
 		return errors.New("invalid genesis block")
 	}
 
-	genesisBlockRoot, err := genesisBlock.Root()
+	genesisBlockRoot, err := d.sszEncoder.GetBlockRoot(genesisBlock)
 	if err != nil {
 		return err
 	}
@@ -173,12 +173,12 @@ func (d *Default) fetchHistoricalCheckpoints(ctx context.Context, checkpoint *v1
 	// We'll derive the current finalized slot and then work back in intervals of SLOTS_PER_EPOCH.
 	currentSlot := uint64(checkpoint.Finalized.Epoch) * uint64(sp.SlotsPerEpoch)
 	for i := 1; i < d.config.HistoricalEpochCount; i++ {
-		//nolint:gosec // This is not a security issue
 		if uint64(i)*uint64(sp.SlotsPerEpoch) > currentSlot {
 			break
 		}
-		//nolint:gosec // This is not a security issue
+
 		slot := phase0.Slot(currentSlot - uint64(i)*uint64(sp.SlotsPerEpoch))
+
 		slotsInScope[slot] = struct{}{}
 	}
 
@@ -260,7 +260,7 @@ func (d *Default) downloadBlock(ctx context.Context, slot phase0.Slot, upstream
 		return nil, err
 	}
 
-	root, err := block.Root()
+	root, err := d.sszEncoder.GetBlockRoot(block)
 	if err != nil {
 		return nil, err
 	}
@@ -302,13 +302,13 @@ func (d *Default) fetchBundle(ctx context.Context, root phase0.Root, upstream *N
 		return nil, fmt.Errorf("failed to get state root from block: %w", err)
 	}
 
-	blockRoot, err := block.Root()
+	blockRoot, err := d.sszEncoder.GetBlockRoot(block)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get block root from block: %w", err)
 	}
 
 	if blockRoot != root {
-		return nil, errors.New("block root does not match")
+		return nil, fmt.Errorf("block root does not match: %#x != %#x", blockRoot, root)
 	}
 
 	slot, err := block.Slot()

pkg/beacon/expire_test.go

@@ -13,12 +13,10 @@ var (
 )
 
 func CalculateSlotExpiration(slot phase0.Slot, slotsOfHistory int) phase0.Slot {
-	//nolint:gosec // This is not a security issue
 	return slot + phase0.Slot(slotsOfHistory)
 }
 
 func GetSlotTime(slot phase0.Slot, secondsPerSlot time.Duration, genesis time.Time) time.Time {
-	//nolint:gosec // This is not a security issue
 	return genesis.Add(time.Duration(slot) * secondsPerSlot)
 }
 

pkg/beacon/finality_provider.go

@@ -9,6 +9,7 @@ import (
 	"github.com/attestantio/go-eth2-client/spec/phase0"
 	"github.com/ethpandaops/beacon/pkg/beacon/api/types"
 	"github.com/ethpandaops/beacon/pkg/beacon/state"
+	"github.com/ethpandaops/checkpointz/pkg/beacon/ssz"
 	"github.com/ethpandaops/checkpointz/pkg/eth"
 )
 
@@ -34,6 +35,8 @@ type FinalityProvider interface {
 	Genesis(ctx context.Context) (*v1.Genesis, error)
 	// Spec returns the chain spec.
 	Spec() (*state.Spec, error)
+	// SSZEncoder returns the SSZ encoder for the provider.
+	SSZEncoder() *ssz.Encoder
 	// UpstreamsStatus returns the status of all the upstreams.
 	UpstreamsStatus(ctx context.Context) (map[string]*UpstreamStatus, error)
 	// GetBlockBySlot returns the block at the given slot.