From fa5e5c0d64710f0ca838f5e7f8e73ac96bdb03e0 Mon Sep 17 00:00:00 2001 From: lishubo1 Date: Fri, 3 Jul 2026 17:08:02 +0800 Subject: [PATCH] feat(embedding): make vector dimension configurable - Add dim field to EmbeddingSettings - Use settings.dim in build_embedding_provider - Read dimension from settings in LanceDB table schemas - Add default dim = 1024 in default.toml Co-Authored-By: Claude Opus 4.8 (1M context) --- src/everos/component/embedding/factory.py | 12 +++++------- src/everos/config/default.toml | 3 +++ src/everos/config/settings.py | 2 ++ .../infra/persistence/lancedb/tables/agent_case.py | 4 +++- .../infra/persistence/lancedb/tables/agent_skill.py | 4 +++- .../infra/persistence/lancedb/tables/atomic_fact.py | 4 +++- .../infra/persistence/lancedb/tables/episode.py | 4 +++- .../infra/persistence/lancedb/tables/foresight.py | 4 +++- .../persistence/lancedb/tables/knowledge_topic.py | 4 +++- 9 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/everos/component/embedding/factory.py b/src/everos/component/embedding/factory.py index 304ab56ab..1d73c70e3 100644 --- a/src/everos/component/embedding/factory.py +++ b/src/everos/component/embedding/factory.py @@ -7,22 +7,19 @@ from .openai_provider import OpenAIEmbeddingProvider from .protocol import EmbeddingProvider -# Vector dim for the LanceDB index column — see ``17_lancedb_tables_design.md``. -_DEFAULT_DIM = 1024 - def build_embedding_provider( settings: EmbeddingSettings, *, - dim: int = _DEFAULT_DIM, + dim: int | None = None, ) -> EmbeddingProvider: """Build an OpenAI-compatible embedding provider from settings. Args: settings: The :class:`EmbeddingSettings` slice from :func:`everos.config.load_settings`. - dim: Target vector dimension; defaults to 1024 to match the - LanceDB ``vector`` column shape. + dim: Optional target vector dimension. When omitted, the dimension + from ``settings.dim`` is used. Returns: An :class:`EmbeddingProvider` ready to call ``embed`` / @@ -45,11 +42,12 @@ def build_embedding_provider( raise ValueError( "Embedding base_url is not configured (set EVEROS_EMBEDDING__BASE_URL)" ) + effective_dim = dim if dim is not None else settings.dim return OpenAIEmbeddingProvider( model=settings.model, api_key=api_key, base_url=settings.base_url, - dim=dim, + dim=effective_dim, timeout=settings.timeout_seconds, max_retries=settings.max_retries, batch_size=settings.batch_size, diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index ad9bf7b58..2dc49cb35 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -84,6 +84,9 @@ timeout_seconds = 30.0 max_retries = 3 batch_size = 10 max_concurrent = 5 +# Vector dimension used by LanceDB vector columns and the embedding provider. +# Override via EVEROS_EMBEDDING__DIM. +dim = 1024 [rerank] # Rerank provider. Override via env: diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index bdde544a9..9579060bb 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -180,6 +180,7 @@ class EmbeddingSettings(BaseModel): EVEROS_EMBEDDING__MAX_RETRIES EVEROS_EMBEDDING__BATCH_SIZE EVEROS_EMBEDDING__MAX_CONCURRENT + EVEROS_EMBEDDING__DIM """ model: str | None = None @@ -189,6 +190,7 @@ class EmbeddingSettings(BaseModel): max_retries: int = Field(default=3, ge=0) batch_size: int = Field(default=10, ge=1) max_concurrent: int = Field(default=5, ge=1) + dim: int = Field(default=1024, gt=0) class RerankSettings(BaseModel): diff --git a/src/everos/infra/persistence/lancedb/tables/agent_case.py b/src/everos/infra/persistence/lancedb/tables/agent_case.py index be10131d9..2fbed8392 100644 --- a/src/everos/infra/persistence/lancedb/tables/agent_case.py +++ b/src/everos/infra/persistence/lancedb/tables/agent_case.py @@ -8,6 +8,8 @@ from __future__ import annotations +from everos.config.settings import load_settings + import datetime as _dt from typing import ClassVar @@ -15,7 +17,7 @@ from ._parent_type import ParentType -_DIM = 1024 +_DIM = load_settings().embedding.dim class AgentCase(BaseLanceTable): diff --git a/src/everos/infra/persistence/lancedb/tables/agent_skill.py b/src/everos/infra/persistence/lancedb/tables/agent_skill.py index ea249930b..0b7c1212b 100644 --- a/src/everos/infra/persistence/lancedb/tables/agent_skill.py +++ b/src/everos/infra/persistence/lancedb/tables/agent_skill.py @@ -10,11 +10,13 @@ from __future__ import annotations +from everos.config.settings import load_settings + from typing import ClassVar from everos.core.persistence.lancedb import BaseLanceTable, Vector -_DIM = 1024 +_DIM = load_settings().embedding.dim class AgentSkill(BaseLanceTable): diff --git a/src/everos/infra/persistence/lancedb/tables/atomic_fact.py b/src/everos/infra/persistence/lancedb/tables/atomic_fact.py index 488ff8699..92f84c765 100644 --- a/src/everos/infra/persistence/lancedb/tables/atomic_fact.py +++ b/src/everos/infra/persistence/lancedb/tables/atomic_fact.py @@ -7,6 +7,8 @@ from __future__ import annotations +from everos.config.settings import load_settings + import datetime as _dt from typing import ClassVar @@ -14,7 +16,7 @@ from ._parent_type import ParentType -_DIM = 1024 +_DIM = load_settings().embedding.dim class AtomicFact(BaseLanceTable): diff --git a/src/everos/infra/persistence/lancedb/tables/episode.py b/src/everos/infra/persistence/lancedb/tables/episode.py index d4e2f4e30..5d102aa2c 100644 --- a/src/everos/infra/persistence/lancedb/tables/episode.py +++ b/src/everos/infra/persistence/lancedb/tables/episode.py @@ -7,6 +7,8 @@ from __future__ import annotations +from everos.config.settings import load_settings + import datetime as _dt from typing import ClassVar @@ -16,7 +18,7 @@ # Vector dimension is settings-managed at runtime; the class-level # constant pins the schema dim used at table creation. -_DIM = 1024 +_DIM = load_settings().embedding.dim class Episode(BaseLanceTable): diff --git a/src/everos/infra/persistence/lancedb/tables/foresight.py b/src/everos/infra/persistence/lancedb/tables/foresight.py index 8070d2474..5d6576c83 100644 --- a/src/everos/infra/persistence/lancedb/tables/foresight.py +++ b/src/everos/infra/persistence/lancedb/tables/foresight.py @@ -8,6 +8,8 @@ from __future__ import annotations +from everos.config.settings import load_settings + import datetime as _dt from typing import ClassVar @@ -15,7 +17,7 @@ from ._parent_type import ParentType -_DIM = 1024 +_DIM = load_settings().embedding.dim class Foresight(BaseLanceTable): diff --git a/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py b/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py index 31e12ad07..31021654b 100644 --- a/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py +++ b/src/everos/infra/persistence/lancedb/tables/knowledge_topic.py @@ -2,12 +2,14 @@ from __future__ import annotations +from everos.config.settings import load_settings + import datetime as dt from typing import ClassVar from everos.core.persistence.lancedb import BaseLanceTable, Vector -_DIM = 1024 +_DIM = load_settings().embedding.dim class KnowledgeTopic(BaseLanceTable):