Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ The Node.js backend layer built with `@databricks/appkit`:

Integration with Databricks services:
- **SQL Warehouses**: Execute analytical queries with Arrow or JSON format
- **Lakebase**: Access data from Lakebase
- **Lakebase V1 (Provisioned)**: Access data from Lakebase Provisioned. Support for Lakebase Autoscaling coming soon.

## See also

Expand Down
2 changes: 1 addition & 1 deletion docs/docs/plugins.md
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ await createApp({
});
```

Storage auto-selects **Lakebase persistent cache when healthy**, otherwise falls back to in-memory.
Storage auto-selects **Lakebase V1 (Provisioned) persistent cache when healthy**, otherwise falls back to in-memory. Support for Lakebase Autoscaling coming soon.

### Plugin-level caching

Expand Down
4 changes: 2 additions & 2 deletions packages/appkit/src/cache/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { createHash } from "node:crypto";
import { WorkspaceClient } from "@databricks/sdk-experimental";
import type { CacheConfig, CacheStorage } from "shared";
import { LakebaseConnector } from "@/connectors";
import { LakebaseV1Connector } from "@/connectors";
import { AppKitError, ExecutionError, InitializationError } from "../errors";
import { createLogger } from "../logging/logger";
import type { Counter, TelemetryProvider } from "../telemetry";
Expand Down Expand Up @@ -147,7 +147,7 @@ export class CacheManager {
// try to use lakebase storage
try {
const workspaceClient = new WorkspaceClient({});
const connector = new LakebaseConnector({ workspaceClient });
const connector = new LakebaseV1Connector({ workspaceClient });
const isHealthy = await connector.healthCheck();

if (isHealthy) {
Expand Down
6 changes: 3 additions & 3 deletions packages/appkit/src/cache/storage/persistent.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import type { CacheConfig, CacheEntry, CacheStorage } from "shared";
import type { LakebaseConnector } from "../../connectors";
import type { LakebaseV1Connector } from "../../connectors";
import { InitializationError, ValidationError } from "../../errors";
import { createLogger } from "../../logging/logger";
import { lakebaseStorageDefaults } from "./defaults";
Expand All @@ -22,15 +22,15 @@ const logger = createLogger("cache:persistent");
*
*/
export class PersistentStorage implements CacheStorage {
private readonly connector: LakebaseConnector;
private readonly connector: LakebaseV1Connector;
private readonly tableName: string;
private readonly maxBytes: number;
private readonly maxEntryBytes: number;
private readonly evictionBatchSize: number;
private readonly evictionCheckProbability: number;
private initialized: boolean;

constructor(config: CacheConfig, connector: LakebaseConnector) {
constructor(config: CacheConfig, connector: LakebaseV1Connector) {
this.connector = connector;
this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes;
this.maxEntryBytes =
Expand Down
4 changes: 2 additions & 2 deletions packages/appkit/src/cache/tests/cache-manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ import type { CacheStorage } from "shared";
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { CacheManager } from "../../index";

// Mock LakebaseConnector
// Mock LakebaseV1Connector
const mockLakebaseHealthCheck = vi.fn();
vi.mock("@/connectors", () => ({
LakebaseConnector: vi.fn().mockImplementation(() => ({
LakebaseV1Connector: vi.fn().mockImplementation(() => ({
healthCheck: mockLakebaseHealthCheck,
close: vi.fn().mockResolvedValue(undefined),
})),
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/cache/tests/persistent.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { beforeEach, describe, expect, test, vi } from "vitest";
import { PersistentStorage } from "../storage";

/** Mock LakebaseConnector for testing */
/** Mock LakebaseV1Connector for testing */
const createMockConnector = () => ({
query: vi.fn(),
healthCheck: vi.fn().mockResolvedValue(true),
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/connectors/index.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export * from "./lakebase";
export * from "./lakebase-v1";
export * from "./sql-warehouse";
Original file line number Diff line number Diff line change
Expand Up @@ -18,37 +18,51 @@ import {
} from "../../errors";
import { createLogger } from "../../logging/logger";
import { deepMerge } from "../../utils";
import { lakebaseDefaults } from "./defaults";
import { lakebaseV1Defaults } from "./defaults";
import type {
LakebaseConfig,
LakebaseConnectionConfig,
LakebaseCredentials,
LakebaseV1Config,
LakebaseV1ConnectionConfig,
LakebaseV1Credentials,
} from "./types";

const logger = createLogger("connectors:lakebase");
const logger = createLogger("connectors:lakebase-v1");

/**
* Enterprise-grade connector for Databricks Lakebase
* Enterprise-grade connector for Databricks Lakebase Provisioned
*
* @deprecated This connector is for Lakebase Provisioned only.
* For new projects, use Lakebase Autoscaling instead: https://docs.databricks.com/aws/en/oltp/projects/
*
* This connector is compatible with Lakebase Provisioned: https://docs.databricks.com/aws/en/oltp/instances/
*
* Lakebase Autoscaling offers:
* - Automatic compute scaling
* - Scale-to-zero for cost optimization
* - Database branching for development
* - Instant restore capabilities
*
* Use the new LakebaseConnector (coming in a future release) for Lakebase Autoscaling support.
*
* @example Simplest - everything from env/context
* ```typescript
* const connector = new LakebaseConnector();
* const connector = new LakebaseV1Connector();
* await connector.query('SELECT * FROM users');
* ```
*
* @example With explicit connection string
* ```typescript
* const connector = new LakebaseConnector({
* const connector = new LakebaseV1Connector({
* connectionString: 'postgresql://...'
* });
* ```
*/
export class LakebaseConnector {
private readonly name: string = "lakebase";
export class LakebaseV1Connector {
private readonly name: string = "lakebase-v1";
private readonly CACHE_BUFFER_MS = 2 * 60 * 1000;
private readonly config: LakebaseConfig;
private readonly connectionConfig: LakebaseConnectionConfig;
private readonly config: LakebaseV1Config;
private readonly connectionConfig: LakebaseV1ConnectionConfig;
private pool: pg.Pool | null = null;
private credentials: LakebaseCredentials | null = null;
private credentials: LakebaseV1Credentials | null = null;

// telemetry
private readonly telemetry: TelemetryProvider;
Expand All @@ -57,8 +71,8 @@ export class LakebaseConnector {
queryDuration: Histogram;
};

constructor(userConfig?: Partial<LakebaseConfig>) {
this.config = deepMerge(lakebaseDefaults, userConfig);
constructor(userConfig?: Partial<LakebaseV1Config>) {
this.config = deepMerge(lakebaseV1Defaults, userConfig);
this.connectionConfig = this.parseConnectionConfig();

this.telemetry = TelemetryManager.getProvider(
Expand All @@ -68,13 +82,13 @@ export class LakebaseConnector {
this.telemetryMetrics = {
queryCount: this.telemetry
.getMeter()
.createCounter("lakebase.query.count", {
.createCounter("lakebase.v1.query.count", {
description: "Total number of queries executed",
unit: "1",
}),
queryDuration: this.telemetry
.getMeter()
.createHistogram("lakebase.query.duration", {
.createHistogram("lakebase.v1.query.duration", {
description: "Duration of queries executed",
unit: "ms",
}),
Expand Down Expand Up @@ -107,10 +121,10 @@ export class LakebaseConnector {
const startTime = Date.now();

return this.telemetry.startActiveSpan(
"lakebase.query",
"lakebase.v1.query",
{
attributes: {
"db.system": "lakebase",
"db.system": "lakebase-v1",
"db.statement": sql.substring(0, 500),
"db.retry_count": retryCount,
},
Expand Down Expand Up @@ -178,10 +192,10 @@ export class LakebaseConnector {
): Promise<T> {
const startTime = Date.now();
return this.telemetry.startActiveSpan(
"lakebase.transaction",
"lakebase.v1.transaction",
{
attributes: {
"db.system": "lakebase",
"db.system": "lakebase-v1",
"db.retry_count": retryCount,
},
},
Expand Down Expand Up @@ -249,7 +263,7 @@ export class LakebaseConnector {
/** Check if database connection is healthy */
async healthCheck(): Promise<boolean> {
return this.telemetry.startActiveSpan(
"lakebase.healthCheck",
"lakebase.v1.healthCheck",
{},
async (span) => {
try {
Expand Down Expand Up @@ -491,7 +505,7 @@ export class LakebaseConnector {
}

/** Parse connection configuration from config or environment */
private parseConnectionConfig(): LakebaseConnectionConfig {
private parseConnectionConfig(): LakebaseV1ConnectionConfig {
if (this.config.connectionString) {
return this.parseConnectionString(this.config.connectionString);
}
Expand Down Expand Up @@ -539,7 +553,7 @@ export class LakebaseConnector {

private parseConnectionString(
connectionString: string,
): LakebaseConnectionConfig {
): LakebaseV1ConnectionConfig {
const url = new URL(connectionString);
const appName = url.searchParams.get("appName");
if (!appName) {
Expand Down
15 changes: 15 additions & 0 deletions packages/appkit/src/connectors/lakebase-v1/defaults.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import type { LakebaseV1Config } from "./types";

/**
* Default configuration for Lakebase V1 connector
*
* @deprecated This connector is for Lakebase Provisioned only.
* For new projects, use Lakebase Autoscaling: https://docs.databricks.com/aws/en/oltp/projects/
*/
export const lakebaseV1Defaults: LakebaseV1Config = {
port: 5432,
sslMode: "require",
maxPoolSize: 10,
idleTimeoutMs: 30_000,
connectionTimeoutMs: 10_000,
};
6 changes: 6 additions & 0 deletions packages/appkit/src/connectors/lakebase-v1/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
export { LakebaseV1Connector } from "./client";
export type {
LakebaseV1Config,
LakebaseV1ConnectionConfig,
LakebaseV1Credentials,
} from "./types";
Original file line number Diff line number Diff line change
@@ -1,8 +1,23 @@
import type { WorkspaceClient } from "@databricks/sdk-experimental";
import type { TelemetryOptions } from "shared";

/** Configuration for LakebaseConnector */
export interface LakebaseConfig {
/**
* Configuration for LakebaseV1Connector
*
* @deprecated This connector is for Lakebase Provisioned only.
* For new projects, use Lakebase Autoscaling instead: https://docs.databricks.com/aws/en/oltp/projects/
*
* This connector is compatible with Lakebase Provisioned: https://docs.databricks.com/aws/en/oltp/instances/
*
* Lakebase Autoscaling offers:
* - Automatic compute scaling
* - Scale-to-zero for cost optimization
* - Database branching for development
* - Instant restore capabilities
*
* Use the new LakebaseConnector (coming in a future release) for Lakebase Autoscaling support.
*/
export interface LakebaseV1Config {
/** Databricks workspace client */
workspaceClient?: WorkspaceClient;

Expand Down Expand Up @@ -40,8 +55,13 @@ export interface LakebaseConfig {
[key: string]: unknown;
}

/** Lakebase credentials for authentication */
export interface LakebaseCredentials {
/**
* Lakebase V1 credentials for authentication
*
* @deprecated This type is for Lakebase Provisioned only.
* For new projects, use Lakebase Autoscaling: https://docs.databricks.com/aws/en/oltp/projects/
*/
export interface LakebaseV1Credentials {
/** Username */
username: string;
/** Password */
Expand All @@ -50,8 +70,13 @@ export interface LakebaseCredentials {
expiresAt: number;
}

/** Internal connection configuration */
export interface LakebaseConnectionConfig {
/**
* Internal connection configuration for Lakebase V1
*
* @deprecated This type is for Lakebase Provisioned only.
* For new projects, use Lakebase Autoscaling: https://docs.databricks.com/aws/en/oltp/projects/
*/
export interface LakebaseV1ConnectionConfig {
/** Database host */
readonly host: string;
/** Database name */
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { ConfigurationError, ValidationError } from "../../errors";
import type { LakebaseConnectionConfig } from "./types";
import type { LakebaseV1ConnectionConfig } from "./types";

export interface ParsedConnectionString {
connectionParams: LakebaseConnectionConfig;
connectionParams: LakebaseV1ConnectionConfig;
originalConnectionString: string;
}

Expand Down Expand Up @@ -40,7 +40,7 @@ export function parseConnectionString(
(url.searchParams.get("sslmode") as "require" | "disable" | "prefer") ||
"require";

const connectionParams: LakebaseConnectionConfig = {
const connectionParams: LakebaseV1ConnectionConfig = {
host: url.hostname,
database: dbName,
port: url.port ? parseInt(url.port, 10) : 5432,
Expand All @@ -57,7 +57,7 @@ export function parseConnectionString(
throw ValidationError.missingField("database");
}

const connectionParams: LakebaseConnectionConfig = {
const connectionParams: LakebaseV1ConnectionConfig = {
host: connectionStringOrHost,
database,
port: port || 5432,
Expand All @@ -71,7 +71,7 @@ export function parseConnectionString(
}

/** Parse connection configuration from environment variables */
export function parseFromEnv(): LakebaseConnectionConfig {
export function parseFromEnv(): LakebaseV1ConnectionConfig {
const host = process.env.PGHOST;
const database = process.env.PGDATABASE;
const port = process.env.PGPORT ? parseInt(process.env.PGPORT, 10) : 5432;
Expand Down
10 changes: 0 additions & 10 deletions packages/appkit/src/connectors/lakebase/defaults.ts

This file was deleted.

2 changes: 0 additions & 2 deletions packages/appkit/src/connectors/lakebase/index.ts

This file was deleted.

Loading