Skip to content

Commit 08092cd

Browse files
afirstenbergafirstenberg
authored
google-common[major]: Media manager (#5835)
1 parent 6a723fa commit 08092cd

File tree

27 files changed

+3949
-740
lines changed

27 files changed

+3949
-740
lines changed

‎libs/langchain-google-common/.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@ types.cjs
1010
types.js
1111
types.d.ts
1212
types.d.cts
13+
experimental/media.cjs
14+
experimental/media.js
15+
experimental/media.d.ts
16+
experimental/media.d.cts
17+
experimental/utils/media_core.cjs
18+
experimental/utils/media_core.js
19+
experimental/utils/media_core.d.ts
20+
experimental/utils/media_core.d.cts
1321
node_modules
1422
dist
1523
.yarn

‎libs/langchain-google-common/langchain.config.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ export const config = {
1616
index: "index",
1717
utils: "utils/index",
1818
types: "types",
19+
"experimental/media": "experimental/media",
20+
"experimental/utils/media_core": "experimental/utils/media_core",
1921
},
2022
tsConfigPath: resolve("./tsconfig.json"),
2123
cjsSource: "./dist-cjs",

‎libs/langchain-google-common/package.json‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,24 @@
9292
"import": "./types.js",
9393
"require": "./types.cjs"
9494
},
95+
"./experimental/media": {
96+
"types": {
97+
"import": "./experimental/media.d.ts",
98+
"require": "./experimental/media.d.cts",
99+
"default": "./experimental/media.d.ts"
100+
},
101+
"import": "./experimental/media.js",
102+
"require": "./experimental/media.cjs"
103+
},
104+
"./experimental/utils/media_core": {
105+
"types": {
106+
"import": "./experimental/utils/media_core.d.ts",
107+
"require": "./experimental/utils/media_core.d.cts",
108+
"default": "./experimental/utils/media_core.d.ts"
109+
},
110+
"import": "./experimental/utils/media_core.js",
111+
"require": "./experimental/utils/media_core.cjs"
112+
},
95113
"./package.json": "./package.json"
96114
},
97115
"files": [
@@ -107,6 +125,14 @@
107125
"types.cjs",
108126
"types.js",
109127
"types.d.ts",
110-
"types.d.cts"
128+
"types.d.cts",
129+
"experimental/media.cjs",
130+
"experimental/media.js",
131+
"experimental/media.d.ts",
132+
"experimental/media.d.cts",
133+
"experimental/utils/media_core.cjs",
134+
"experimental/utils/media_core.js",
135+
"experimental/utils/media_core.d.ts",
136+
"experimental/utils/media_core.d.cts"
111137
]
112138
}

‎libs/langchain-google-common/src/auth.ts‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { ReadableJsonStream } from "./utils/stream.js";
22
import { GooglePlatformType } from "./types.js";
33

4-
export type GoogleAbstractedClientOpsMethod = "GET" | "POST";
4+
export type GoogleAbstractedClientOpsMethod = "GET" | "POST" | "DELETE";
55

6-
export type GoogleAbstractedClientOpsResponseType = "json" | "stream";
6+
export type GoogleAbstractedClientOpsResponseType = "json" | "stream" | "blob";
77

88
export type GoogleAbstractedClientOps = {
99
url?: string;
@@ -28,6 +28,17 @@ export abstract class GoogleAbstractedFetchClient
2828

2929
abstract request(opts: GoogleAbstractedClientOps): unknown;
3030

31+
async _buildData(res: Response, opts: GoogleAbstractedClientOps) {
32+
switch (opts.responseType) {
33+
case "json":
34+
return res.json();
35+
case "stream":
36+
return new ReadableJsonStream(res.body);
37+
default:
38+
return res.blob();
39+
}
40+
}
41+
3142
async _request(
3243
url: string | undefined,
3344
opts: GoogleAbstractedClientOps,
@@ -47,7 +58,11 @@ export abstract class GoogleAbstractedFetchClient
4758
},
4859
};
4960
if (opts.data !== undefined) {
50-
fetchOptions.body = JSON.stringify(opts.data);
61+
if (typeof opts.data === "string") {
62+
fetchOptions.body = opts.data;
63+
} else {
64+
fetchOptions.body = JSON.stringify(opts.data);
65+
}
5166
}
5267

5368
const res = await fetch(url, fetchOptions);
@@ -57,16 +72,21 @@ export abstract class GoogleAbstractedFetchClient
5772
const error = new Error(
5873
`Google request failed with status code ${res.status}: ${resText}`
5974
);
60-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
75+
/* eslint-disable @typescript-eslint/no-explicit-any */
6176
(error as any).response = res;
77+
(error as any).details = {
78+
url,
79+
opts,
80+
fetchOptions,
81+
result: res,
82+
};
83+
/* eslint-enable @typescript-eslint/no-explicit-any */
6284
throw error;
6385
}
6486

87+
const data = await this._buildData(res, opts);
6588
return {
66-
data:
67-
opts.responseType === "json"
68-
? await res.json()
69-
: new ReadableJsonStream(res.body),
89+
data,
7090
config: {},
7191
status: res.status,
7292
statusText: res.statusText,

‎libs/langchain-google-common/src/chat_models.ts‎

Lines changed: 49 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,7 @@ import {
3939
copyAndValidateModelParamsInto,
4040
} from "./utils/common.js";
4141
import { AbstractGoogleLLMConnection } from "./connection.js";
42-
import {
43-
baseMessageToContent,
44-
safeResponseToChatGeneration,
45-
safeResponseToChatResult,
46-
DefaultGeminiSafetyHandler,
47-
} from "./utils/gemini.js";
42+
import { DefaultGeminiSafetyHandler } from "./utils/gemini.js";
4843
import { ApiKeyGoogleAuth, GoogleAbstractedClient } from "./auth.js";
4944
import { JsonStream } from "./utils/stream.js";
5045
import { ensureParams } from "./utils/failed_handler.js";
@@ -55,6 +50,7 @@ import type {
5550
GeminiFunctionDeclaration,
5651
GeminiFunctionSchema,
5752
GoogleAIToolType,
53+
GeminiAPIConfig,
5854
} from "./types.js";
5955
import { zodToGeminiParameters } from "./utils/zod_to_gemini_parameters.js";
6056

@@ -100,61 +96,69 @@ class ChatConnection<AuthOptions> extends AbstractGoogleLLMConnection<
10096
return true;
10197
}
10298

103-
formatContents(
99+
async formatContents(
104100
input: BaseMessage[],
105101
_parameters: GoogleAIModelParams
106-
): GeminiContent[] {
107-
return input
108-
.map((msg, i) =>
109-
baseMessageToContent(msg, input[i - 1], this.useSystemInstruction)
102+
): Promise<GeminiContent[]> {
103+
const inputPromises: Promise<GeminiContent[]>[] = input.map((msg, i) =>
104+
this.api.baseMessageToContent(
105+
msg,
106+
input[i - 1],
107+
this.useSystemInstruction
110108
)
111-
.reduce((acc, cur) => {
112-
// Filter out the system content
113-
if (cur.every((content) => content.role === "system")) {
114-
return acc;
115-
}
116-
117-
// Combine adjacent function messages
118-
if (
119-
cur[0]?.role === "function" &&
120-
acc.length > 0 &&
121-
acc[acc.length - 1].role === "function"
122-
) {
123-
acc[acc.length - 1].parts = [
124-
...acc[acc.length - 1].parts,
125-
...cur[0].parts,
126-
];
127-
} else {
128-
acc.push(...cur);
129-
}
109+
);
110+
const inputs = await Promise.all(inputPromises);
130111

112+
return inputs.reduce((acc, cur) => {
113+
// Filter out the system content
114+
if (cur.every((content) => content.role === "system")) {
131115
return acc;
132-
}, [] as GeminiContent[]);
116+
}
117+
118+
// Combine adjacent function messages
119+
if (
120+
cur[0]?.role === "function" &&
121+
acc.length > 0 &&
122+
acc[acc.length - 1].role === "function"
123+
) {
124+
acc[acc.length - 1].parts = [
125+
...acc[acc.length - 1].parts,
126+
...cur[0].parts,
127+
];
128+
} else {
129+
acc.push(...cur);
130+
}
131+
132+
return acc;
133+
}, [] as GeminiContent[]);
133134
}
134135

135-
formatSystemInstruction(
136+
async formatSystemInstruction(
136137
input: BaseMessage[],
137138
_parameters: GoogleAIModelParams
138-
): GeminiContent {
139+
): Promise<GeminiContent> {
139140
if (!this.useSystemInstruction) {
140141
return {} as GeminiContent;
141142
}
142143

143144
let ret = {} as GeminiContent;
144-
input.forEach((message, index) => {
145+
for (let index = 0; index < input.length; index += 1) {
146+
const message = input[index];
145147
if (message._getType() === "system") {
146148
// For system types, we only want it if it is the first message,
147149
// if it appears anywhere else, it should be an error.
148150
if (index === 0) {
149151
// eslint-disable-next-line prefer-destructuring
150-
ret = baseMessageToContent(message, undefined, true)[0];
152+
ret = (
153+
await this.api.baseMessageToContent(message, undefined, true)
154+
)[0];
151155
} else {
152156
throw new Error(
153157
"System messages are only permitted as the first passed message."
154158
);
155159
}
156160
}
157-
});
161+
}
158162

159163
return ret;
160164
}
@@ -168,6 +172,7 @@ export interface ChatGoogleBaseInput<AuthOptions>
168172
GoogleConnectionParams<AuthOptions>,
169173
GoogleAIModelParams,
170174
GoogleAISafetyParams,
175+
GeminiAPIConfig,
171176
Pick<GoogleAIBaseLanguageModelCallOptions, "streamUsage"> {}
172177

173178
/**
@@ -338,7 +343,10 @@ export abstract class ChatGoogleBase<AuthOptions>
338343
parameters,
339344
options
340345
);
341-
const ret = safeResponseToChatResult(response, this.safetyHandler);
346+
const ret = this.connection.api.safeResponseToChatResult(
347+
response,
348+
this.safetyHandler
349+
);
342350
await runManager?.handleLLMNewToken(ret.generations[0].text);
343351
return ret;
344352
}
@@ -378,7 +386,10 @@ export abstract class ChatGoogleBase<AuthOptions>
378386
}
379387
const chunk =
380388
output !== null
381-
? safeResponseToChatGeneration({ data: output }, this.safetyHandler)
389+
? this.connection.api.safeResponseToChatGeneration(
390+
{ data: output },
391+
this.safetyHandler
392+
)
382393
: new ChatGenerationChunk({
383394
text: "",
384395
generationInfo: { finishReason: "stop" },

0 commit comments

Comments
 (0)