diff --git a/genkit-tools/common/src/types/document.ts b/genkit-tools/common/src/types/document.ts index 327cedd408..c21fbd9362 100644 --- a/genkit-tools/common/src/types/document.ts +++ b/genkit-tools/common/src/types/document.ts @@ -27,6 +27,7 @@ const EmptyPartSchema = z.object({ data: z.unknown().optional(), metadata: z.record(z.unknown()).optional(), custom: z.record(z.unknown()).optional(), + reasoning: z.never().optional(), }); /** @@ -37,6 +38,14 @@ export const TextPartSchema = EmptyPartSchema.extend({ text: z.string(), }); +/** + * Zod schema for a reasoning part. + */ +export const ReasoningPartSchema = EmptyPartSchema.extend({ + /** The reasoning text of the message. */ + reasoning: z.string(), +}); + /** * Text part. */ diff --git a/genkit-tools/common/src/types/model.ts b/genkit-tools/common/src/types/model.ts index 1493d6d283..4f1b99deac 100644 --- a/genkit-tools/common/src/types/model.ts +++ b/genkit-tools/common/src/types/model.ts @@ -22,6 +22,7 @@ import { DocumentDataSchema, MediaPart, MediaPartSchema, + ReasoningPartSchema, TextPart, TextPartSchema, ToolRequestPart, @@ -58,6 +59,7 @@ export const PartSchema = z.union([ ToolResponsePartSchema, DataPartSchema, CustomPartSchema, + ReasoningPartSchema, ]); /** diff --git a/genkit-tools/genkit-schema.json b/genkit-tools/genkit-schema.json index 5113130e33..b95df092e5 100644 --- a/genkit-tools/genkit-schema.json +++ b/genkit-tools/genkit-schema.json @@ -24,6 +24,9 @@ "custom": { "type": "object", "additionalProperties": {} + }, + "reasoning": { + "not": {} } }, "required": [ @@ -53,6 +56,9 @@ "custom": { "type": "object", "additionalProperties": {} + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" } }, "additionalProperties": false @@ -109,6 +115,9 @@ }, "custom": { "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" } }, "required": [ @@ -131,6 +140,39 @@ ], "additionalProperties": false }, + "ReasoningPart": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/CustomPart/properties/text" + }, + "media": { + "$ref": "#/$defs/CustomPart/properties/media" + }, + "toolRequest": { + "$ref": "#/$defs/CustomPart/properties/toolRequest" + }, + "toolResponse": { + "$ref": "#/$defs/CustomPart/properties/toolResponse" + }, + "data": { + "$ref": "#/$defs/CustomPart/properties/data" + }, + "metadata": { + "$ref": "#/$defs/CustomPart/properties/metadata" + }, + "custom": { + "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "type": "string" + } + }, + "required": [ + "reasoning" + ], + "additionalProperties": false + }, "TextPart": { "type": "object", "properties": { @@ -154,6 +196,9 @@ }, "custom": { "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" } }, "required": [ @@ -184,6 +229,9 @@ }, "custom": { "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" } }, "required": [ @@ -230,6 +278,9 @@ }, "custom": { "$ref": "#/$defs/DataPart/properties/custom" + }, + "reasoning": { + "$ref": "#/$defs/CustomPart/properties/reasoning" } }, "required": [ @@ -1043,6 +1094,9 @@ }, { "$ref": "#/$defs/CustomPart" + }, + { + "$ref": "#/$defs/ReasoningPart" } ] }, diff --git a/js/ai/src/document.ts b/js/ai/src/document.ts index e04812c81e..eb7c0d1693 100644 --- a/js/ai/src/document.ts +++ b/js/ai/src/document.ts @@ -25,6 +25,7 @@ const EmptyPartSchema = z.object({ data: z.unknown().optional(), metadata: z.record(z.unknown()).optional(), custom: z.record(z.unknown()).optional(), + reasoning: z.never().optional(), }); /** @@ -35,6 +36,14 @@ export const TextPartSchema = EmptyPartSchema.extend({ text: z.string(), }); +/** + * Zod schema for a reasoning part. + */ +export const ReasoningPartSchema = EmptyPartSchema.extend({ + /** The reasoning text of the message. */ + reasoning: z.string(), +}); + /** * Text part. */ diff --git a/js/ai/src/generate/chunk.ts b/js/ai/src/generate/chunk.ts index 231cbf9390..fb061bb531 100644 --- a/js/ai/src/generate/chunk.ts +++ b/js/ai/src/generate/chunk.ts @@ -70,6 +70,14 @@ export class GenerateResponseChunk return this.content.map((part) => part.text || '').join(''); } + /** + * Concatenates all `reasoning` parts present in the chunk with no delimiter. + * @returns A string of all concatenated reasoning parts. + */ + get reasoning(): string { + return this.content.map((part) => part.reasoning || '').join(''); + } + /** * Concatenates all `text` parts of all chunks from the response thus far. * @returns A string of all concatenated chunk text content. diff --git a/js/ai/src/generate/response.ts b/js/ai/src/generate/response.ts index c03970fd7a..03e1763a3c 100644 --- a/js/ai/src/generate/response.ts +++ b/js/ai/src/generate/response.ts @@ -44,6 +44,8 @@ export class GenerateResponse implements ModelResponseData { usage: GenerationUsage; /** Provider-specific response data. */ custom: unknown; + /** Provider-specific response data. */ + raw: unknown; /** The request that generated this response. */ request?: GenerateRequest; /** The parser for output parsing of this response. */ @@ -70,6 +72,7 @@ export class GenerateResponse implements ModelResponseData { response.finishMessage || response.candidates?.[0]?.finishMessage; this.usage = response.usage || {}; this.custom = response.custom || {}; + this.raw = response.raw || this.custom; this.request = options?.request; } @@ -133,6 +136,14 @@ export class GenerateResponse implements ModelResponseData { return this.message?.text || ''; } + /** + * Concatenates all `reasoning` parts present in the generated message with no delimiter. + * @returns A string of all concatenated reasoning parts. + */ + get reasoning(): string { + return this.message?.reasoning || ''; + } + /** * Returns the first detected media part in the generated message. Useful for * extracting (for example) an image from a generation expected to create one. @@ -184,10 +195,6 @@ export class GenerateResponse implements ModelResponseData { return [...this.request?.messages, this.message.toJSON()]; } - get raw(): unknown { - return this.raw ?? this.custom; - } - toJSON(): ModelResponseData { const out = { message: this.message?.toJSON(), diff --git a/js/ai/src/message.ts b/js/ai/src/message.ts index a30274f95e..ec2a620b4b 100644 --- a/js/ai/src/message.ts +++ b/js/ai/src/message.ts @@ -95,6 +95,14 @@ export class Message implements MessageData { return this.content.map((part) => part.text || '').join(''); } + /** + * Concatenates all `reasoning` parts present in the message with no delimiter. + * @returns A string of all concatenated reasoning parts. + */ + get reasoning(): string { + return this.content.map((part) => part.reasoning || '').join(''); + } + /** * Returns the first media part detected in the message. Useful for extracting * (for example) an image from a generation expected to create one. diff --git a/js/ai/src/model.ts b/js/ai/src/model.ts index 5409534bcc..019847aa55 100644 --- a/js/ai/src/model.ts +++ b/js/ai/src/model.ts @@ -36,6 +36,7 @@ import { DocumentDataSchema, MediaPart, MediaPartSchema, + ReasoningPartSchema, TextPart, TextPartSchema, ToolRequestPart, @@ -81,6 +82,7 @@ export const PartSchema = z.union([ ToolResponsePartSchema, DataPartSchema, CustomPartSchema, + ReasoningPartSchema, ]); /** diff --git a/js/plugins/googleai/src/gemini.ts b/js/plugins/googleai/src/gemini.ts index d8a0296348..1aea1d8d13 100644 --- a/js/plugins/googleai/src/gemini.ts +++ b/js/plugins/googleai/src/gemini.ts @@ -690,7 +690,12 @@ function fromGeminiPart( jsonMode: boolean, ref: string ): Part { - if (part.text !== undefined) return { text: part.text }; + if (part.text !== undefined) { + if ((part as any).thought === true) { + return { reasoning: part.text }; + } + return { text: part.text }; + } if (part.inlineData) return fromInlineData(part); if (part.functionCall) return fromFunctionCall(part, ref); if (part.functionResponse) return fromFunctionResponse(part); diff --git a/js/plugins/vertexai/src/gemini.ts b/js/plugins/vertexai/src/gemini.ts index 19263a9e96..8e68350a7a 100644 --- a/js/plugins/vertexai/src/gemini.ts +++ b/js/plugins/vertexai/src/gemini.ts @@ -819,7 +819,12 @@ function fromGeminiPart( jsonMode: boolean, ref?: string ): Part { - if (part.text !== undefined) return { text: part.text }; + if (part.text !== undefined) { + if ((part as any).thought === true) { + return { reasoning: part.text }; + } + return { text: part.text }; + } if (part.inlineData) return fromGeminiInlineDataPart(part); if (part.fileData) return fromGeminiFileDataPart(part); if (part.functionCall) return fromGeminiFunctionCallPart(part, ref); diff --git a/js/testapps/flow-simple-ai/src/index.ts b/js/testapps/flow-simple-ai/src/index.ts index 77fca76ae4..e4700482ce 100644 --- a/js/testapps/flow-simple-ai/src/index.ts +++ b/js/testapps/flow-simple-ai/src/index.ts @@ -945,3 +945,19 @@ ai.defineFlow('embedders-tester', async () => { }) ); }); + +ai.defineFlow('reasoning', async (_, { sendChunk }) => { + const { message } = await ai.generate({ + prompt: 'whats heavier, one kilo of steel or or one kilo of feathers', + model: googleAI.model('gemini-2.5-flash-preview-04-17'), + config: { + thinkingConfig: { + thinkingBudget: 1024, + includeThoughts: true, + }, + }, + onChunk: sendChunk, + }); + + return message; +}); diff --git a/py/packages/genkit/src/genkit/core/typing.py b/py/packages/genkit/src/genkit/core/typing.py index 526f25c511..fae3706e00 100644 --- a/py/packages/genkit/src/genkit/core/typing.py +++ b/py/packages/genkit/src/genkit/core/typing.py @@ -55,6 +55,7 @@ class CustomPart(BaseModel): data: Any | None = None metadata: dict[str, Any] | None = None custom: dict[str, Any] + reasoning: Any | None = None class Media(BaseModel): @@ -479,6 +480,12 @@ class Metadata(RootModel[dict[str, Any] | None]): root: dict[str, Any] | None = None +class Reasoning(RootModel[Any]): + """Root model for reasoning.""" + + root: Any + + class Text(RootModel[Any]): """Root model for text.""" @@ -574,6 +581,7 @@ class DataPart(BaseModel): data: Any | None = None metadata: Metadata | None = None custom: dict[str, Any] | None = None + reasoning: Reasoning | None = None class MediaPart(BaseModel): @@ -587,6 +595,21 @@ class MediaPart(BaseModel): data: Data | None = None metadata: Metadata | None = None custom: Custom | None = None + reasoning: Reasoning | None = None + + +class ReasoningPart(BaseModel): + """Model for reasoningpart data.""" + + model_config = ConfigDict(extra='forbid', populate_by_name=True) + text: Text | None = None + media: MediaModel | None = None + tool_request: ToolRequestModel | None = Field(None, alias='toolRequest') + tool_response: ToolResponseModel | None = Field(None, alias='toolResponse') + data: Data | None = None + metadata: Metadata | None = None + custom: Custom | None = None + reasoning: str class TextPart(BaseModel): @@ -600,6 +623,7 @@ class TextPart(BaseModel): data: Data | None = None metadata: Metadata | None = None custom: Custom | None = None + reasoning: Reasoning | None = None class ToolRequestPart(BaseModel): @@ -613,6 +637,7 @@ class ToolRequestPart(BaseModel): data: Data | None = None metadata: Metadata | None = None custom: Custom | None = None + reasoning: Reasoning | None = None class ToolResponsePart(BaseModel): @@ -626,6 +651,7 @@ class ToolResponsePart(BaseModel): data: Data | None = None metadata: Metadata | None = None custom: Custom | None = None + reasoning: Reasoning | None = None class EmbedResponse(BaseModel): @@ -673,10 +699,12 @@ class Resume(BaseModel): metadata: dict[str, Any] | None = None -class Part(RootModel[TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart]): +class Part( + RootModel[TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart | ReasoningPart] +): """Root model for part.""" - root: TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart + root: TextPart | MediaPart | ToolRequestPart | ToolResponsePart | DataPart | CustomPart | ReasoningPart class Link(BaseModel):