From 3e31d4d50145758af14456a683610f0940901888 Mon Sep 17 00:00:00 2001 From: Rob Colbert Date: Thu, 7 May 2026 00:10:57 -0400 Subject: [PATCH] agent, tools, toolbox, tool loop, AI environment --- README.md | 2 +- docs/agent-toolbox.md | 860 +++++++++++++++++++++++++ docs/architecture.md | 2 +- docs/configuration.md | 2 +- gadget-code/src/web-cli.ts | 10 +- gadget-drone/package.json | 2 +- gadget-drone/src/services/agent.ts | 38 +- gadget-drone/src/services/ai.ts | 10 +- packages/ai/package.json | 5 +- packages/ai/src/api.ts | 68 +- packages/ai/src/config/env.ts | 25 +- packages/ai/src/index.ts | 2 + packages/ai/src/ollama.ts | 125 +++- packages/ai/src/openai.ts | 136 ++-- packages/ai/src/tools/search/google.ts | 17 +- 15 files changed, 1219 insertions(+), 85 deletions(-) create mode 100644 docs/agent-toolbox.md diff --git a/README.md b/README.md index 621e150..5f698c9 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ pnpm dev # Drone worker (in a workspace directory) cd ~/my-gadget-workspace -pnpm --filter @gadget/drone dev +pnpm --filter gadget-drone dev ``` ### Testing diff --git a/docs/agent-toolbox.md b/docs/agent-toolbox.md new file mode 100644 index 0000000..158db8d --- /dev/null +++ b/docs/agent-toolbox.md @@ -0,0 +1,860 @@ +# Agent Toolbox Documentation + +## Overview + +The Agent Toolbox is a foundational component of the Gadget platform that enables AI agents to perform real-world actions through a standardized, extensible tool system. Tools are functions that agents can invoke to interact with external services, manipulate files, search the web, and execute various operations. + +## Architecture + +### Design Philosophy + +The toolbox system is built on these core principles: + +1. **Provider Agnostic**: Tools work with any AI provider (Ollama, OpenAI, etc.) +2. **Type Safety**: Full TypeScript typing from definition to execution +3. **Extensibility**: Easy to add new tools without modifying core infrastructure +4. **Security**: Tools require explicit credentials configured in the environment +5. **Error Handling**: Comprehensive error reporting with recovery hints + +### Component Overview + +``` +┌─────────────────┐ +│ AiToolbox │ ← Manages tool registration and lookup +│ - env │ ← Holds IAiEnvironment with credentials +│ - tools │ ← Map of all registered tools +│ - modeSets │ ← Tools organized by ChatSessionMode +└────────┬────────┘ + │ + │ register() + ▼ +┌─────────────────┐ +│ AiTool │ ← Abstract base class for all tools +│ - name │ ← Unique tool identifier +│ - category │ ← Tool category (search, file, etc.) +│ - definition │ ← JSON Schema for AI provider +│ - execute() │ ← Tool implementation +│ - toolbox │ ← Access to environment & other tools +└────────┬────────┘ + │ + │ extends + ▼ +┌─────────────────┐ +│ GoogleSearch │ ← Example tool implementation +│ GrepTool │ ← Future tools... +│ ... │ +└─────────────────┘ +``` + +## Core Interfaces + +### IAiEnvironment + +The `IAiEnvironment` interface carries configuration and credentials from the application layer (which has access to YAML configs) down to the `@gadget/ai` package (which cannot read configs directly). + +```typescript +export interface IAiEnvironment { + NODE_ENV: string; + services?: { + google?: { + cse?: { + apiKey?: string; + engineId?: string; + }; + }; + github?: { + token?: string; + }; + slack?: { + token?: string; + signingSecret?: string; + }; + [key: string]: unknown; + }; +} +``` + +**Key Design Decisions:** + +- **Services Object**: Credentials are organized by service (google, github, slack, etc.) +- **Optional Everything**: All fields are optional to support partial configurations +- **Extensible**: The `[key: string]: unknown` index signature allows future services without breaking changes +- **Type Safety**: Known services have typed interfaces + +### AiToolbox + +The toolbox manages tool registration, organization, and retrieval: + +```typescript +export class AiToolbox { + constructor(env: IAiEnvironment); + + // Register a tool for use by agents + register(tool: AiTool, modes?: string[]): void; + + // Get a tool by name (for system tools) + getTool(name: string): AiTool | undefined; + + // Get all tools for a specific mode + getModeSet(mode: string): ToolSet | undefined; + + // Access environment credentials + get env(): IAiEnvironment; +} +``` + +**Registration Modes:** + +- **System Tools** (no modes): Called by the platform itself (e.g., auto-naming chat sessions) +- **Agent Tools** (with modes): Available to AI agents in specific ChatSessionModes (e.g., "code", "research", "debug") + +### AiTool + +All tools extend this abstract base class: + +```typescript +export abstract class AiTool { + protected _toolbox: AiToolbox; + + // Unique identifier for the tool + abstract get name(): string; + + // Category for organization (search, file, code, etc.) + abstract get category(): string; + + // JSON Schema definition sent to AI provider + abstract get definition(): IToolDefinition; + + // Execute the tool with provided arguments + abstract execute(args: IToolArguments, logger: IAiLogger): Promise; +} +``` + +### IToolDefinition + +The tool definition follows the JSON Schema format expected by AI providers: + +```typescript +export interface IToolDefinition { + type: "function"; + function: { + name: string; + description: string; + parameters: IToolArguments; + }; +} + +export interface IToolArguments { + [key: string]: unknown; +} +``` + +## Tool Execution Flow + +The tool execution flow demonstrates the abstraction layer between the common tool interface and provider-specific implementations: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Application Layer (gadget-drone / gadget-code) │ +│ - Reads YAML config │ +│ - Constructs IAiEnvironment │ +│ - Creates AiToolbox │ +│ - Registers tools │ +└────────────────────┬────────────────────────────────────────┘ + │ + │ Pass tools to AI API + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ @gadget/ai - Provider Abstraction Layer │ +│ - AiApi.chat() receives tools array │ +│ - Transforms IToolDefinition → Provider format │ +│ - Sends to AI provider (Ollama/OpenAI) │ +│ - Receives tool_calls from response │ +│ - Executes tools via executeToolCalls() │ +│ - Feeds results back to AI for next iteration │ +└────────────────────┬────────────────────────────────────────┘ + │ + │ Provider-specific SDK calls + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ AI Provider (Ollama / OpenAI) │ +│ - Receives tool definitions in native format │ +│ - Decides when to call tools based on conversation │ +│ - Returns tool_calls in response │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Transformation Process + +1. **Tool Definition Transformation** + + Each provider has its own tool format. The `@gadget/ai` package transforms our common `IToolDefinition` to the provider's expected format: + + **OpenAI:** + + ```typescript + const openaiTool: ChatCompletionTool = { + type: tool.definition.type, + function: { + name: tool.definition.function.name, + description: tool.definition.function.description, + parameters: tool.definition.function.parameters, + }, + }; + ``` + + **Ollama:** + + ```typescript + const ollamaTool: Tool = { + type: tool.definition.type, + function: { + name: tool.definition.function.name, + description: tool.definition.function.description, + parameters: tool.definition.function.parameters, + }, + }; + ``` + +2. **Tool Call Execution** + + When the AI provider returns tool calls, they're executed by the base `AiApi` class: + + ```typescript + protected async executeToolCalls( + toolCalls: IToolCall[], + tools: AiTool[], + ): Promise { + const results: IToolCallResult[] = []; + + for (const toolCall of toolCalls) { + // Find the tool by name + const tool = tools.find(t => t.name === toolCall.function.name); + + // Parse arguments and execute + const args = JSON.parse(toolCall.function.arguments); + const result = await tool.execute(args, this.log); + + results.push({ + callId: toolCall.callId, + functionName: toolCall.function.name, + result, + }); + } + + return results; + } + ``` + +3. **Result Transformation** + + Tool results are converted back to the provider's message format and appended to the conversation: + + **OpenAI:** + + ```typescript + messages.push({ + role: "tool", + tool_call_id: result.callId, + content: result.error || result.result, + }); + ``` + + **Ollama:** + + ```typescript + messages.push({ + role: "tool", + content: result.error || result.result, + tool_name: result.functionName, + }); + ``` + +### Iterative Execution + +Both OpenAI and Ollama implementations support multiple rounds of tool calls: + +1. AI receives tools and conversation +2. AI returns tool calls (or final response) +3. Tools are executed, results collected +4. Results appended to conversation as tool messages +5. Loop back to step 1 (up to `maxToolIterations`, default: 5) +6. Return final response or max iterations reached + +This allows complex multi-step operations where the AI can: + +- Call multiple tools in parallel +- Use results from one tool to inform the next +- Refine its approach based on tool output + +## Configuration + +### Environment Setup + +Tools requiring credentials need them configured in your YAML config files. + +#### gadget-code.yaml + +```yaml +# Add to gadget-code.yaml +google: + cse: + apiKey: "${GOOGLE_CSE_API_KEY}" + engineId: "${GOOGLE_CSE_ENGINE_ID}" +``` + +#### gadget-drone.yaml + +```yaml +# Add to gadget-drone.yaml +google: + cse: + apiKey: "${GOOGLE_CSE_API_KEY}" + engineId: "${GOOGLE_CSE_ENGINE_ID}" +``` + +### Environment Variables + +Set the actual values in your shell or deployment environment: + +```bash +export GOOGLE_CSE_API_KEY="your-api-key-here" +export GOOGLE_CSE_ENGINE_ID="your-engine-id-here" +``` + +### TypeScript Usage + +In consumer applications, construct the environment and pass to the AI API: + +```typescript +// gadget-drone/src/services/ai.ts +import { createAiApi, IAiEnvironment } from "@gadget/ai"; +import env from "../config/env.js"; + +const aiEnv: IAiEnvironment = { + NODE_ENV: env.NODE_ENV, + services: { + google: { + cse: { + apiKey: env.google.cse.apiKey, + engineId: env.google.cse.engineId, + }, + }, + }, +}; + +const api = createAiApi(aiEnv, providerConfig, logger); +``` + +## Example Tool: Google Search + +The `GoogleSearchTool` demonstrates a complete tool implementation. + +### Overview + +**Purpose**: Perform Google Custom Search Engine queries to find relevant web content. + +**Category**: `search` + +**Credentials Required**: + +- Google CSE API Key +- Google CSE Engine ID + +### Tool Definition + +```typescript +{ + type: "function", + function: { + name: "search_google", + description: "Perform a Google search for relevant information on the web.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query string.", + }, + num_results: { + type: "number", + description: "Number of search results to return (default: 10, max: 10).", + }, + siteSearch: { + type: "string", + description: "Optional site to restrict the search to (e.g. github.com).", + }, + dateRestrict: { + type: "string", + description: "Restrict results by date range (d1, d7, d30, d365).", + }, + fileType: { + type: "string", + description: "Restrict results to file type (pdf, doc, xls, ppt).", + }, + sort: { + type: "string", + enum: ["relevance", "date"], + description: "Sort order for results.", + }, + start: { + type: "number", + description: "Starting index for pagination (default: 1).", + }, + }, + required: ["query"], + }, + } +} +``` + +### Usage Examples + +**Basic Search:** + +``` +AI Agent: "Let me search for the latest TypeScript 5.0 features." +Tool Call: search_google({ query: "TypeScript 5.0 new features" }) +``` + +**Site-Restricted Search:** + +``` +AI Agent: "I'll search the React documentation for useEffect best practices." +Tool Call: search_google({ + query: "useEffect best practices", + siteSearch: "react.dev" +}) +``` + +**Date-Restricted Search:** + +``` +AI Agent: "Let me find recent news about AI regulation from the past week." +Tool Call: search_google({ + query: "AI regulation news", + dateRestrict: "d7", + sort: "date" +}) +``` + +**File Type Search:** + +``` +AI Agent: "I'll search for Python style guide PDFs." +Tool Call: search_google({ + query: "Python style guide", + fileType: "pdf" +}) +``` + +### Implementation Details + +**File**: `packages/ai/src/tools/search/google.ts` + +**Key Methods:** + +1. **`execute(args, logger)`**: Validates input, calls search, formats results +2. **`search(query, options)`**: Makes the Google CSE API call +3. **`parseCseError(error)`**: Converts API errors to formatted tool errors + +**Error Handling:** + +The tool provides detailed error messages for common issues: + +- **401 Unauthorized**: Invalid API key +- **403 Forbidden**: Engine ID or permission issues +- **429 Rate Limited**: Quota exceeded with retry-after hint +- **Network Errors**: Connection failures +- **Missing Parameters**: Clear guidance on required fields + +**Result Formatting:** + +Search results are formatted as human-readable text: + +``` +Here are some relevant search results I found: + +Title: TypeScript 5.0 Release Notes +Link: https://devblogs.microsoft.com/typescript/announcing-typescript-5-0/ +Source: devblogs.microsoft.com +Snippet: TypeScript 5.0 introduces decorators, const type parameters, and more... + +Title: What's New in TypeScript 5.0 +Link: https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-0.html +Snippet: A comprehensive guide to all the new features in TypeScript 5.0... +``` + +## Error Handling + +### IToolError Interface + +All tool errors follow a standardized format: + +```typescript +export interface IToolError { + code: ToolErrorCode; + message: string; + parameter?: string; + expected?: string; + example?: string; + recoveryHint?: string; +} +``` + +### Error Codes + +Available error codes in `ToolErrorCode`: + +- `MISSING_PARAMETER` - Required parameter not provided +- `INVALID_PARAMETER` - Parameter value is invalid +- `NOT_FOUND` - Resource not found +- `PERMISSION_DENIED` - Insufficient permissions +- `OPERATION_FAILED` - General operation failure +- `OPERATION_NOT_ALLOWED` - Operation not permitted +- `VALIDATION_ERROR` - Input validation failed +- `RATE_LIMITED` - Rate limit exceeded +- `LIMIT_EXCEEDED` - Quota or limit exceeded +- `INVALID_CRON_SPEC` - Invalid cron expression +- `INVALID_OPERATION` - Operation not supported +- `TIMEOUT` - Operation timed out +- `INVALID_TOOL_ARGUMENTS` - Tool arguments invalid +- `SUBAGENT_FAILED` - Subagent execution failed +- `TOOL_EXECUTION_FAILED` - General tool execution error +- `UNAUTHORIZED` - Authentication required +- `FORBIDDEN` - Access denied +- `RATE_LIMIT_EXCEEDED` - Rate limit exceeded +- `NETWORK_ERROR` - Network connectivity issue +- `SECURITY_VIOLATION` - Security policy violation + +### formatError Function + +The `formatError` function creates consistent error messages: + +```typescript +const error: IToolError = { + code: "MISSING_PARAMETER", + message: "The 'query' parameter is required.", + parameter: "query", + expected: "A non-empty string containing the search query.", + example: 'search_google(query: "latest AI news")', + recoveryHint: + "Provide a 'query' parameter with your search terms and try again.", +}; + +return formatError(error); +``` + +**Output:** + +``` +TOOL ERROR: MISSING_PARAMETER +The 'query' parameter is required. +PARAMETER: query +EXPECTED: A non-empty string containing the search query. +EXAMPLE: search_google(query: "latest AI news") +RECOVERY HINT: Provide a 'query' parameter with your search terms and try again. +``` + +## Creating New Tools + +### Step 1: Create Tool Class + +Extend `AiTool` and implement the required methods: + +```typescript +import { AiTool, IToolArguments, IToolDefinition } from "../tool.js"; +import { IAiLogger } from "../../api.js"; +import { formatError } from "../tool-error.js"; + +export class MyNewTool extends AiTool { + get name(): string { + return "my_tool_name"; + } + + get category(): string { + return "category_name"; // e.g., "search", "file", "code" + } + + get definition(): IToolDefinition { + return { + type: "function", + function: { + name: this.name, + description: "Clear description of what the tool does.", + parameters: { + type: "object", + properties: { + param1: { + type: "string", + description: "Description of param1", + }, + param2: { + type: "number", + description: "Description of param2", + }, + }, + required: ["param1"], + }, + }, + }; + } + + async execute(args: IToolArguments, logger: IAiLogger): Promise { + // Validate parameters + if (!args.param1) { + return formatError({ + code: "MISSING_PARAMETER", + message: "param1 is required", + parameter: "param1", + }); + } + + // Access credentials if needed + const apiKey = this.toolbox.env.services?.myService?.apiKey; + if (!apiKey) { + throw new Error("API key not configured in environment"); + } + + // Perform the operation + logger.debug("executing my tool", { args }); + + try { + const result = await this.doSomething(args.param1); + return `Operation successful: ${result}`; + } catch (error) { + return formatError({ + code: "OPERATION_FAILED", + message: (error as Error).message, + }); + } + } + + private async doSomething(param1: string): Promise { + // Implementation here + return "result"; + } +} +``` + +### Step 2: Add Credentials (if needed) + +Update `IAiEnvironment` in `packages/ai/src/config/env.ts`: + +```typescript +export interface IAiEnvironment { + NODE_ENV: string; + services?: { + google?: { + cse?: { + apiKey?: string; + engineId?: string; + }; + }; + myService?: { + apiKey?: string; + endpoint?: string; + }; + [key: string]: unknown; + }; +} +``` + +Update config types in `packages/config/src/types.ts`: + +```typescript +export interface GadgetDroneConfig { + // ... existing fields + myService?: { + apiKey: string; + endpoint: string; + }; +} +``` + +Update consumer config readers to populate the environment. + +### Step 3: Register the Tool + +In gadget-drone startup code (to be implemented): + +```typescript +import { AiToolbox } from "@gadget/ai"; +import { MyNewTool } from "@gadget/ai/tools/my-tool.js"; + +const toolbox = new AiToolbox(aiEnv); + +// Register as system tool (no modes) +toolbox.register(new MyNewTool()); + +// Or register for specific modes +toolbox.register(new MyNewTool(), ["code", "debug"]); +``` + +## Testing Tools + +### Unit Tests + +Test tool validation and execution: + +```typescript +import { describe, it, expect } from "vitest"; +import { GoogleSearchTool } from "../src/tools/search/google.js"; +import { AiToolbox } from "../src/toolbox.js"; +import { createEmptyEnvironment } from "../src/config/env.js"; + +describe("GoogleSearchTool", () => { + const env = createEmptyEnvironment(); + const toolbox = new AiToolbox(env); + const tool = new GoogleSearchTool(toolbox); + + it("should have correct name", () => { + expect(tool.name).toBe("search_google"); + }); + + it("should have correct category", () => { + expect(tool.category).toBe("search"); + }); + + it("should have query in required parameters", () => { + const params = tool.definition.function.parameters; + expect(params.required).toContain("query"); + }); + + it("should return error for missing query", async () => { + const result = await tool.execute( + {}, + { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} }, + ); + expect(result).toContain("TOOL ERROR: MISSING_PARAMETER"); + expect(result).toContain("query"); + }); +}); +``` + +### Integration Tests + +Test with actual credentials (use mock API): + +```typescript +import { describe, it, expect, vi } from "vitest"; +import { GoogleSearchTool } from "../src/tools/search/google.js"; +import { AiToolbox } from "../src/toolbox.js"; + +describe("GoogleSearchTool integration", () => { + it("should call Google CSE API with correct parameters", async () => { + const env = { + NODE_ENV: "test", + services: { + google: { + cse: { + apiKey: "test-key", + engineId: "test-engine", + }, + }, + }, + }; + + const toolbox = new AiToolbox(env); + const tool = new GoogleSearchTool(toolbox); + + // Mock the googleapis client + // ... mock setup ... + + const result = await tool.execute( + { query: "test query", num_results: 5 }, + logger, + ); + + expect(result).toContain("search results"); + // Verify API was called with correct params + }); +}); +``` + +## Best Practices + +### 1. Validate Early, Validate Often + +Check all required parameters at the start of `execute()`: + +```typescript +async execute(args: IToolArguments, logger: IAiLogger): Promise { + if (!args.query || typeof args.query !== "string") { + return formatError({ + code: "INVALID_PARAMETER", + message: "query must be a string", + parameter: "query", + expected: "A string containing the search query", + }); + } + // ... rest of implementation +} +``` + +### 2. Use Structured Errors + +Always use `formatError()` for consistent error reporting: + +```typescript +// ✅ Good +return formatError({ + code: "NOT_FOUND", + message: "File not found", + parameter: "path", + recoveryHint: "Check that the file path exists and is accessible", +}); + +// ❌ Bad +throw new Error("File not found"); +``` + +### 3. Log Appropriately + +Use the logger for debugging and auditing: + +```typescript +logger.debug("starting tool execution", { args }); +logger.info("tool completed successfully", { resultLength: result.length }); +logger.warn("rate limit approaching", { remaining: 10 }); +logger.error("tool failed", { error: error.message }); +``` + +### 4. Handle Credentials Safely + +Never log credentials or include them in error messages: + +```typescript +// ✅ Good +if (!apiKey) { + throw new Error("API key not configured"); +} + +// ❌ Bad - exposes credential +logger.debug("using API key", { apiKey }); +``` + +### 5. Document Thoroughly + +Include in your tool: + +- Clear description in the definition +- Parameter descriptions with examples +- Error scenarios and recovery hints +- Usage examples in comments + +## Future Enhancements + +Planned improvements to the toolbox system: + +1. **Parallel Execution**: Execute independent tool calls in parallel +2. **Caching**: Cache tool results for repeated queries +3. **Rate Limiting**: Built-in rate limiting per tool +4. **Tool Metadata**: Version, author, usage statistics +5. **Result Streaming**: Stream large results back to AI + +## Related Documentation + +- [Configuration Guide](./configuration.md) - Setting up tool credentials +- [Architecture Overview](./architecture.md) - System architecture +- [AI API Reference](../packages/ai/README.md) - `@gadget/ai` package documentation diff --git a/docs/architecture.md b/docs/architecture.md index ed4a7f1..6fc2788 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -535,7 +535,7 @@ pnpm dev # Terminal 3: Drone (separate workspace directory) cd ~/my-gadget-workspace -pnpm --filter @gadget/drone dev +pnpm --filter gadget-drone dev # Registers with platform, waits for work orders ``` diff --git a/docs/configuration.md b/docs/configuration.md index c4d2d86..eedf611 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -258,7 +258,7 @@ For production deployment, the packages would be published to npm and installed ```bash # Install from npm (when published) -npm install -g gadget-code @gadget/drone +npm install -g gadget-code gadget-drone # Run from anywhere gadget-code-web diff --git a/gadget-code/src/web-cli.ts b/gadget-code/src/web-cli.ts index 877964d..c66a5f6 100644 --- a/gadget-code/src/web-cli.ts +++ b/gadget-code/src/web-cli.ts @@ -5,10 +5,12 @@ import env from "./config/env.js"; const aiEnv: IAiEnvironment = { NODE_ENV: env.NODE_ENV || "develop", - google: { - cse: { - apiKey: env.google.cse.apiKey, - engineId: env.google.cse.engineId, + services: { + google: { + cse: { + apiKey: env.google.cse.apiKey, + engineId: env.google.cse.engineId, + }, }, }, }; diff --git a/gadget-drone/package.json b/gadget-drone/package.json index 702bfd6..224e35a 100644 --- a/gadget-drone/package.json +++ b/gadget-drone/package.json @@ -1,5 +1,5 @@ { - "name": "@gadget/drone", + "name": "gadget-drone", "version": "1.0.0", "description": "Gadget Code drone process", "type": "module", diff --git a/gadget-drone/src/services/agent.ts b/gadget-drone/src/services/agent.ts index a650dab..a1b48de 100644 --- a/gadget-drone/src/services/agent.ts +++ b/gadget-drone/src/services/agent.ts @@ -2,10 +2,14 @@ // Copyright (C) 2026 Rob Colbert // Licensed under the Apache License, Version 2.0 -import { Types } from "@gadget/api"; +import env from "../config/env.ts"; +import assert from "node:assert"; + import { Socket } from "socket.io-client"; import { + GoogleSearchTool, IAiChatOptions, + IAiEnvironment, IAiStreamChunk, type IContextChatMessage, } from "@gadget/ai"; @@ -16,11 +20,13 @@ import { IUser, ServerToClientEvents, ClientToServerEvents, + ChatSessionMode, } from "@gadget/api"; import AiService from "./ai.ts"; import { GadgetService } from "../lib/service.ts"; +import { AiToolbox } from "../../../packages/ai/dist/toolbox.js"; export interface IToolCall { name: string; @@ -44,6 +50,8 @@ interface IAgentWorkflow { type DroneSocket = Socket; class AgentService extends GadgetService { + private toolbox: AiToolbox | undefined; + get name(): string { return "AgentService"; } @@ -52,6 +60,7 @@ class AgentService extends GadgetService { } async start(): Promise { + this.createAgentToolbox(); this.log.info("started"); } @@ -63,6 +72,8 @@ class AgentService extends GadgetService { workOrder: IAgentWorkOrder, socket: DroneSocket, ): Promise { + assert(this.toolbox, "service uninitialized"); + const { turn } = workOrder; const task: IAgentWorkflow = { chatOptions: {}, @@ -83,6 +94,7 @@ class AgentService extends GadgetService { systemPrompt: turn.prompts.system, context: task.context, userPrompt: turn.prompts.user, + tools: Array.from(this.toolbox.getModeSet(turn.mode) || []), }; } catch (cause) { socket.emit( @@ -235,6 +247,30 @@ class AgentService extends GadgetService { pruneSessionContext(messages: IContextChatMessage[]): void { // TODO } + + createAgentToolbox(): void { + const aiEnv: IAiEnvironment = { + NODE_ENV: env.NODE_ENV || "develop", + services: { + google: { + cse: { + apiKey: env.google.cse.apiKey, + engineId: env.google.cse.engineId, + }, + }, + }, + }; + this.toolbox = new AiToolbox(aiEnv); + + let tool = new GoogleSearchTool(this.toolbox); + this.toolbox.register(tool, [ + ChatSessionMode.Plan, + ChatSessionMode.Build, + ChatSessionMode.Test, + ChatSessionMode.Ship, + ChatSessionMode.Develop, + ]); + } } export default new AgentService(); diff --git a/gadget-drone/src/services/ai.ts b/gadget-drone/src/services/ai.ts index eb73a26..e675ab1 100644 --- a/gadget-drone/src/services/ai.ts +++ b/gadget-drone/src/services/ai.ts @@ -130,10 +130,12 @@ class AiService extends GadgetService { getApi(provider: AiProviderConfig) { const aiEnv: IAiEnvironment = { NODE_ENV: env.NODE_ENV, - google: { - cse: { - apiKey: env.google.cse.apiKey, - engineId: env.google.cse.engineId, + services: { + google: { + cse: { + apiKey: env.google.cse.apiKey, + engineId: env.google.cse.engineId, + }, }, }, }; diff --git a/packages/ai/package.json b/packages/ai/package.json index c0a3e3b..f0ca668 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -13,7 +13,10 @@ }, "scripts": { "build": "tsc", - "dev": "tsc --watch" + "dev": "tsc --watch", + "clean": "rm -rf dist/", + "typecheck": "tsc --noEmit", + "test": "echo \"No tests configured yet\"" }, "keywords": [ "gadget", diff --git a/packages/ai/src/api.ts b/packages/ai/src/api.ts index 1a0151d..874d2a8 100644 --- a/packages/ai/src/api.ts +++ b/packages/ai/src/api.ts @@ -62,13 +62,6 @@ export interface IContextChatMessage { }; } -export interface IAiChatOptions { - systemPrompt?: string; - userPrompt?: string; - context?: IContextChatMessage[]; - tools?: AiTool[]; -} - export interface IToolCall { callId: string; function: { @@ -77,6 +70,21 @@ export interface IToolCall { }; } +export interface IToolCallResult { + callId: string; + functionName: string; + result: string; + error?: string; +} + +export interface IAiChatOptions { + systemPrompt?: string; + userPrompt?: string; + context?: IContextChatMessage[]; + tools?: AiTool[]; + maxToolIterations?: number; +} + export interface IAiChatResponse { response: string; thinking?: string; @@ -84,6 +92,7 @@ export interface IAiChatResponse { done: boolean; doneReason?: string; toolCalls?: IToolCall[]; + toolCallResults?: IToolCallResult[]; } export interface IAiStreamChunk { @@ -161,4 +170,49 @@ export abstract class AiApi { options: IAiChatOptions, streamCallback?: IAiResponseStreamFn, ): Promise; + + protected async executeToolCalls( + toolCalls: IToolCall[], + tools: AiTool[], + ): Promise { + const results: IToolCallResult[] = []; + + for (const toolCall of toolCalls) { + const tool = tools.find((t) => t.name === toolCall.function.name); + + if (!tool) { + this.log.warn(`tool not found: ${toolCall.function.name}`); + results.push({ + callId: toolCall.callId, + functionName: toolCall.function.name, + result: "", + error: `Tool '${toolCall.function.name}' not found`, + }); + continue; + } + + try { + const args = JSON.parse(toolCall.function.arguments); + const result = await tool.execute(args, this.log); + results.push({ + callId: toolCall.callId, + functionName: toolCall.function.name, + result, + }); + } catch (error) { + const errorMessage = (error as Error).message; + this.log.error(`tool execution failed: ${toolCall.function.name}`, { + error: errorMessage, + }); + results.push({ + callId: toolCall.callId, + functionName: toolCall.function.name, + result: "", + error: errorMessage, + }); + } + } + + return results; + } } diff --git a/packages/ai/src/config/env.ts b/packages/ai/src/config/env.ts index 7e1057d..d2ffb4c 100644 --- a/packages/ai/src/config/env.ts +++ b/packages/ai/src/config/env.ts @@ -3,10 +3,27 @@ export interface IAiEnvironment { NODE_ENV: string; - google: { - cse: { - apiKey: string | undefined; - engineId: string | undefined; + services?: { + google?: { + cse?: { + apiKey?: string; + engineId?: string; + }; }; + github?: { + token?: string; + }; + slack?: { + token?: string; + signingSecret?: string; + }; + [key: string]: unknown; + }; +} + +export function createEmptyEnvironment(): IAiEnvironment { + return { + NODE_ENV: "development", + services: {}, }; } diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index e4492bd..cb0dac0 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -23,6 +23,8 @@ export { type IAiModelProbeResult, } from "./api.js"; +export * from "./tools/search/google.ts"; + export { OllamaAiApi } from "./ollama.js"; export { OpenAiApi } from "./openai.js"; diff --git a/packages/ai/src/ollama.ts b/packages/ai/src/ollama.ts index 4d44828..f88b9f2 100644 --- a/packages/ai/src/ollama.ts +++ b/packages/ai/src/ollama.ts @@ -10,6 +10,7 @@ import { AiApi, IAiChatOptions, IAiChatResponse, + IToolCallResult, IAiGenerateOptions, IAiGenerateResponse, IAiLogger, @@ -20,6 +21,7 @@ import { IAiResponseStreamFn, } from "./api.js"; import { IAiEnvironment } from "./config/env.ts"; +import type { Message as OllamaMessage } from "ollama"; export class OllamaAiApi extends AiApi { protected client: Ollama; @@ -195,33 +197,118 @@ export class OllamaAiApi extends AiApi { modelId: model.modelId, }); - const response = await this.client.chat({ - model: model.modelId, - messages: options.context, - stream: true, - think: model.params.reasoning, - }); + const maxIterations = options.maxToolIterations ?? 5; + let iteration = 0; - let lastChunk; - for await (const chunk of response) { - await this.log.debug("stream chunk received", { chunk }); - lastChunk = chunk; + const messages: OllamaMessage[] = options.context + ? options.context.map((msg) => ({ + role: msg.role, + content: msg.content, + })) + : []; + const allToolCallResults: IToolCallResult[] = []; + + while (iteration < maxIterations) { + iteration++; + + const ollamaTools = options.tools + ? options.tools.map((tool) => ({ + type: tool.definition.type, + function: { + name: tool.definition.function.name, + description: tool.definition.function.description, + parameters: tool.definition.function.parameters, + }, + })) + : undefined; + + const response = await this.client.chat({ + model: model.modelId, + messages, + stream: true, + think: model.params.reasoning, + tools: ollamaTools, + }); + + let lastChunk; + for await (const chunk of response) { + await this.log.debug("stream chunk received", { chunk }); + lastChunk = chunk; + } + assert(lastChunk, "no response chunks received"); + + const toolCalls = lastChunk.message.tool_calls?.map((tc) => ({ + callId: `tool_${tc.function.name}_${Date.now()}`, + function: { + name: tc.function.name, + arguments: JSON.stringify(tc.function.arguments), + }, + })); + + if (!toolCalls || toolCalls.length === 0) { + return { + done: lastChunk.done, + doneReason: lastChunk.done_reason, + response: lastChunk.message.content, + thinking: lastChunk.message.thinking, + toolCalls: undefined, + toolCallResults: allToolCallResults.length > 0 ? allToolCallResults : undefined, + stats: { + duration: { + seconds: lastChunk.total_duration, + text: numeral(lastChunk.total_duration).format("hh:mm:ss"), + }, + tokenCounts: { + input: lastChunk.prompt_eval_count, + response: lastChunk.eval_count, + thinking: 0, + }, + }, + }; + } + + const toolCallResults = await this.executeToolCalls( + toolCalls, + options.tools || [], + ); + allToolCallResults.push(...toolCallResults); + + const assistantMsg: OllamaMessage = { + role: "assistant", + content: lastChunk.message.content, + }; + if (lastChunk.message.thinking) { + assistantMsg.thinking = lastChunk.message.thinking; + } + if (lastChunk.message.tool_calls) { + assistantMsg.tool_calls = lastChunk.message.tool_calls; + } + messages.push(assistantMsg); + + for (const result of toolCallResults) { + messages.push({ + role: "tool", + content: result.error || result.result, + tool_name: result.functionName, + }); + } } - assert(lastChunk, "no response chunks received"); return { - done: lastChunk.done, - doneReason: lastChunk.done_reason, - response: lastChunk.message.content, - thinking: lastChunk.message.thinking, + done: false, + doneReason: "max_tool_iterations_reached", + response: "", + thinking: undefined, + toolCalls: undefined, + toolCallResults: allToolCallResults, stats: { duration: { - seconds: lastChunk.total_duration, - text: numeral(lastChunk.total_duration).format("hh:mm:ss"), + seconds: 0, + text: "00:00:00", }, tokenCounts: { - input: lastChunk.prompt_eval_count, - response: lastChunk.eval_count, + input: 0, + response: 0, thinking: 0, }, }, diff --git a/packages/ai/src/openai.ts b/packages/ai/src/openai.ts index 304b5ba..095df7b 100644 --- a/packages/ai/src/openai.ts +++ b/packages/ai/src/openai.ts @@ -8,6 +8,7 @@ import { AiApi, IAiChatOptions, IAiChatResponse, + IToolCallResult, IAiGenerateOptions, IAiGenerateResponse, IAiLogger, @@ -18,8 +19,11 @@ import { IAiResponseStreamFn, } from "./api.js"; import { + ChatCompletionAssistantMessageParam, ChatCompletionFunctionTool, + ChatCompletionMessageParam, ChatCompletionTool, + ChatCompletionToolMessageParam, } from "openai/resources"; import { IAiEnvironment } from "./config/env.ts"; @@ -233,10 +237,12 @@ export class OpenAiApi extends AiApi { }); const startTime = Date.now(); + const maxIterations = options.maxToolIterations ?? 5; + let iteration = 0; - const messages = []; + const messages: ChatCompletionMessageParam[] = []; if (options.systemPrompt) { - messages.push({ role: "system" as const, content: options.systemPrompt }); + messages.push({ role: "system", content: options.systemPrompt }); } if (options.context) { for (const msg of options.context) { @@ -247,57 +253,113 @@ export class OpenAiApi extends AiApi { } } if (options.userPrompt) { - messages.push({ role: "user" as const, content: options.userPrompt }); + messages.push({ role: "user", content: options.userPrompt }); } - const tools: ChatCompletionTool[] = options.tools - ? options.tools.map((tool) => { - const openaiTool: ChatCompletionFunctionTool = { - type: tool.definition.type, - function: { - name: tool.definition.function.name, - description: tool.definition.function.description, - parameters: tool.definition.function.parameters, + const allToolCallResults: IToolCallResult[] = []; + + while (iteration < maxIterations) { + iteration++; + + const tools: ChatCompletionTool[] = options.tools + ? options.tools.map((tool) => { + const openaiTool: ChatCompletionFunctionTool = { + type: tool.definition.type, + function: { + name: tool.definition.function.name, + description: tool.definition.function.description, + parameters: tool.definition.function.parameters, + }, + }; + return openaiTool; + }) + : []; + + const response = await this.client.chat.completions.create({ + model: model.modelId, + messages, + tools, + stream: false, + }); + + const choice = response.choices[0]; + const endTime = Date.now(); + const durationMs = endTime - startTime; + + const toolCalls = choice.message.tool_calls + ?.filter((tc) => tc.type === "function") + .map((tc) => ({ + callId: tc.id, + function: { + name: tc.function.name, + arguments: tc.function.arguments, + }, + })); + + if (!toolCalls || toolCalls.length === 0) { + return { + done: true, + response: choice.message.content || "", + thinking: undefined, + toolCalls: undefined, + toolCallResults: allToolCallResults.length > 0 ? allToolCallResults : undefined, + stats: { + duration: { + seconds: durationMs / 1000, + text: numeral(durationMs / 1000).format("hh:mm:ss"), }, - }; - return openaiTool; - }) - : []; + tokenCounts: { + input: response.usage?.prompt_tokens || 0, + response: response.usage?.completion_tokens || 0, + thinking: 0, + }, + }, + }; + } - const response = await this.client.chat.completions.create({ - model: model.modelId, - messages, - tools, - stream: false, - }); + const toolCallResults = await this.executeToolCalls( + toolCalls, + options.tools || [], + ); + allToolCallResults.push(...toolCallResults); + + const assistantMsg: ChatCompletionAssistantMessageParam = { + role: "assistant", + content: choice.message.content, + }; + if (choice.message.tool_calls) { + assistantMsg.tool_calls = choice.message.tool_calls; + } + messages.push(assistantMsg); + + for (const result of toolCallResults) { + const toolMsg: ChatCompletionToolMessageParam = { + role: "tool", + tool_call_id: result.callId, + content: result.error || result.result, + }; + messages.push(toolMsg); + } + } - const choice = response.choices[0]; const endTime = Date.now(); const durationMs = endTime - startTime; - const toolCalls = choice.message.tool_calls - ?.filter((tc) => tc.type === "function") - .map((tc) => ({ - callId: tc.id, - function: { - name: tc.function.name, - arguments: tc.function.arguments, - }, - })); - return { - done: true, - response: choice.message.content || "", + done: false, + doneReason: "max_tool_iterations_reached", + response: "", thinking: undefined, - toolCalls, + toolCalls: undefined, + toolCallResults: allToolCallResults, stats: { duration: { seconds: durationMs / 1000, text: numeral(durationMs / 1000).format("hh:mm:ss"), }, tokenCounts: { - input: response.usage?.prompt_tokens || 0, - response: response.usage?.completion_tokens || 0, + input: 0, + response: 0, thinking: 0, }, }, diff --git a/packages/ai/src/tools/search/google.ts b/packages/ai/src/tools/search/google.ts index b8584fd..6c11894 100644 --- a/packages/ai/src/tools/search/google.ts +++ b/packages/ai/src/tools/search/google.ts @@ -165,14 +165,23 @@ export class GoogleSearchTool extends AiTool { query: string, options: ISearchOptions, ): Promise { + const apiKey = this.toolbox.env.services?.google?.cse?.apiKey; + const engineId = this.toolbox.env.services?.google?.cse?.engineId; + + if (!apiKey || !engineId) { + throw new Error( + "Google CSE credentials not configured in environment", + ); + } + const customSearch = google.customsearch({ version: "v1", - auth: this.toolbox.env.google.cse.apiKey, + auth: apiKey, }); const params: any = { q: query, - cx: this.toolbox.env.google.cse.engineId, + cx: engineId, num: options.num || 10, }; @@ -229,9 +238,9 @@ export class GoogleSearchTool extends AiTool { const apiError = error.response.data.error; const statusCode = error.response.status; - switch (statusCode) { + switch (statusCode) { case 401: - formatError({ + return formatError({ code: "UNAUTHORIZED", message: `401 Unauthorized - Invalid API key. ${apiError.message || ""}`, });