From f35a0ce92162adced1d6c5b14779bfc119818cee Mon Sep 17 00:00:00 2001 From: Rob Colbert Date: Tue, 5 May 2026 05:28:09 -0400 Subject: [PATCH] a lot of review and by-hand cleanup (wip) --- .../src/controllers/api/v1/chat-session.ts | 93 ++++++------------- gadget-code/src/controllers/lib/populators.ts | 31 +++++++ gadget-code/src/models/api-client.ts | 16 +++- gadget-code/src/services/api-client.ts | 47 +++++++++- gadget-code/src/services/chat-session.ts | 48 +++++++++- gadget-code/src/web-app.ts | 3 + gadget-code/src/web-cli.ts | 61 +++++++++--- gadget-drone/src/gadget-drone.ts | 15 +-- gadget-drone/src/services/platform.ts | 44 ++++++++- 9 files changed, 264 insertions(+), 94 deletions(-) diff --git a/gadget-code/src/controllers/api/v1/chat-session.ts b/gadget-code/src/controllers/api/v1/chat-session.ts index 7b15048..4d14ef8 100644 --- a/gadget-code/src/controllers/api/v1/chat-session.ts +++ b/gadget-code/src/controllers/api/v1/chat-session.ts @@ -7,6 +7,7 @@ import { Request, Response } from "express"; import { DtpController } from "../../../lib/controller.js"; import ChatSessionService from "../../../services/chat-session.js"; import { ChatSessionMode } from "@gadget/api"; +import { populateChatSessionById } from "@/controllers/lib/populators.js"; class ChatSessionController extends DtpController { get name(): string { @@ -20,20 +21,19 @@ class ChatSessionController extends DtpController { } async start(): Promise { - this.router.get("/", this.requireUser(), this.listSessions.bind(this)); - this.router.post("/", this.requireUser(), this.createSession.bind(this)); - this.router.get("/:id", this.requireUser(), this.getSession.bind(this)); - this.router.put("/:id", this.requireUser(), this.updateSession.bind(this)); - this.router.delete( - "/:id", - this.requireUser(), - this.deleteSession.bind(this), - ); - this.router.get( - "/:id/turns", - this.requireUser(), - this.getSessionTurns.bind(this), - ); + this.router.use(this.requireUser()); + + this.router.param("sessionId", populateChatSessionById(this)); + + this.router.post("/", this.createSession.bind(this)); + + this.router.get("/:sessionId/turns", this.getSessionTurns.bind(this)); + this.router.get("/:sessionId", this.getSession.bind(this)); + this.router.get("/", this.listSessions.bind(this)); + + this.router.put("/:sessionId", this.updateSession.bind(this)); + + this.router.delete("/:sessionId", this.deleteSession.bind(this)); } /** @@ -140,24 +140,11 @@ class ChatSessionController extends DtpController { * GET /api/v1/chat-sessions/:id * Get a specific chat session. */ - private async getSession(req: Request, res: Response): Promise { + private async getSession(_req: Request, res: Response): Promise { try { - const id = Array.isArray(req.params.id) - ? req.params.id[0] - : req.params.id; - if (!id) { - res.status(400).json({ - success: false, - message: "Session ID is required", - }); - return; - } - - const session = await ChatSessionService.getById(id); - res.json({ success: true, - data: session, + data: res.locals.chatSession, }); } catch (error) { const err = error as Error; @@ -183,16 +170,6 @@ class ChatSessionController extends DtpController { */ private async updateSession(req: Request, res: Response): Promise { try { - const id = Array.isArray(req.params.id) - ? req.params.id[0] - : req.params.id; - if (!id) { - res.status(400).json({ - success: false, - message: "Session ID is required", - }); - return; - } const updates = req.body; // Validate allowed updates @@ -217,7 +194,10 @@ class ChatSessionController extends DtpController { ChatSessionMode[updates.mode as keyof typeof ChatSessionMode]; } - const session = await ChatSessionService.update(id, allowedUpdates); + const session = await ChatSessionService.update( + res.locals.chatSession._id, + allowedUpdates, + ); res.json({ success: true, @@ -245,21 +225,9 @@ class ChatSessionController extends DtpController { * DELETE /api/v1/chat-sessions/:id * Delete a chat session. */ - private async deleteSession(req: Request, res: Response): Promise { + private async deleteSession(_req: Request, res: Response): Promise { try { - const id = Array.isArray(req.params.id) - ? req.params.id[0] - : req.params.id; - if (!id) { - res.status(400).json({ - success: false, - message: "Session ID is required", - }); - return; - } - - await ChatSessionService.delete(id); - + await ChatSessionService.delete(res.locals.chatSession._id); res.json({ success: true, message: "Chat session deleted", @@ -286,20 +254,11 @@ class ChatSessionController extends DtpController { * GET /api/v1/chat-sessions/:id/turns * Get all turns for a chat session. */ - private async getSessionTurns(req: Request, res: Response): Promise { + private async getSessionTurns(_req: Request, res: Response): Promise { try { - const id = Array.isArray(req.params.id) - ? req.params.id[0] - : req.params.id; - if (!id) { - res.status(400).json({ - success: false, - message: "Session ID is required", - }); - return; - } - - const turns = await ChatSessionService.getTurns(id); + const turns = await ChatSessionService.getTurns( + res.locals.chatSession._id, + ); res.json({ success: true, diff --git a/gadget-code/src/controllers/lib/populators.ts b/gadget-code/src/controllers/lib/populators.ts index 4fe73a2..11f42a2 100644 --- a/gadget-code/src/controllers/lib/populators.ts +++ b/gadget-code/src/controllers/lib/populators.ts @@ -9,11 +9,42 @@ import { DtpController } from "../../lib/controller.ts"; import DroneService from "../../services/drone.ts"; import UserService from "../../services/user.ts"; +import { ChatSessionService } from "../../services/index.js"; export interface PopulateOptions { requireObject?: boolean; } +export function populateChatSessionById( + controller: DtpController, + options?: PopulateOptions, +): RequestHandler { + options = Object.assign({ requireObject: true }, options); + return async function ( + _req: Request, + res: Response, + next: NextFunction, + sessionId?: string, + ): Promise { + assert(sessionId, "ChatSession ID is required"); + try { + res.locals.chatSession = await ChatSessionService.getById(sessionId); + if (options.requireObject && !res.locals.chatSession) { + const error = new Error("ChatSession not found"); + error.statusCode = 404; + throw error; + } + return next(); + } catch (error) { + controller.log.error("failed to populate ChatSession by ID", { + sessionId, + error, + }); + return next(error); + } + }; +} + export function populateUserById( controller: DtpController, options?: PopulateOptions, diff --git a/gadget-code/src/models/api-client.ts b/gadget-code/src/models/api-client.ts index ff30ac9..304f4ca 100644 --- a/gadget-code/src/models/api-client.ts +++ b/gadget-code/src/models/api-client.ts @@ -4,7 +4,7 @@ import { Schema, model } from "mongoose"; -import { GadgetId } from "@gadget/api"; +import { GadgetId, IUser } from "@gadget/api"; import { nanoid } from "nanoid"; export enum ApiClientStatus { @@ -13,6 +13,18 @@ export enum ApiClientStatus { Archived = "archived", } +/** + * An API client is either machine-to-machine (no User), or a User access client + * that can call services on the User's scoped behalf. When a request presents + * an ApiClient token, we populate req.user with that User account. The + * remainder of the request proceeds with that User's authentication and + * authorization. + * + * An example of non-User communication is a gadget-drone performing an initial + * Platform registration. It presents the User's credentials to authenticate the + * request, and we verify the API key to authenticate that client before + * trusting the credentials presented. + */ export interface IApiClient { _id: GadgetId; createdAt: Date; @@ -21,6 +33,7 @@ export interface IApiClient { name: string; description?: string; secret: string; + user?: IUser | GadgetId; } const ApiClientSchema = new Schema({ _id: { type: String, default: () => nanoid() }, @@ -36,6 +49,7 @@ const ApiClientSchema = new Schema({ name: { type: String, required: true }, description: { type: String }, secret: { type: String, required: true }, + user: { type: String, ref: "User", index: 1 }, }); export const ApiClient = model("ApiClient", ApiClientSchema); diff --git a/gadget-code/src/services/api-client.ts b/gadget-code/src/services/api-client.ts index cf11b84..7449e66 100644 --- a/gadget-code/src/services/api-client.ts +++ b/gadget-code/src/services/api-client.ts @@ -5,7 +5,7 @@ // import env, { getCountryName } from "../config/env.js"; import assert from "node:assert"; -import { Request } from "express"; +import { NextFunction, Request, RequestHandler, Response } from "express"; import { filterText } from "dtp-cleantext"; import { v4 as uuidv4 } from "uuid"; @@ -18,8 +18,16 @@ import ApiClientLog, { IApiClientLog } from "../models/api-client-log.js"; import { DtpService } from "../lib/service.js"; import { GadgetId } from "@gadget/api"; +import { PopulateOptions } from "mongoose"; class ApiClientService extends DtpService { + private populateApiClient: PopulateOptions[] = [ + { + path: "user", + select: "-passwordSalt -password", + }, + ]; + get name(): string { return "ApiClientService"; } @@ -35,12 +43,45 @@ class ApiClientService extends DtpService { async stop(): Promise {} + middleware(): RequestHandler { + return async ( + req: Request, + res: Response, + next: NextFunction, + ): Promise => { + try { + const apiClientId = req.header("X-Gadget-Key") as string; + if (!apiClientId) { + return next(); + } + + const apiClient = await this.getById(apiClientId); + if (!apiClient) { + return next(); + } + + if (apiClient.user && !req.user) { + req.user = apiClient.user; + res.locals.user = apiClient.user; + } + + await this.logRequest(apiClient, req); + + return next(); + } catch (error) { + this.log.error("failed to process ApiClient request", { error }); + return next(error); + } + }; + } + async create(definition: Partial): Promise { const NOW = new Date(); const apiClient = new ApiClient(); apiClient.createdAt = NOW; apiClient.updatedAt = NOW; apiClient.status = ApiClientStatus.Active; + apiClient.user = definition.user; assert(definition.name, "ApiClient name is required"); apiClient.name = filterText(definition.name); @@ -77,7 +118,9 @@ class ApiClientService extends DtpService { } async getById(clientId: GadgetId): Promise { - const client = await ApiClient.findOne({ _id: clientId }); + const client = await ApiClient.findOne({ _id: clientId }) + .populate(this.populateApiClient) + .lean(); return client; } diff --git a/gadget-code/src/services/chat-session.ts b/gadget-code/src/services/chat-session.ts index 1245e6e..952589e 100644 --- a/gadget-code/src/services/chat-session.ts +++ b/gadget-code/src/services/chat-session.ts @@ -2,6 +2,11 @@ // Copyright (C) 2026 Robert Colbert // All Rights Reserved +import env from "../config/env.js"; + +import path from "node:path"; +import fs from "node:fs"; + import { IChatSession, ChatSessionMode, @@ -47,7 +52,7 @@ class ChatSessionService extends DtpService { }, { path: "provider", - select: "-models", + select: "-models +apiKey", }, ]; @@ -234,6 +239,8 @@ class ChatSessionService extends DtpService { const user: IUser = session.user as IUser; const project: IProject = session.project as IProject; + const systemPrompt = await this.buildSystemPrompt(session); + let turn = new ChatTurn({ createdAt: NOW, user: user._id, @@ -244,8 +251,8 @@ class ChatSessionService extends DtpService { mode: session.mode, status: ChatTurnStatus.Processing, prompts: { + system: systemPrompt, user: prompt, - system: undefined, }, toolCalls: [], subagents: [], @@ -265,6 +272,40 @@ class ChatSessionService extends DtpService { return turn; } + async buildSystemPrompt(session: IChatSession): Promise { + const commonDir = path.join(env.installDir, "data", "prompts", "common"); + const promptsDir = path.join( + env.installDir, + "data", + "prompts", + "agent", + session.mode, + ); + + const common = { + scopeBlock: await fs.promises.readFile( + path.join(commonDir, "scope-block.md"), + "utf-8", + ), + subagentsBlock: await fs.promises.readFile( + path.join(commonDir, "subagents.md"), + "utf-8", + ), + }; + + const templateFilename = path.join(promptsDir, "system.md"); + const promptTemplate = await fs.promises.readFile( + templateFilename, + "utf-8", + ); + + let prompt = promptTemplate + .replace("{{scope_block}}", common.scopeBlock) + .replace("{{subagent_section}}", common.subagentsBlock); + + return prompt; + } + /** * Gets all turns for a chat session. */ @@ -272,10 +313,9 @@ class ChatSessionService extends DtpService { const turns = await ChatTurn.find({ session: chatSessionId }) .populate("user", "-passwordSalt -password") .populate("project") - .populate("provider") + .populate("provider", "-models") .sort({ createdAt: 1 }) .lean(); - return turns; } diff --git a/gadget-code/src/web-app.ts b/gadget-code/src/web-app.ts index dbba391..4ffef73 100644 --- a/gadget-code/src/web-app.ts +++ b/gadget-code/src/web-app.ts @@ -42,6 +42,7 @@ import { HomeController } from "./controllers/home.js"; import { UserController } from "./controllers/user.js"; import { + ApiClientService, SessionService, SocketService, startServices, @@ -198,7 +199,9 @@ class DtpWebAppServer implements DtpComponent { }, store, }; + this.app.use(session(sessionConfig)); + this.app.use(ApiClientService.middleware()); this.app.use(this.restoreUserSession.bind(this)); this.app.use("/", await this.createAppRouter()); diff --git a/gadget-code/src/web-cli.ts b/gadget-code/src/web-cli.ts index b9f6511..bbbaa0e 100644 --- a/gadget-code/src/web-cli.ts +++ b/gadget-code/src/web-cli.ts @@ -10,7 +10,7 @@ import "./lib/db.js"; * Models */ -import ApiClient, { ApiClientStatus } from "./models/api-client.js"; +import ApiClient, { ApiClientStatus, IApiClient } from "./models/api-client.js"; import User from "./models/user.js"; import AiProvider from "./models/ai-provider.js"; @@ -33,6 +33,7 @@ import { import { createAiApi, type IAiLogger } from "@gadget/ai"; import { + IUser, type IAiModel, type IAiModelCapabilities, type IAiModelSettings, @@ -157,29 +158,49 @@ class DtpWebCli extends DtpProcess { const name = argv.shift(); const description = argv.shift(); - const client = await ApiClientService.create({ + const email = argv.shift(); + let user; + if (email) { + user = await UserService.getByEmail(email); + if (!user) { + throw new Error("user not found"); + } + } + + const definition: Partial = { name, description, - }); - this.log.info("api client added", { - client: { - _id: client._id, - secret: client.secret, - name: client.name, - }, - }); + }; + if (user) { + definition.user = user._id; + } + + const client = await ApiClientService.create(definition); + this.printApiClientList([client]); } async onApiClientList(_argv: string[]): Promise { - const clients = await ApiClient.find({ status: ApiClientStatus.Active }) + const clients: IApiClient[] = await ApiClient.find({ + status: ApiClientStatus.Active, + }) .sort({ name: 1 }) + .populate([{ path: "user", select: "-passwordSalt -password" }]) .lean(); + this.printApiClientList(clients); + } + + printApiClientList(clients: IApiClient[]) { console.log("Name".padEnd(20), "Client ID".padEnd(24), "Secret"); console.log( "--------------------------------------------------------------------------------", ); for (const client of clients) { - console.log(client.name.padEnd(20), client._id.toString(), client.secret); + let log = `${client.name.padEnd(20)} ${client._id.toString()} ${client.secret}`; + if (client.user) { + const user = client.user as IUser; + log += ` ${user._id} ${user.email}`; + } + console.log(log); } } @@ -223,6 +244,8 @@ class DtpWebCli extends DtpProcess { switch (action) { case "add": return this.onUserAdd(argv); + case "view": + return this.onUserView(argv); case "password": return this.onUserPassword(argv); case "remove": @@ -253,6 +276,20 @@ class DtpWebCli extends DtpProcess { this.log.info(`user created: id:${user._id}, email:${user.email}`); } + async onUserView(argv: string[]): Promise { + const email = argv.shift(); + if (!email) { + throw new Error("must specify email address"); + } + + const user = await UserService.getByEmail(email); + if (!user) { + throw new Error("user not found"); + } + + this.log.info("user account", user); + } + async onUserRemove(argv: string[]): Promise { let email = argv.shift(); if (!email) { diff --git a/gadget-drone/src/gadget-drone.ts b/gadget-drone/src/gadget-drone.ts index 7b34d27..5cbddcb 100644 --- a/gadget-drone/src/gadget-drone.ts +++ b/gadget-drone/src/gadget-drone.ts @@ -442,9 +442,16 @@ class GadgetDrone extends GadgetProcess { return cb(false, "this drone's workspace is not in Agent mode"); } + if (!this.socket) { + this.log.error("cannot process work order: no socket connection"); + cb(false, "No socket connection"); + return; + } + + const context = await PlatformService.getChatSessionContext(session); const order: IAgentWorkOrder = { createdAt: turn.createdAt, - context: [], + context: context.data, turn, }; this.log.info("processWorkOrder received", { @@ -454,12 +461,6 @@ class GadgetDrone extends GadgetProcess { turn: { _id: turn._id, mode: turn.mode, userPrompt: turn.prompts.user }, }); - if (!this.socket) { - this.log.error("cannot process work order: no socket connection"); - cb(false, "No socket connection"); - return; - } - // Write work order cache BEFORE processing (for crash recovery) try { await WorkspaceService.writeWorkOrderCache(turn); diff --git a/gadget-drone/src/services/platform.ts b/gadget-drone/src/services/platform.ts index 36e7014..67f1ff4 100644 --- a/gadget-drone/src/services/platform.ts +++ b/gadget-drone/src/services/platform.ts @@ -9,7 +9,14 @@ import path from "node:path"; import os from "node:os"; import { GadgetService } from "../lib/service.ts"; -import { DroneStatus, IDroneRegistration, IUser, Types } from "@gadget/api"; +import { + DroneStatus, + IChatSession, + IChatTurn, + IDroneRegistration, + IUser, + Types, +} from "@gadget/api"; interface PlatformApiResponse { success: boolean; @@ -20,6 +27,10 @@ interface PlatformRegistrationResponse extends PlatformApiResponse { data: IDroneRegistration; } +interface ChatSessionContextResponse extends PlatformApiResponse { + data: IChatTurn[]; +} + class PlatformService extends GadgetService { registration: IDroneRegistration | undefined; @@ -157,6 +168,37 @@ class PlatformService extends GadgetService { this.log.info("drone status updated on platform", { status }); } + async getChatSessionContext( + session: IChatSession, + ): Promise { + assert( + this.registration, + "must register with platform before setting status", + ); + + const url = this.getApiUrl(`/chat-sessions/${session._id}/turns`); + const response = await fetch(url, { + method: "GET", + headers: { + Accept: "application/json", + "X-Gadget-Key": env.platform.apiKey, + }, + }); + + const json = (await response.json()) as ChatSessionContextResponse; + if (!json.success) { + const error = new Error("failed to retrieve chat session context"); + error.name = "PlatformError"; + error.statusCode = response.status; + throw error; + } + + this.log.info("chat session context received", { + turnCount: json.data.length, + }); + return json; + } + getApiUrl(url: string): string { return `${env.platform.baseUrl}/api/v1${url}`; }