From 26e0ba5cafcd585f86edaaf40206835415b6d488 Mon Sep 17 00:00:00 2001 From: Bastian Wagner Date: Fri, 12 Jun 2026 14:02:33 +0200 Subject: [PATCH] mpc logging --- listify-api/README.md | 2 + .../assistant/assistant-chat-log.entity.ts | 46 ++++++ listify-api/src/assistant/assistant.module.ts | 4 +- .../src/assistant/assistant.service.spec.ts | 77 +++++++++- .../src/assistant/assistant.service.ts | 131 +++++++++++++++--- listify-api/src/database/data-source.ts | 2 + 6 files changed, 235 insertions(+), 27 deletions(-) create mode 100644 listify-api/src/assistant/assistant-chat-log.entity.ts diff --git a/listify-api/README.md b/listify-api/README.md index e28f9d8..f5da11c 100644 --- a/listify-api/README.md +++ b/listify-api/README.md @@ -55,6 +55,8 @@ MISTRAL_AGENT_ID=your-mistral-agent-id The Mistral agent should have the `listify` connector attached. The authenticated frontend calls `POST /api/assistant/chat`; the API forwards the conversation to `POST /v1/agents/completions` and returns the assistant text. Listify does not parse or execute tool calls locally in this path. +Every Mistral response is stored in `assistant_chat_logs`. The table includes the sanitized provider request, the full raw provider response, the extracted assistant text sent back to the UI, response status and timing metadata. + ## Run tests ```bash diff --git a/listify-api/src/assistant/assistant-chat-log.entity.ts b/listify-api/src/assistant/assistant-chat-log.entity.ts new file mode 100644 index 0000000..fb14893 --- /dev/null +++ b/listify-api/src/assistant/assistant-chat-log.entity.ts @@ -0,0 +1,46 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryColumn } from 'typeorm'; + +@Entity('assistant_chat_logs') +export class AssistantChatLogEntity { + @PrimaryColumn({ type: 'varchar', length: 36 }) + id!: string; + + @Index('IDX_assistant_chat_logs_user_id') + @Column({ type: 'varchar', length: 36 }) + userId!: string; + + @Column({ type: 'varchar', length: 120 }) + provider!: string; + + @Column({ type: 'varchar', length: 255 }) + endpoint!: string; + + @Column({ type: 'varchar', length: 255, nullable: true }) + agentId?: string | null; + + @Column({ type: 'int', nullable: true }) + statusCode?: number | null; + + @Column({ type: 'int', nullable: true }) + durationMs?: number | null; + + @Column({ type: 'json' }) + requestPayload!: Record; + + @Column({ type: 'json', nullable: true }) + responsePayload?: unknown; + + @Column({ type: 'text', nullable: true }) + assistantContent?: string | null; + + @Column({ type: 'text', nullable: true }) + errorMessage?: string | null; + + @Index('IDX_assistant_chat_logs_created_at') + @CreateDateColumn({ + type: 'datetime', + precision: 3, + default: () => 'CURRENT_TIMESTAMP(3)', + }) + createdAt!: Date; +} diff --git a/listify-api/src/assistant/assistant.module.ts b/listify-api/src/assistant/assistant.module.ts index abc03f8..7bd9356 100644 --- a/listify-api/src/assistant/assistant.module.ts +++ b/listify-api/src/assistant/assistant.module.ts @@ -1,10 +1,12 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; import { AuthModule } from '../auth/auth.module'; +import { AssistantChatLogEntity } from './assistant-chat-log.entity'; import { AssistantController } from './assistant.controller'; import { AssistantService } from './assistant.service'; @Module({ - imports: [AuthModule], + imports: [AuthModule, TypeOrmModule.forFeature([AssistantChatLogEntity])], controllers: [AssistantController], providers: [AssistantService], }) diff --git a/listify-api/src/assistant/assistant.service.spec.ts b/listify-api/src/assistant/assistant.service.spec.ts index 02401cb..8a87731 100644 --- a/listify-api/src/assistant/assistant.service.spec.ts +++ b/listify-api/src/assistant/assistant.service.spec.ts @@ -5,13 +5,21 @@ describe('AssistantService', () => { const originalFetch = global.fetch; const originalApiKey = process.env.MISTRAL_API_KEY; const originalAgentId = process.env.MISTRAL_AGENT_ID; + let chatLogsRepository: { + create: jest.Mock; + save: jest.Mock; + }; let service: AssistantService; beforeEach(() => { process.env.MISTRAL_API_KEY = 'test-key'; process.env.MISTRAL_AGENT_ID = 'agent-listify'; global.fetch = jest.fn(); - service = new AssistantService(); + chatLogsRepository = { + create: jest.fn((input) => input), + save: jest.fn(async (input) => input), + }; + service = new AssistantService(chatLogsRepository as never); }); afterEach(() => { @@ -21,7 +29,7 @@ describe('AssistantService', () => { }); it('forwards messages to the configured Mistral agent', async () => { - mockMistralResponse({ + const providerResponse = { choices: [ { message: { @@ -29,7 +37,12 @@ describe('AssistantService', () => { }, }, ], - }); + usage: { + prompt_tokens: 12, + completion_tokens: 8, + }, + }; + mockMistralResponse(providerResponse); const result = await service.chat('user-1', { messages: [ @@ -51,6 +64,13 @@ describe('AssistantService', () => { messages: [ { role: 'assistant', content: 'Hallo' }, { role: 'user', content: 'Erstelle eine Liste' }, + { role: 'system', content: 'benutze immer den listify connector' }, + ], + tools: [ + { + type: 'connector', + connector_id: 'listify', + }, ], stream: false, response_format: { @@ -66,6 +86,46 @@ describe('AssistantService', () => { }, actions: [], }); + expect(chatLogsRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ + userId: 'user-1', + provider: 'mistral', + endpoint: 'https://api.mistral.ai/v1/agents/completions', + agentId: 'agent-listify', + statusCode: 200, + requestPayload: expect.objectContaining({ + agent_id: 'agent-listify', + tools: [{ type: 'connector', connector_id: 'listify' }], + }), + responsePayload: providerResponse, + assistantContent: 'Ich habe den Listify-Connector verwendet.', + errorMessage: null, + }), + ); + }); + + it('logs full failed provider responses before throwing', async () => { + const providerResponse = { + message: 'connector failed', + details: { connector_id: 'listify' }, + }; + mockMistralResponse(providerResponse, false, 502); + + await expect( + service.chat('user-1', { + messages: [{ role: 'user', content: 'Hallo' }], + }), + ).rejects.toThrow(ServiceUnavailableException); + + expect(chatLogsRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ + userId: 'user-1', + statusCode: 502, + responsePayload: providerResponse, + assistantContent: null, + errorMessage: 'Mistral agent request failed.', + }), + ); }); it('fails clearly when the api key is missing', async () => { @@ -89,9 +149,14 @@ describe('AssistantService', () => { }); }); -function mockMistralResponse(response: object): void { +function mockMistralResponse( + response: object, + ok = true, + status = 200, +): void { jest.mocked(global.fetch).mockResolvedValue({ - ok: true, - json: async () => response, + ok, + status, + text: async () => JSON.stringify(response), } as Response); } diff --git a/listify-api/src/assistant/assistant.service.ts b/listify-api/src/assistant/assistant.service.ts index 9063364..1bcbce3 100644 --- a/listify-api/src/assistant/assistant.service.ts +++ b/listify-api/src/assistant/assistant.service.ts @@ -3,6 +3,10 @@ import { Injectable, ServiceUnavailableException, } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { randomUUID } from 'crypto'; +import { Repository } from 'typeorm'; +import { AssistantChatLogEntity } from './assistant-chat-log.entity'; import { AssistantChatMessage, AssistantChatRequest, @@ -21,12 +25,17 @@ interface MistralAgentCompletionResponse { export class AssistantService { private readonly endpoint = 'https://api.mistral.ai/v1/agents/completions'; + constructor( + @InjectRepository(AssistantChatLogEntity) + private readonly chatLogsRepository: Repository, + ) {} + async chat( - _userId: string, + userId: string, request: AssistantChatRequest, ): Promise { const messages = this.normalizeMessages(request.messages); - const response = await this.callMistralAgent(messages); + const response = await this.callMistralAgent(userId, messages); const content = response.choices?.[0]?.message?.content?.trim(); if (!content) { @@ -43,6 +52,7 @@ export class AssistantService { } private async callMistralAgent( + userId: string, messages: AssistantChatMessage[], ): Promise { const apiKey = process.env.MISTRAL_API_KEY; @@ -56,36 +66,117 @@ export class AssistantService { throw new ServiceUnavailableException('Mistral agent id is not configured.'); } + const requestPayload = { + agent_id: agentId, + messages: [ + ...messages, + { "role": "system", "content": "benutze immer den listify connector"} + ], + tools: [ + { + "type": "connector", + "connector_id": "listify" + } + ], + stream: false, + response_format: { + type: 'text', + }, + }; + const startedAt = Date.now(); + let statusCode: number | null = null; + let responsePayload: unknown = null; + let assistantContent: string | null = null; + const response = await fetch(this.endpoint, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, - body: JSON.stringify({ - agent_id: agentId, - messages: [ - ...messages, - { "role": "system", "content": "benutze immer den listify connector"} - ], - tools: [ - { - "type": "connector", - "connector_id": "listify" - } - ], - stream: false, - response_format: { - type: 'text', - }, - }), + body: JSON.stringify(requestPayload), }); + statusCode = response.status; + responsePayload = await this.readResponsePayload(response); if (!response.ok) { + await this.recordChatLog({ + userId, + agentId, + requestPayload, + responsePayload, + statusCode, + durationMs: Date.now() - startedAt, + assistantContent, + errorMessage: 'Mistral agent request failed.', + }); throw new ServiceUnavailableException('Mistral agent request failed.'); } - return (await response.json()) as MistralAgentCompletionResponse; + assistantContent = this.extractAssistantContent(responsePayload); + + await this.recordChatLog({ + userId, + agentId, + requestPayload, + responsePayload, + statusCode, + durationMs: Date.now() - startedAt, + assistantContent, + errorMessage: null, + }); + + return responsePayload as MistralAgentCompletionResponse; + } + + private async readResponsePayload(response: Response): Promise { + const rawBody = await response.text(); + + if (!rawBody) { + return null; + } + + try { + return JSON.parse(rawBody) as unknown; + } catch { + return { rawBody }; + } + } + + private extractAssistantContent(responsePayload: unknown): string | null { + if (!responsePayload || typeof responsePayload !== 'object') { + return null; + } + + const response = responsePayload as MistralAgentCompletionResponse; + return response.choices?.[0]?.message?.content?.trim() ?? null; + } + + private async recordChatLog(input: { + userId: string; + agentId: string; + requestPayload: Record; + responsePayload: unknown; + statusCode: number | null; + durationMs: number; + assistantContent: string | null; + errorMessage: string | null; + }): Promise { + await this.chatLogsRepository.save( + this.chatLogsRepository.create({ + id: randomUUID(), + userId: input.userId, + provider: 'mistral', + endpoint: this.endpoint, + agentId: input.agentId, + statusCode: input.statusCode, + durationMs: input.durationMs, + requestPayload: input.requestPayload, + responsePayload: input.responsePayload, + assistantContent: input.assistantContent, + errorMessage: input.errorMessage, + }), + ); } private normalizeMessages( diff --git a/listify-api/src/database/data-source.ts b/listify-api/src/database/data-source.ts index 0fa0ef6..b274a0f 100644 --- a/listify-api/src/database/data-source.ts +++ b/listify-api/src/database/data-source.ts @@ -1,6 +1,7 @@ import 'dotenv/config'; import 'reflect-metadata'; import { DataSource } from 'typeorm'; +import { AssistantChatLogEntity } from '../assistant/assistant-chat-log.entity'; import { AuditLogEntity } from '../audit/audit-log.entity'; import { UserEntity } from '../auth/user.entity'; import { RefreshTokenEntity } from '../auth/refresh-token.entity'; @@ -28,6 +29,7 @@ export default new DataSource({ logger: new DatabaseLogger(databaseLoggerOptionsFromEnv(process.env)), maxQueryExecutionTime: slowQueryThresholdFromEnv(process.env), entities: [ + AssistantChatLogEntity, AuditLogEntity, UserEntity, RefreshTokenEntity,