getappmap / getappmap/appmap-js
Navie history symlinks don't work on Windows, cause a Navie failure
Open
Nobody has claimed this yet.
bug
navie-plan
- Dominant language
- TypeScript
- Stars
- 53
- Forks
- 18
- Avg merge
- 3d 18h
- Merged PRs (30d)
- 8
Description
Error example
10896 [Stderr] Handling exception: {
10896 [Stderr] code: 'EPERM',
10896 [Stderr] message: "EPERM: operation not permitted, symlink '..\\messages\\bb5d3f14-b4ae-4049-a7ba-b7babf4feec8\\question.txt' -> 'C:\\Users\\dusti\\.appmap\\navie\\history\\threads\\9d435cd1-5d42-466d-b6e9-2a4ce58284c5\\sequence\\1725543743379.question.txt'",
10896 [Stderr] data: {
10896 [Stderr] data: {
10896 [Stderr] stack: "Error: EPERM: operation not permitted, symlink '..\\messages\\bb5d3f14-b4ae-4049-a7ba-b7babf4feec8\\question.txt' -> 'C:\\Users\\dusti\\.appmap\\navie\\history\\threads\\9d435cd1-5d42-466d-b6e9-2a4ce58284c5\\sequence\\1725543743379.question.txt'"
10896 [Stderr] }
10896 [Stderr] }
10896 [Stderr] }
10896 [Stderr] Data: [object Object]
Patch 1
diff --git a/packages/cli/src/rpc/explain/navie/historyHelper.ts b/packages/cli/src/rpc/explain/navie/historyHelper.ts
index 68c561b1a..0d3664d6e 100644
--- a/packages/cli/src/rpc/explain/navie/historyHelper.ts
+++ b/packages/cli/src/rpc/explain/navie/historyHelper.ts
@@ -1,5 +1,4 @@
import { homedir } from 'os';
-import { mkdirSync, existsSync } from 'fs';
import History, { ThreadAccessError } from './history';
import Thread from './thread';
import { join } from 'path';
@@ -7,12 +6,9 @@ import { warn } from 'console';
import configuration from '../../configuration';
export function initializeHistory(): History {
- const historyDir = join(homedir(), '.appmap', 'navie', 'history');
- if (!existsSync(historyDir)) {
- mkdirSync(historyDir, { recursive: true });
- }
- return new History(historyDir);
+ return new History(join(homedir(), '.appmap', 'navie', 'history'));
}
+
export async function loadThread(history: History, threadId: string): Promise<Thread> {
let thread: Thread;
diff --git a/packages/cli/tests/unit/rpc/explain/historyHelper.spec.ts b/packages/cli/tests/unit/rpc/explain/historyHelper.spec.ts
deleted file mode 100644
index 517af6a7c..000000000
--- a/packages/cli/tests/unit/rpc/explain/historyHelper.spec.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import { mkdirSync, existsSync } from 'fs';
-import { join } from 'path';
-import { homedir } from 'os';
-import { initializeHistory } from '../../../../src/rpc/explain/navie/historyHelper';
-
-// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-jest.mock('fs', () => ({
- ...jest.requireActual('fs'),
- mkdirSync: jest.fn(),
- existsSync: jest.fn(),
-}));
-
-describe('historyHelper', () => {
- beforeEach(() => jest.resetAllMocks());
-
- it('creates history directory if it does not exist', () => {
- const historyDir = join(homedir(), '.appmap', 'navie', 'history');
- (existsSync as jest.Mock).mockReturnValue(false);
-
- initializeHistory();
-
- expect(mkdirSync).toHaveBeenCalledWith(historyDir, { recursive: true });
- });
-
- it('does not create history directory if it already exists', () => {
- (existsSync as jest.Mock).mockReturnValue(true);
-
- initializeHistory();
-
- expect(mkdirSync).not.toHaveBeenCalled();
- });
-});
diff --git a/packages/navie/CHANGELOG.md b/packages/navie/CHANGELOG.md
index b2e2cafca..d0b98e7d7 100644
--- a/packages/navie/CHANGELOG.md
+++ b/packages/navie/CHANGELOG.md
@@ -1,10 +1,3 @@
-# [@appland/navie-v1.27.1](https://github.com/getappmap/appmap-js/compare/@appland/navie-v1.27.0...@appland/navie-v1.27.1) (2024-09-04)
-
-
-### Bug Fixes
-
-* Don't retry on 422 with OpenAI completion ([91d9711](https://github.com/getappmap/appmap-js/commit/91d9711a46bbae3d587d3f8909d21f67a4d4dd07))
-
# [@appland/navie-v1.27.0](https://github.com/getappmap/appmap-js/compare/@appland/navie-v1.26.1...@appland/navie-v1.27.0) (2024-09-03)
diff --git a/packages/navie/package.json b/packages/navie/package.json
index 69bb8f0b7..5eb2db0e3 100644
--- a/packages/navie/package.json
+++ b/packages/navie/package.json
@@ -1,6 +1,6 @@
{
"name": "@appland/navie",
- "version": "1.27.1",
+ "version": "1.27.0",
"description": "",
"main": "dist/index.js",
"types": "dist/index.d.ts",
Patch 2
diff --git a/.gitignore b/.gitignore
index 14174ff1b..e5308338e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,5 +45,4 @@ notebooks
tmp/appmap
packages/*/tmp/
*.appmap.json
-.navie
diff --git a/packages/cli/src/cmds/index/rpc.ts b/packages/cli/src/cmds/index/rpc.ts
index a4fe69d3b..221ea0155 100644
--- a/packages/cli/src/cmds/index/rpc.ts
+++ b/packages/cli/src/cmds/index/rpc.ts
@@ -10,7 +10,7 @@ import appmapFilter from '../../rpc/appmap/filter';
import { RpcHandler } from '../../rpc/rpc';
import metadata from '../../rpc/appmap/metadata';
import sequenceDiagram from '../../rpc/appmap/sequenceDiagram';
-import { explainHandler, explainStatusHandler, loadThreadHandler } from '../../rpc/explain/explain';
+import { explainHandler, explainStatusHandler } from '../../rpc/explain/explain';
import { buildNavieProvider, commonNavieArgsBuilder as navieBuilder } from '../navie';
import RPCServer from './rpcServer';
import appmapData from '../../rpc/appmap/data';
@@ -27,10 +27,6 @@ import { update } from '../../rpc/file/update';
import { INavieProvider } from '../../rpc/explain/navie/inavie';
import { navieMetadataV1 } from '../../rpc/navie/metadata';
import { navieSuggestHandlerV1 } from '../../rpc/navie/suggest';
-import { initializeHistory } from '../../rpc/explain/navie/historyHelper';
-import History from '../../rpc/explain/navie/history';
-import { join } from 'path';
-import { homedir } from 'os';
export const command = 'rpc';
export const describe = 'Run AppMap JSON-RPC server';
@@ -63,7 +59,6 @@ export function rpcMethods(navie: INavieProvider, codeEditor?: string): RpcHandl
sequenceDiagram(),
explainHandler(navie, codeEditor),
explainStatusHandler(),
- loadThreadHandler(),
update(navie),
setConfigurationV1(),
getConfigurationV1(),
@@ -87,12 +82,6 @@ export const handler = async (argv: HandlerArguments) => {
loadConfiguration(false);
await configureRpcDirectories(argv.directory);
- {
- const history = initializeHistory();
- const oldHistoryDir = join(homedir(), '.appmap', 'navie', 'history');
- await History.migrate(oldHistoryDir, history);
- }
-
const rpcServer = new RPCServer(argv.port, rpcMethods(navie, codeEditor));
rpcServer.start();
};
diff --git a/packages/cli/src/cmds/navie.ts b/packages/cli/src/cmds/navie.ts
index 6fa66fcc7..3191a3c1a 100644
--- a/packages/cli/src/cmds/navie.ts
+++ b/packages/cli/src/cmds/navie.ts
@@ -27,7 +27,6 @@ interface ExplainArgs {
navieProvider?: string;
logNavie?: boolean;
prompt?: string;
- threadId?: string;
}
interface NavieCommonCmdArgs extends ExplainArgs {
@@ -80,11 +79,6 @@ export function commonNavieArgsBuilder<T>(args: yargs.Argv<T>): yargs.Argv<T & N
type: 'string',
// Allow this to be any string. The code editor brand name may be a clue to the language
// in use, or the user's intent.
- })
- .option('thread-id', {
- describe:
- 'The thread ID to use for the question. If not provided, a new thread ID will be allocated. Valid only for local Navie provider.',
- type: 'string',
});
}
@@ -144,8 +138,6 @@ export function buildNavieProvider(argv: ExplainArgs) {
) => {
loadConfiguration(false);
const navie = new LocalNavie(contextProvider, projectInfoProvider, helpProvider);
- if (argv.threadId) navie.setThreadId(argv.threadId);
-
applyAIOptions(navie);
let START: number | undefined;
@@ -172,11 +164,6 @@ export function buildNavieProvider(argv: ExplainArgs) {
loadConfiguration(true);
const navie = new RemoteNavie(contextProvider, projectInfoProvider, helpProvider);
applyAIOptions(navie);
-
- if (argv.threadId) {
- warn(`Ignoring --thread-id option for remote Navie provider`);
- }
-
return navie;
};
diff --git a/packages/cli/src/fulltext/SourceIndex.ts b/packages/cli/src/fulltext/SourceIndex.ts
index a6320c30f..455240bd8 100644
--- a/packages/cli/src/fulltext/SourceIndex.ts
+++ b/packages/cli/src/fulltext/SourceIndex.ts
@@ -10,7 +10,7 @@ import { existsSync } from 'fs';
import { FileIndexMatch } from './FileIndex';
import { verbose } from '../utils';
import { readFile } from 'fs/promises';
-import { basename, join } from 'path';
+import { join } from 'path';
import queryKeywords from './queryKeywords';
const debug = makeDebug('appmap:source-index');
@@ -128,7 +128,7 @@ export class SourceIndex {
try {
debug(`Indexing document ${fileName} from ${from} to ${to}`);
- const terms = queryKeywords([fileName, chunk.pageContent]).join(' ');
+ const terms = queryKeywords([chunk.pageContent]).join(' ');
this.#insert.run(directory, fileName, from, to, chunk.pageContent, terms);
} catch (error) {
console.warn(`Error indexing document ${fileName} from ${from} to ${to}`);
diff --git a/packages/cli/src/rpc/explain/explain.ts b/packages/cli/src/rpc/explain/explain.ts
index ff8620ccb..a20328d37 100644
--- a/packages/cli/src/rpc/explain/explain.ts
+++ b/packages/cli/src/rpc/explain/explain.ts
@@ -21,9 +21,6 @@ import { getLLMConfiguration } from '../llmConfiguration';
import detectAIEnvVar from '../../cmds/index/aiEnvVar';
import reportFetchError from './navie/report-fetch-error';
import { LRUCache } from 'lru-cache';
-import { initializeHistory } from './navie/historyHelper';
-import { ThreadAccessError } from './navie/history';
-import Thread from './navie/thread';
const searchStatusByUserMessageId = new Map<string, ExplainRpc.ExplainStatusResponse>();
@@ -318,21 +315,6 @@ export function explainStatus(userMessageId: string): ExplainRpc.ExplainStatusRe
return searchStatus;
}
-export async function loadThread(threadId: string): Promise<ExplainRpc.LoadThreadResponse> {
- const history = initializeHistory();
-
- let thread: Thread;
- try {
- thread = await history.load(threadId);
- } catch (e) {
- if (e instanceof ThreadAccessError) throw new RpcError(404, `Thread ${threadId} not found`);
-
- throw e;
- }
-
- return thread;
-}
-
const explainHandler: (
navieProvider: INavieProvider,
codeEditor: string | undefined
@@ -367,16 +349,4 @@ const explainStatusHandler: () => RpcHandler<
};
};
-const loadThreadHandler: () => RpcHandler<
- ExplainRpc.LoadThreadOptions,
- ExplainRpc.LoadThreadResponse
-> = () => {
- return {
- name: ExplainRpc.ExplainThreadLoadFunctionName,
- handler: async (options: ExplainRpc.LoadThreadOptions) => {
- return loadThread(options.threadId);
- },
- };
-};
-
-export { explainHandler, explainStatusHandler, loadThreadHandler };
+export { explainHandler, explainStatusHandler };
diff --git a/packages/cli/src/rpc/explain/navie/history.ts b/packages/cli/src/rpc/explain/navie/history.ts
deleted file mode 100644
index 48b3c4138..000000000
--- a/packages/cli/src/rpc/explain/navie/history.ts
+++ /dev/null
@@ -1,351 +0,0 @@
-import { join } from 'path';
-import Thread from './thread';
-import { mkdir, readdir, readFile, readlink, rm, symlink, writeFile } from 'fs/promises';
-import { warn } from 'console';
-import { Message } from '@appland/navie';
-import { exists } from '../../../utils';
-import { OpenMode } from 'fs';
-import configuration from '../../configuration';
-
-export const THREAD_ID_REGEX = /^[0-9a-f]{4,16}(-[0-9a-f]{4,16}){3,6}$/;
-
-export class ThreadAccessError extends Error {
- constructor(public readonly threadId: string, public action: string, public cause?: Error) {
- super(ThreadAccessError.errorMessage(threadId, action, cause));
- }
-
- static errorMessage(threadId: string, action: string, cause?: Error): string {
- const messages = [`Failed to ${action} thread ${threadId}`];
- if (cause) messages.push(cause.message);
- return messages.join(': ');
- }
-}
-
-export enum QuestionField {
- Question = 'question',
- CodeSelection = 'codeSelection',
- Prompt = 'prompt',
-}
-
-export enum ResponseField {
- AssistantMessageId = 'assistantMessageId',
- Answer = 'answer',
-}
-
-type SequenceFile = { timestamp: number; messageId: string };
-
-const parseSequenceFile = (dir: string): SequenceFile => ({
- timestamp: parseInt(dir.split('.')[0]),
- messageId: dir.split('.')[1],
-});
-
-export default class History {
- private firstResponse = new Map<string, number>();
-
- public constructor(public readonly directory: string) {}
-
- async question(
- threadId: string,
- userMessageId: string,
- question: string,
- codeSelection: string | undefined,
- prompt: string | undefined,
- extensions: Record<QuestionField, string> = {
- question: 'txt',
- codeSelection: 'txt',
- prompt: 'md',
- }
- ) {
- const threadDir = await this.ensureThreadDir(threadId);
- const messageDir = await History.findOrCreateMessageDir(threadDir, userMessageId);
-
- const writeMessageFile = async (field: QuestionField, content: string) => {
- const messageFile = join(messageDir, [field, extensions[field]].join('.'));
- await writeFile(messageFile, content);
-
- const sequenceDir = join(threadDir, 'sequence');
- await mkdir(sequenceDir, { recursive: true });
- const sequenceFile = join(
- sequenceDir,
- [Date.now().toString(), field, extensions[field]].join('.')
- );
- await History.createSymlinkIfNotExists(
- join('..', 'messages', userMessageId, [field, extensions[field]].join('.')),
- sequenceFile
- );
- };
-
- await writeMessageFile(QuestionField.Question, question);
- if (codeSelection) await writeMessageFile(QuestionField.CodeSelection, codeSelection);
- if (prompt) await writeMessageFile(QuestionField.Prompt, prompt);
-
- const date = new Date().toISOString().split('T')[0];
- const dateDir = join(this.directory, 'dates', date);
- await mkdir(dateDir, { recursive: true });
- const dateThreadFile = join(dateDir, threadId);
- await History.createSymlinkIfNotExists(threadDir, dateThreadFile);
- }
-
- async token(
- threadId: string,
- userMessageId: string,
- assistantMessageId: string,
- token: string,
- extensions: Record<ResponseField, string> = {
- answer: 'md',
- assistantMessageId: 'txt',
- }
- ) {
- const threadDir = await this.ensureThreadDir(threadId);
- const messageDir = await History.findOrCreateMessageDir(threadDir, userMessageId);
- const timestamp = Date.now();
-
- const writeMessageFile = async (
- field: ResponseField,
- content: string,
- modeFlag: { flag: OpenMode | undefined } = { flag: 'w' }
- ) => {
- const messageFile = join(messageDir, [field, extensions[field]].join('.'));
- await writeFile(messageFile, content, modeFlag);
-
- const sequenceDir = join(threadDir, 'sequence');
- await mkdir(sequenceDir, { recursive: true });
- const sequenceFile = join(sequenceDir, [timestamp, field, extensions[field]].join('.'));
- await History.createSymlinkIfNotExists(
- join('..', 'messages', userMessageId, [field, extensions[field]].join('.')),
- sequenceFile
- );
- };
-
- if (!this.firstResponse.has(assistantMessageId)) {
- this.firstResponse.set(assistantMessageId, timestamp);
-
- await writeMessageFile(ResponseField.AssistantMessageId, assistantMessageId);
- await writeMessageFile(ResponseField.Answer, token, { flag: 'a' });
- } else {
- const messageFile = join(
- messageDir,
- [ResponseField.Answer, extensions[ResponseField.Answer]].join('.')
- );
- await writeFile(messageFile, token, { flag: 'a' });
- }
- }
-
- async load(threadId: string): Promise<Thread> {
- const threadDir = join(this.directory, 'threads', threadId);
- let projectDirectories: string[];
- try {
- projectDirectories = (await readFile(join(threadDir, 'projectDirectories.txt'), 'utf-8'))
- .split('\n')
- .filter(Boolean);
- } catch (e) {
- throw History.threadAccessError(threadId, 'load', e);
- }
-
- const messagesDir = join(this.directory, 'threads', threadId, 'messages');
- const sequenceDir = join(this.directory, 'threads', threadId, 'sequence');
-
- let messageSequenceFiles: string[];
- try {
- messageSequenceFiles = await readdir(sequenceDir);
- } catch (e) {
- throw History.threadAccessError(threadId, 'load', e);
- }
-
- messageSequenceFiles.sort(
- (a, b) => parseSequenceFile(a).timestamp - parseSequenceFile(b).timestamp
- );
- const contentFileFieldName = (contentFile: string) => contentFile.split('.')[0];
-
- const timestamp =
- messageSequenceFiles.length > 0
- ? parseSequenceFile(messageSequenceFiles[0]).timestamp
- : Date.now();
- const thread = new Thread(threadId, timestamp, projectDirectories);
-
- const userMessageIds = new Set<string>();
- for (const sequenceFile of messageSequenceFiles) {
- const sequenceFilePath = join(sequenceDir, sequenceFile);
- // Resolve the file name that the symlink points to.
- let messageFile: string;
- try {
- messageFile = await readlink(sequenceFilePath);
- } catch (e) {
- warn(e);
- continue;
- }
- const messageFileTokens = messageFile.split('/');
- const userMessageId = messageFileTokens[messageFileTokens.length - 2];
- userMessageIds.add(userMessageId);
- }
-
- for (const userMessageId of userMessageIds) {
- const contentFiles = await readdir(join(messagesDir, userMessageId));
- let threadTimestamp: number | undefined;
-
- const readRecordFile = async (recordName: string): Promise<string | undefined> => {
- const matchingContentFile = contentFiles.find(
- (file) => contentFileFieldName(file) === recordName
- );
- if (!matchingContentFile) return undefined;
-
- const contentFile = join(messagesDir, userMessageId, matchingContentFile);
-
- // Read the file timestamp to use as the thread timestamp.
- try {
- const contentFileStat = await readFile(contentFile, 'utf-8');
- const messageTimestamp = parseInt(contentFileStat.split('.')[0]);
- if (!threadTimestamp || messageTimestamp < threadTimestamp)
- threadTimestamp = messageTimestamp;
- } catch (e) {
- throw History.threadAccessError(threadId, 'read', e);
- }
-
- try {
- return await readFile(contentFile, 'utf-8');
- } catch (e) {
- throw History.threadAccessError(threadId, 'read', e);
- }
- };
-
- const question = await readRecordFile('question');
- const codeSelection = await readRecordFile('codeSelection');
- const prompt = await readRecordFile('prompt');
- const assistantMessageId = await readRecordFile('assistantMessageId');
- const answer = await readRecordFile('answer');
-
- if (userMessageId && question)
- thread.question(
- threadTimestamp ?? Date.now(),
- userMessageId,
- question,
- codeSelection,
- prompt
- );
- if (userMessageId && assistantMessageId && answer) {
- thread.answer(userMessageId, assistantMessageId, answer);
- }
- }
-
- return thread;
- }
-
- // Message are stored within threadDir/message in subdirectories. Each subdirectory is named
- // by joining the timestamp and the user message id with a period. This naming convention
- // makes it easy to view and sort the messages in chronological order.
- private static async findOrCreateMessageDir(
- threadDir: string,
- userMessageId: string
- ): Promise<string> {
- // List message directories in the thread directory.
- const messagesDir = join(threadDir, 'messages');
- await mkdir(messagesDir, { recursive: true });
-
- let messageIds: string[];
- try {
- messageIds = await readdir(messagesDir);
- } catch (e) {
- throw this.threadAccessError(threadDir, 'initialize storage for', e);
- }
-
- // Find the message directory for the user message.
- if (messageIds.includes(userMessageId)) {
- return join(messagesDir, userMessageId);
- }
-
- const messagePath = join(messagesDir, userMessageId);
- await mkdir(messagePath, { recursive: true });
- return messagePath;
- }
-
- private static async createSymlinkIfNotExists(target: string, path: string) {
- try {
- await readlink(path);
- } catch (e) {
- const err = e as Error & { code?: string };
- if (err.code === 'ENOENT') {
- await symlink(target, path);
- } else if (err.code === 'EEXIST') {
- // Symlink already exists, do nothing.
- } else {
- throw err;
- }
- }
- }
-
- private async ensureThreadDir(threadId: string): Promise<string> {
- const threadDir = join(this.directory, 'threads', threadId);
- await mkdir(threadDir, { recursive: true });
- const projectDirectoriesFile = join(threadDir, 'projectDirectories.txt');
- if (!(await exists(projectDirectoriesFile))) {
- const projectDirectories = configuration().projectDirectories;
- await writeFile(projectDirectoriesFile, projectDirectories.join('\n'));
- }
- return threadDir;
- }
-
- static threadAccessError(threadId: string, action: string, e: any): ThreadAccessError {
- const err = e as Error & { code?: string };
- if (err.code === 'ENOENT') warn(`Thread ${threadId} not found`);
-
- return new ThreadAccessError(threadId, action, e instanceof Error ? e : undefined);
- }
-
- // In the old-style history format, threads are stored in files named by their thread id
- // and timestamp. In the new-style history format, threads are stored in files named by
- // their thread id, and the thread file is symlinked to a file named for the date of the thread.
- static async migrate(
- oldDirectory: string,
- history: History,
- options: { cleanup: boolean } = { cleanup: true }
- ): Promise<void> {
- // List the contents of the old directory. Each one is a threadId directory, containing
- // timestamped messages.
- const threadIds = (await readdir(oldDirectory)).filter(
- // Match UUIDs like 5278527e-c4ed-4e45-9fb6-372ed6a036f6 in the thread dir
- // Being careful not to touch anything else, like the dates and threads directories that are
- // created in the new history format.
- (dir) => dir.match(THREAD_ID_REGEX)
- );
-
- const threadFileAsTimestamp = (threadFile: string) => parseInt(threadFile.split('.')[0]);
-
- for (const threadId of threadIds) {
- warn(`[history] Migrating thread ${threadId} from old history format`);
-
- const oldThreadDir = join(oldDirectory, threadId);
- const threadFiles = await readdir(oldThreadDir);
- threadFiles.sort((a, b) => threadFileAsTimestamp(a) - threadFileAsTimestamp(b));
-
- let lastUserMessageId: string | undefined;
- for (const threadFile of threadFiles) {
- const timestamp = threadFileAsTimestamp(threadFile);
- // MessageIds are not available, so fake them with timestamps.
- const messageId = timestamp.toString();
- const messageFile = join(oldThreadDir, threadFile);
- const messageStr = await readFile(messageFile, 'utf-8');
- try {
- const message = JSON.parse(messageStr) as Message;
- if (message.role === 'user') {
- lastUserMessageId = messageId;
- // codeSelection and prompt are not available in the old format.
- await history.question(threadId, messageId, message.content, undefined, undefined);
- } else if (message.role === 'assistant' || message.role === 'system') {
- if (lastUserMessageId)
- await history.token(threadId, lastUserMessageId, messageId, message.content);
- }
- } catch (e) {
- warn(`[history] Failed to parse message from ${messageFile}. Skipping.`);
- warn(e);
- }
-
- // Project directories are unknown, so overwrite the file contents with empty text.
- const newThreadDir = join(history.directory, 'threads', threadId);
- if (await exists(newThreadDir))
- await writeFile(join(newThreadDir, 'projectDirectories.txt'), '');
- }
-
- if (options.cleanup) await rm(oldThreadDir, { recursive: true });
- }
- }
-}
diff --git a/packages/cli/src/rpc/explain/navie/historyHelper.ts b/packages/cli/src/rpc/explain/navie/historyHelper.ts
deleted file mode 100644
index 0d3664d6e..000000000
--- a/packages/cli/src/rpc/explain/navie/historyHelper.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import { homedir } from 'os';
-import History, { ThreadAccessError } from './history';
-import Thread from './thread';
-import { join } from 'path';
-import { warn } from 'console';
-import configuration from '../../configuration';
-
-export function initializeHistory(): History {
- return new History(join(homedir(), '.appmap', 'navie', 'history'));
-}
-
-export async function loadThread(history: History, threadId: string): Promise<Thread> {
- let thread: Thread;
-
- try {
- thread = await history.load(threadId);
- } catch (e) {
- if (e instanceof ThreadAccessError) {
- warn(`[remote-navie] Creating new thread ${threadId} (thread not found)`);
- const projectDirectories = configuration().projectDirectories;
- thread = new Thread(threadId, Date.now(), projectDirectories);
- } else {
- throw e;
- }
- }
-
- return thread;
-}
diff --git a/packages/cli/src/rpc/explain/navie/navie-local.ts b/packages/cli/src/rpc/explain/navie/navie-local.ts
index f2edab2ae..5e146a148 100644
--- a/packages/cli/src/rpc/explain/navie/navie-local.ts
+++ b/packages/cli/src/rpc/explain/navie/navie-local.ts
@@ -1,7 +1,10 @@
import { log, warn } from 'console';
import EventEmitter from 'events';
+import { mkdir, readFile, readdir, writeFile } from 'fs/promises';
import { randomUUID } from 'crypto';
-import { ContextV2, Navie, Help, ProjectInfo, navie } from '@appland/navie';
+import { join } from 'path';
+import { homedir } from 'os';
+import { ContextV2, Navie, Help, Message, ProjectInfo, navie } from '@appland/navie';
import INavie from './inavie';
import Telemetry from '../../../telemetry';
@@ -14,8 +17,45 @@ import {
} from '@appland/client';
import reportFetchError from './report-fetch-error';
import assert from 'assert';
-import { initializeHistory, loadThread } from './historyHelper';
-import { THREAD_ID_REGEX } from './history';
+
+class LocalHistory {
+ constructor(public readonly threadId: string) {}
+
+ async saveMessage(message: Message) {
+ if (!['user', 'assistant'].includes(message.role))
+ throw new Error(`Invalid message role for conversation history : ${message.role}`);
+
+ await this.initHistory();
+ const timestampNumber = Date.now();
+ const historyFile = join(this.historyDir, `${timestampNumber}.json`);
+ await writeFile(historyFile, JSON.stringify(message, null, 2));
+ }
+
+ async restoreMessages(): Promise<Message[]> {
+ await this.initHistory();
+ const historyFiles = (await readdir(this.historyDir)).sort();
+ const history: Message[] = [];
+ for (const historyFile of historyFiles) {
+ const historyPath = join(this.historyDir, historyFile);
+ const historyString = await readFile(historyPath, 'utf-8');
+ const message = JSON.parse(historyString);
+ // Fix messages that were miscategorized.
+ if (message.role === 'system') {
+ message.role = 'assistant';
+ }
+ history.push(message);
+ }
+ return history;
+ }
+
+ protected async initHistory() {
+ await mkdir(this.historyDir, { recursive: true });
+ }
+
+ protected get historyDir() {
+ return join(homedir(), '.appmap', 'navie', 'history', this.threadId);
+ }
+}
const OPTION_SETTERS: Record<
string,
@@ -38,8 +78,6 @@ const OPTION_SETTERS: Record<
export default class LocalNavie extends EventEmitter implements INavie {
public navieOptions = new Navie.NavieOptions();
- assignedThreadId: string | undefined;
-
constructor(
private readonly contextProvider: ContextV2.ContextProvider,
private readonly projectInfoProvider: ProjectInfo.ProjectInfoProvider,
@@ -48,14 +86,6 @@ export default class LocalNavie extends EventEmitter implements INavie {
super();
}
- // Sets a thread id to use with the request.
- // The caller is responsible for ensuring that the thread id is a unique, valid uuid.
- setThreadId(threadId: string) {
- if (!THREAD_ID_REGEX.test(threadId)) throw new Error(`Invalid thread id: ${threadId}`);
-
- this.assignedThreadId = threadId;
- }
-
get providerName() {
return 'local';
}
@@ -76,15 +106,9 @@ export default class LocalNavie extends EventEmitter implements INavie {
prompt?: string
): Promise<void> {
if (!threadId) {
- if (this.assignedThreadId) {
- warn(`[local-navie] No threadId provided for question. Using client-specified threadId.`);
- // eslint-disable-next-line no-param-reassign
- threadId = this.assignedThreadId;
- } else {
- warn(`[local-navie] No threadId provided for question. Allocating a new threadId.`);
- // eslint-disable-next-line no-param-reassign
- threadId = randomUUID();
- }
+ warn(`[local-navie] No threadId provided for question. Allocating a new threadId.`);
+ // eslint-disable-next-line no-param-reassign
+ threadId = randomUUID();
}
let userMessageId: string;
@@ -116,8 +140,7 @@ export default class LocalNavie extends EventEmitter implements INavie {
)?.id ?? randomUUID();
}
- const history = initializeHistory();
- const thread = await loadThread(history, threadId);
+ const history = new LocalHistory(threadId);
this.#reportConfigTelemetry();
log(`[local-navie] Processing question ${userMessageId} in thread ${threadId}`);
@@ -130,7 +153,8 @@ export default class LocalNavie extends EventEmitter implements INavie {
prompt,
};
- await history.question(threadId, userMessageId, question, codeSelection, prompt);
+ const messages = await history.restoreMessages();
+ await history.saveMessage({ content: question, role: 'user' });
const startTime = Date.now();
@@ -140,7 +164,7 @@ export default class LocalNavie extends EventEmitter implements INavie {
this.projectInfoProvider,
this.helpProvider,
this.navieOptions,
- thread.messages
+ messages
);
let agentName: string | undefined;
@@ -153,7 +177,6 @@ export default class LocalNavie extends EventEmitter implements INavie {
const response = new Array<string>();
for await (const token of navieFn.execute()) {
response.push(token);
- await history.token(threadId, userMessageId, agentMessageId, token);
this.emit('token', token, agentMessageId);
}
const endTime = Date.now();
@@ -161,6 +184,8 @@ export default class LocalNavie extends EventEmitter implements INavie {
warn(`[local-navie] Completed question ${userMessageId} in ${duration}ms`);
+ await history.saveMessage({ content: response.join(''), role: 'assistant' });
+
{
const userMessage: UpdateUserMessage = {
agentName,
diff --git a/packages/cli/src/rpc/explain/navie/navie-remote.ts b/packages/cli/src/rpc/explain/navie/navie-remote.ts
index 4e2472c86..2101d9747 100644
--- a/packages/cli/src/rpc/explain/navie/navie-remote.ts
+++ b/packages/cli/src/rpc/explain/navie/navie-remote.ts
@@ -6,126 +6,8 @@ import { ContextV1, ContextV2, Help, ProjectInfo } from '@appland/navie';
import { verbose } from '../../../utils';
import { default as INavie } from './inavie';
import assert from 'assert';
-import { initializeHistory, loadThread } from './historyHelper';
-import Thread from './thread';
-import History from './history';
-
-export class RemtoteCallbackHandler {
- thread: Thread | undefined;
- userMessageId: string | undefined;
- assistantMessageId: string | undefined;
- tokens: string[] = [];
-
- constructor(
- private readonly history: History,
- private readonly contextProvider: ContextV2.ContextProvider,
- private readonly projectInfoProvider: ProjectInfo.ProjectInfoProvider,
- private readonly helpProvider: Help.HelpProvider
- ) {}
-
- async onAck(
- assignedUserMessageId: string,
- threadId: string,
- question: string,
- codeSelection?: string,
- prompt?: string
- ): Promise<void> {
- if (verbose())
- warn(`Explain received ack (userMessageId=${assignedUserMessageId}, threadId=${threadId})`);
-
- this.thread = await loadThread(this.history, threadId);
- this.userMessageId = assignedUserMessageId;
- await this.history.question(threadId, this.userMessageId, question, codeSelection, prompt);
- }
-
- async onToken(token: string, messageId: string): Promise<void> {
- if (!this.assistantMessageId) this.assistantMessageId = messageId;
-
- this.tokens.push(token);
-
- if (this.thread && this.userMessageId)
- await this.history.token(
- this.thread.threadId,
- this.userMessageId,
- this.assistantMessageId,
- token
- );
- else warn(`[remote-navie] Received token but no thread is available to store it`);
- }
-
- async onRequestContext(
- data: Record<string, unknown>
- ): Promise<Record<string, unknown> | unknown[]> {
- try {
- if (data.type === 'search') {
- const { version } = data;
- const isVersion1 = !version || version === 1;
-
- // ContextV2.ContextRequest is a superset of ContextV1.ContextRequest, so whether the input
- // version is V1 or V2, we can treat it as V2.
- const contextRequestV2: ContextV2.ContextRequest = data as ContextV2.ContextRequest;
-
- const responseV2: ContextV2.ContextResponse = await this.contextProvider({
- ...contextRequestV2,
- type: 'search',
- version: 2,
- });
-
- if (isVersion1) {
- // Adapt from V2 response back to V1. Some data may be lost in this process.
- const responseV1: ContextV1.ContextResponse = {
- sequenceDiagrams: responseV2
- .filter((item) => item.type === ContextV2.ContextItemType.SequenceDiagram)
- .map((item) => item.content),
- codeSnippets: responseV2
- .filter((item) => item.type === ContextV2.ContextItemType.CodeSnippet)
- .reduce((acc, item) => {
- assert(item.type === ContextV2.ContextItemType.CodeSnippet);
- if (ContextV2.isFileContextItem(item)) {
- acc[item.location] = item.content;
- }
- return acc;
- }, {} as Record<string, string>),
- codeObjects: responseV2
- .filter((item) => item.type === ContextV2.ContextItemType.DataRequest)
- .map((item) => item.content),
- };
- return responseV1;
- } else {
- return responseV2;
- }
- }
- if (data.type === 'projectInfo') {
- return (
- (await this.projectInfoProvider(data as unknown as ProjectInfo.ProjectInfoRequest)) ?? {}
- );
- }
- if (data.type === 'help') {
- return (await this.helpProvider(data as unknown as Help.HelpRequest)) || {};
- } else {
- warn(`Unhandled context request type: ${data.type}`);
- // A response is required from this function.
- return {};
- }
- } catch (e) {
- warn(`Explain context function ${JSON.stringify(data)} threw an error: ${e}`);
- return {};
- }
- }
-
- // eslint-disable-next-line @typescript-eslint/no-misused-promises
- onComplete(): void {
- if (verbose()) warn(`Explain is complete`);
- }
-
- onError(err: Error): void {
- if (verbose()) warn(`Error handled by Explain: ${err}`);
- }
-}
export default class RemoteNavie extends EventEmitter implements INavie {
- private history = initializeHistory();
-
constructor(
private contextProvider: ContextV2.ContextProvider,
private projectInfoProvider: ProjectInfo.ProjectInfoProvider,
@@ -142,53 +24,95 @@ export default class RemoteNavie extends EventEmitter implements INavie {
throw new Error(`RemoteNavie does not support option '${key}'`);
}
- async ask(threadId: string, question: string, codeSelection?: string, prompt?: string) {
- const callbackHandler = new RemtoteCallbackHandler(
- this.history,
- this.contextProvider,
- this.projectInfoProvider,
- this.helpProvider
- );
-
- const onAck = async (userMessageId: string, threadId: string) => {
- await callbackHandler.onAck(userMessageId, threadId, question, codeSelection, prompt);
-
- this.emit('ack', userMessageId, threadId);
- };
-
- const onToken = async (token: string, messageId: string): Promise<void> => {
- await callbackHandler.onToken(token, messageId);
-
- this.emit('token', token, messageId);
- };
-
- const onRequestContext = async (
- data: Record<string, unknown>
- ): Promise<Record<string, unknown> | unknown[]> => {
- return await callbackHandler.onRequestContext(data);
- };
-
- const onComplete = () => {
- callbackHandler.onComplete();
-
- this.emit('complete');
- };
-
- const onError = (err: Error) => {
- callbackHandler.onError(err);
-
- this.emit('error', err);
- };
-
- const callbacksObj = {
- onAck,
- onToken,
- onRequestContext,
- onComplete,
- onError,
- };
-
- (await AI.connect(callbacksObj)).inputPrompt(
+ async ask(
+ threadId: string,
+ question: string,
+ codeSelection?: string,
+ prompt?: string
+ ) {
+ if (prompt) {
+ warn(`RemoteNavie does not support a custom prompt option.`);
+ }
+ (
+ await AI.connect({
+ onAck: (userMessageId, threadId) => {
+ if (verbose())
+ warn(`Explain received ack (userMessageId=${userMessageId}, threadId=${threadId})`);
+ this.emit('ack', userMessageId, threadId);
+ },
+ onToken: (token, _messageId) => {
+ this.emit('token', token);
+ },
+ onRequestContext: async (data) => {
+ try {
+ if (data.type === 'search') {
+ const { version } = data;
+ const isVersion1 = !version || version === 1;
+
+ // ContextV2.ContextRequest is a superset of ContextV1.ContextRequest, so whether the input
+ // version is V1 or V2, we can treat it as V2.
+ const contextRequestV2: ContextV2.ContextRequest = data as ContextV2.ContextRequest;
+
+ const responseV2: ContextV2.ContextResponse = await this.contextProvider({
+ ...contextRequestV2,
+ type: 'search',
+ version: 2,
+ });
+
+ if (isVersion1) {
+ // Adapt from V2 response back to V1. Some data may be lost in this process.
+ const responseV1: ContextV1.ContextResponse = {
+ sequenceDiagrams: responseV2
+ .filter((item) => item.type === ContextV2.ContextItemType.SequenceDiagram)
+ .map((item) => item.content),
+ codeSnippets: responseV2
+ .filter((item) => item.type === ContextV2.ContextItemType.CodeSnippet)
+ .reduce((acc, item) => {
+ assert(item.type === ContextV2.ContextItemType.CodeSnippet);
+ if (ContextV2.isFileContextItem(item)) {
+ acc[item.location] = item.content;
+ }
+ return acc;
+ }, {} as Record<string, string>),
+ codeObjects: responseV2
+ .filter((item) => item.type === ContextV2.ContextItemType.DataRequest)
+ .map((item) => item.content),
+ };
+ return responseV1;
+ } else {
+ return responseV2;
+ }
+ }
+ if (data.type === 'projectInfo') {
+ return (
+ (await this.projectInfoProvider(
+ data as unknown as ProjectInfo.ProjectInfoRequest
+ )) || {}
+ );
+ }
+ if (data.type === 'help') {
+ return (await this.helpProvider(data as unknown as Help.HelpRequest)) || {};
+ } else {
+ warn(`Unhandled context request type: ${data.type}`);
+ // A response is required from this function.
+ return {};
+ }
+ } catch (e) {
+ warn(`Explain context function ${JSON.stringify(data)} threw an error: ${e}`);
+ // TODO: Report an error object instead?
+ return {};
+ }
+ },
+ onComplete: () => {
+ if (verbose()) warn(`Explain is complete`);
+ this.emit('complete');
+ },
+ onError: (err: Error) => {
+ if (verbose()) warn(`Error handled by Explain: ${err}`);
+ this.emit('error', err);
+ },
+ })
+ ).inputPrompt(
{ question: question, codeSelection: codeSelection },
{ threadId, tool: 'explain' }
);
diff --git a/packages/cli/src/rpc/explain/navie/thread.ts b/packages/cli/src/rpc/explain/navie/thread.ts
deleted file mode 100644
index 018b1334c..000000000
--- a/packages/cli/src/rpc/explain/navie/thread.ts
+++ /dev/null
@@ -1,122 +0,0 @@
-import { warn } from 'console';
-
-import { ExplainRpc } from '@appland/rpc';
-import configuration from '../../configuration';
-import { getLLMConfiguration } from '../../llmConfiguration';
-
-export type Message = ExplainRpc.Message;
-
-export type Question = ExplainRpc.Question;
-
-export type Answer = ExplainRpc.Answer;
-
-export class Exchange {
- readonly question: Question;
- answer?: Answer;
-
- constructor(
- timestamp: number,
- messageId: string,
- question: string,
- codeSelection?: string,
- prompt?: string
- ) {
- this.question = {
- timestamp,
- messageId,
- content: question,
- role: 'user',
- codeSelection,
- prompt,
- };
- }
-
- setAnswer(messageId: string, answer: string) {
- this.answer = {
- messageId,
- content: answer,
- role: 'assistant',
- };
- }
-}
-
-export type ThreadData = ExplainRpc.Thread;
-
-export default class Thread implements ThreadData {
- public readonly exchanges: Exchange[] = [];
-
- public constructor(
- public readonly threadId: string,
- public readonly timestamp: number,
- public readonly projectDirectories: string[]
- ) {}
-
- get messages(): Message[] {
- const result = new Array<Message>();
- for (const exchange of this.exchanges) {
- result.push(exchange.question);
- if (exchange.answer) result.push(exchange.answer);
- }
- return result;
- }
-
- /**
- * Gets the thread timestamp formatted as 'YYYY-mm-dd'.
- */
- date() {
- const date = new Date(this.timestamp);
- return date.toISOString().split('T')[0];
- }
-
- question(
- timestamp: number,
- messageId: string,
- question: string,
- codeSelection?: string,
- prompt?: string
- ) {
- const exchange = new Exchange(timestamp, messageId, question, codeSelection, prompt);
- this.exchanges.push(exchange);
- }
-
- answer(userMessageId: string, messageId: string, answer: string) {
- const exchange = this.exchanges[this.exchanges.length - 1];
- if (!exchange) {
- warn(`[history/thread] No question to answer for message ${messageId}`);
- return;
- }
- if (exchange.question.messageId !== userMessageId) {
- warn(`[history/thread] Received an answer to a different question than the last one asked`);
- return;
- }
- exchange.setAnswer(messageId, answer);
- }
-
- asJSON(): ThreadData {
- return {
- timestamp: this.timestamp,
- projectDirectories: this.projectDirectories,
- exchanges: this.exchanges,
- };
- }
-
- static fromJSON(threadId: string, data: ThreadData): Thread {
- const thread = new Thread(threadId, data.timestamp, data.projectDirectories);
- for (const exchange of data.exchanges) {
- thread.question(
- exchange.question.timestamp,
- exchange.question.messageId,
- exchange.question.content,
- exchange.question.codeSelection,
- exchange.question.prompt
- );
- if (exchange.answer)
- thread.answer(
- exchange.question.messageId,
- exchange.answer.messageId,
- exchange.answer.content
- );
- }
- return thread;
- }
-}
diff --git a/packages/client/src/aiClient.ts b/packages/client/src/aiClient.ts
index 60edc54ea..784994478 100644
--- a/packages/client/src/aiClient.ts
+++ b/packages/client/src/aiClient.ts
@@ -13,13 +13,13 @@ export class CodedError extends Error {
}
export type Callbacks = {
- onToken: (token: string, messageId: string) => void | Promise<void>;
- onComplete(): void | Promise<void>;
+ onToken: (token: string, messageId: string) => void;
+ onComplete(): void;
onRequestContext?: (
data: Record<string, unknown>
) => Record<string, unknown> | unknown[] | Promise<Record<string, unknown> | unknown[]>;
- onAck?: (userMessageId: string, threadId: string) => void | Promise<void>;
- onError?: (error: Error) => void | Promise<void>;
+ onAck?: (userMessageId: string, threadId: string) => void;
+ onError?: (error: Error) => void;
};
export type UserInput = {
@@ -67,13 +67,13 @@ export default class AIClient {
if (!('userMessageId' in message))
this.panic(new Error('Unexpected ack message: no userMessageId'));
if (!('threadId' in message)) this.panic(new Error('Unexpected ack message: no threadId'));
- await this.callbacks.onAck?.(message.userMessageId as string, message.threadId as string);
+ this.callbacks.onAck?.(message.userMessageId as string, message.threadId as string);
break;
case 'token':
if (!('token' in message)) this.panic(new Error('Unexpected token message: no token'));
if (!('messageId' in message))
this.panic(new Error('Unexpected token message: no messageId'));
- await this.callbacks.onToken(message.token as string, message.messageId as string);
+ this.callbacks.onToken(message.token as string, message.messageId as string);
break;
case 'request-context': {
if (!this.callbacks.onRequestContext) {
@@ -86,7 +86,7 @@ export default class AIClient {
break;
}
case 'end':
- await this.callbacks.onComplete();
+ this.callbacks.onComplete();
this.disconnect();
break;
default:
diff --git a/packages/rpc/src/explain.ts b/packages/rpc/src/explain.ts
index 3e5c624f0..ec12a18e3 100644
--- a/packages/rpc/src/explain.ts
+++ b/packages/rpc/src/explain.ts
@@ -1,10 +1,8 @@
-import { ConfigurationRpc } from './configuration';
import { SearchRpc } from './search';
export namespace ExplainRpc {
export const ExplainFunctionName = 'explain';
export const ExplainStatusFunctionName = 'explain.status';
- export const ExplainThreadLoadFunctionName = 'explain.thread.load';
export enum Step {
NEW = 'new',
@@ -62,38 +60,4 @@ export namespace ExplainRpc {
contextResponse?: ContextItem[];
explanation?: string[];
};
-
- export type Message = {
- messageId: string;
- content: string;
- role: 'user' | 'assistant';
- };
-
- export type Question = Message & {
- role: 'user';
- timestamp: number;
- codeSelection?: string;
- prompt?: string;
- };
-
- export type Answer = Message & {
- role: 'assistant';
- };
-
- export type Exchange = {
- question: Question;
- answer?: Answer;
- };
-
- export type Thread = {
- timestamp: number;
- projectDirectories: string[];
- exchanges: Exchange[];
- };
-
- export type LoadThreadOptions = {
- threadId: string;
- };
-
- export type LoadThreadResponse = Thread;
}
Directives
- Analyze the impact of the changes to file handling, in particular the usage of symlinks and the permissions and behavior of symlinks on Windows.
- Symlink management on Windows can fail if the user does not have admin permissions.
- Make a recommendation about how to trap and log the symlink failures
- Symlink failure should be logged once per process only.
- If the symlink errors are handled and ignored, what failure modes may be introduced?
Contributor guide
No contributing guide indexed for this repository
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Start with packages/cli/src/rpc/explain/navie/history.ts and historyHelper.ts, focusing on the symlink operation represented by the Windows EPERM error. Review the affected CLI files in Patch 2 and the deleted historyHelper.spec.ts; done means Navie history no longer fails on Windows when creating history links.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- typescript
- Domain
- cli
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 32/100