Compare commits
5 Commits
main
...
feat/memor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b3a6fbaa5c | ||
|
|
33de27e616 | ||
|
|
8f776f7c7c | ||
|
|
4aa1584c65 | ||
|
|
5a8747332b |
250
packages/core/src/services/memoryDiagnosticsDumper.test.ts
Normal file
250
packages/core/src/services/memoryDiagnosticsDumper.test.ts
Normal file
@ -0,0 +1,250 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Qwen Team
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { MemoryDiagnosticsDumper } from './memoryDiagnosticsDumper.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
|
||||
vi.mock('node:fs', () => ({
|
||||
mkdirSync: vi.fn(),
|
||||
writeFileSync: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('node:v8', () => ({
|
||||
getHeapStatistics: vi.fn().mockReturnValue({
|
||||
heap_size_limit: 4_096_000_000,
|
||||
total_heap_size: 2_048_000_000,
|
||||
used_heap_size: 1_800_000_000,
|
||||
total_available_size: 2_000_000_000,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('../utils/memoryDiagnostics.js', () => ({
|
||||
collectMemoryDiagnostics: vi.fn().mockResolvedValue({
|
||||
timestamp: '2026-05-31T00:00:00.000Z',
|
||||
memoryUsage: {
|
||||
rss: 2_000_000_000,
|
||||
heapUsed: 1_800_000_000,
|
||||
heapTotal: 2_048_000_000,
|
||||
external: 50_000_000,
|
||||
arrayBuffers: 10_000_000,
|
||||
},
|
||||
v8HeapStats: {
|
||||
heapSizeLimit: 4_096_000_000,
|
||||
totalHeapSize: 2_048_000_000,
|
||||
usedHeapSize: 1_800_000_000,
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
function createMockConfig(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
return {
|
||||
getSessionId: vi.fn().mockReturnValue('test-session-id-12345678'),
|
||||
getCliVersion: vi.fn().mockReturnValue('0.17.0'),
|
||||
getGeminiClient: vi.fn().mockReturnValue({
|
||||
getChat: () => ({
|
||||
getHistoryLength: () => 500,
|
||||
}),
|
||||
}),
|
||||
storage: {
|
||||
getProjectDir: vi.fn().mockReturnValue('/tmp/test-project'),
|
||||
},
|
||||
...overrides,
|
||||
} as unknown as Config;
|
||||
}
|
||||
|
||||
describe('MemoryDiagnosticsDumper', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('writes diagnostics JSON on first dump', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
const result = await dumper.dump('hard');
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result!.trigger).toBe('hard');
|
||||
expect(result!.filePath).toContain(
|
||||
path.join('/tmp/test-project', 'diagnostics') + path.sep,
|
||||
);
|
||||
expect(result!.filePath).toContain('memory-test-ses');
|
||||
expect(fs.mkdirSync).toHaveBeenCalledWith(
|
||||
expect.stringContaining('diagnostics'),
|
||||
{ recursive: true },
|
||||
);
|
||||
// Two-phase write: Phase 1 (minimal) + Phase 2 (full)
|
||||
expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
|
||||
|
||||
const phase1Content = JSON.parse(
|
||||
vi.mocked(fs.writeFileSync).mock.calls[0][1] as string,
|
||||
);
|
||||
expect(phase1Content.trigger).toBe('hard');
|
||||
expect(phase1Content.dumpNumber).toBe(1);
|
||||
expect(phase1Content.collectionComplete).toBe(false);
|
||||
expect(phase1Content.memoryUsage).toBeDefined();
|
||||
expect(phase1Content.v8HeapStats).toBeDefined();
|
||||
|
||||
const phase2Content = JSON.parse(
|
||||
vi.mocked(fs.writeFileSync).mock.calls[1][1] as string,
|
||||
);
|
||||
expect(phase2Content.trigger).toBe('hard');
|
||||
expect(phase2Content.dumpNumber).toBe(1);
|
||||
expect(phase2Content.collectionComplete).toBe(true);
|
||||
expect(phase2Content.memoryUsage.rss).toBe(2_000_000_000);
|
||||
expect(phase2Content.session.historyEntries).toBe(500);
|
||||
expect(phase2Content.suggestion).toContain('/compress');
|
||||
});
|
||||
|
||||
it('respects per-session cap of 3 dumps', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
// Bypass cooldown by mocking Date.now
|
||||
let mockNow = 1000000;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => {
|
||||
mockNow += 60_000;
|
||||
return mockNow;
|
||||
});
|
||||
|
||||
const r1 = await dumper.dump('hard');
|
||||
const r2 = await dumper.dump('critical');
|
||||
const r3 = await dumper.dump('hard');
|
||||
const r4 = await dumper.dump('critical');
|
||||
|
||||
expect(r1).toBeDefined();
|
||||
expect(r2).toBeDefined();
|
||||
expect(r3).toBeDefined();
|
||||
expect(r4).toBeUndefined();
|
||||
// 3 successful dumps × 2 writes each (Phase 1 + Phase 2)
|
||||
expect(fs.writeFileSync).toHaveBeenCalledTimes(6);
|
||||
});
|
||||
|
||||
it('respects cooldown between dumps', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
const mockNow = 1000000;
|
||||
vi.spyOn(Date, 'now').mockReturnValue(mockNow);
|
||||
|
||||
const r1 = await dumper.dump('hard');
|
||||
const r2 = await dumper.dump('hard');
|
||||
|
||||
expect(r1).toBeDefined();
|
||||
expect(r2).toBeUndefined();
|
||||
// 1 successful dump × 2 writes (Phase 1 + Phase 2)
|
||||
expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('resets state on new session', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
let mockNow = 1000000;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => {
|
||||
mockNow += 60_000;
|
||||
return mockNow;
|
||||
});
|
||||
|
||||
await dumper.dump('hard');
|
||||
await dumper.dump('hard');
|
||||
await dumper.dump('hard');
|
||||
|
||||
// Cap reached
|
||||
const r4 = await dumper.dump('hard');
|
||||
expect(r4).toBeUndefined();
|
||||
|
||||
// Reset
|
||||
dumper.resetForNewSession();
|
||||
|
||||
const r5 = await dumper.dump('critical');
|
||||
expect(r5).toBeDefined();
|
||||
expect(r5!.trigger).toBe('critical');
|
||||
});
|
||||
|
||||
it('includes critical suggestion for critical pressure', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
await dumper.dump('critical');
|
||||
|
||||
// Phase 2 (full payload) is the second write
|
||||
const writtenContent = JSON.parse(
|
||||
vi.mocked(fs.writeFileSync).mock.calls[1][1] as string,
|
||||
);
|
||||
expect(writtenContent.suggestion).toContain('critically high');
|
||||
expect(writtenContent.collectionComplete).toBe(true);
|
||||
});
|
||||
|
||||
it('writes Phase 1 synchronously before any await (survives crash during Phase 2)', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
// Fire dump() but do not await — Phase 1 must have already written to disk
|
||||
// because async functions execute synchronously up to the first await.
|
||||
const promise = dumper.dump('hard');
|
||||
|
||||
// At this point Phase 2 has not run yet (its await is pending), but Phase 1
|
||||
// must have completed its writeFileSync call.
|
||||
expect(fs.writeFileSync).toHaveBeenCalledTimes(1);
|
||||
const phase1Content = JSON.parse(
|
||||
vi.mocked(fs.writeFileSync).mock.calls[0][1] as string,
|
||||
);
|
||||
expect(phase1Content.collectionComplete).toBe(false);
|
||||
|
||||
await promise;
|
||||
// Phase 2 has now overwritten the file
|
||||
expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('reserves slot synchronously to prevent concurrent dumps from bypassing cap', async () => {
|
||||
const config = createMockConfig();
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
let mockNow = 1000000;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => {
|
||||
mockNow += 60_000;
|
||||
return mockNow;
|
||||
});
|
||||
|
||||
// Fire 4 concurrent dumps without awaiting between them. The synchronous
|
||||
// slot reservation must enforce the cap of 3 even though all 4 calls happen
|
||||
// before any of them complete their async Phase 2.
|
||||
const results = await Promise.all([
|
||||
dumper.dump('hard'),
|
||||
dumper.dump('hard'),
|
||||
dumper.dump('hard'),
|
||||
dumper.dump('hard'),
|
||||
]);
|
||||
|
||||
const successful = results.filter((r) => r !== undefined);
|
||||
expect(successful).toHaveLength(3);
|
||||
expect(results[3]).toBeUndefined();
|
||||
});
|
||||
|
||||
it('handles missing geminiClient gracefully', async () => {
|
||||
const config = createMockConfig({
|
||||
getGeminiClient: vi.fn().mockReturnValue(null),
|
||||
});
|
||||
const dumper = new MemoryDiagnosticsDumper(config);
|
||||
|
||||
const result = await dumper.dump('hard');
|
||||
|
||||
expect(result).toBeDefined();
|
||||
// Phase 2 (full payload) is the second write
|
||||
const writtenContent = JSON.parse(
|
||||
vi.mocked(fs.writeFileSync).mock.calls[1][1] as string,
|
||||
);
|
||||
expect(writtenContent.session.available).toBe(false);
|
||||
});
|
||||
});
|
||||
175
packages/core/src/services/memoryDiagnosticsDumper.ts
Normal file
175
packages/core/src/services/memoryDiagnosticsDumper.ts
Normal file
@ -0,0 +1,175 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Qwen Team
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* Writes a lightweight memory diagnostics JSON to disk when the memory
|
||||
* pressure monitor detects hard or critical pressure. The file survives
|
||||
* a subsequent OOM crash, giving maintainers actionable data from bug
|
||||
* reports without requiring the user to manually run `/doctor memory`.
|
||||
*
|
||||
* Design: diagnostics JSON is written BEFORE any expensive operation
|
||||
* (like heap snapshots) so it lands on disk even if the process crashes
|
||||
* during the heavier step.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as v8 from 'node:v8';
|
||||
import { collectMemoryDiagnostics } from '../utils/memoryDiagnostics.js';
|
||||
import { createDebugLogger } from '../utils/debugLogger.js';
|
||||
import { getErrorMessage } from '../utils/errors.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
|
||||
const debugLogger = createDebugLogger('MEMORY_DUMP');
|
||||
|
||||
/** Maximum dumps per session to avoid disk flooding. */
|
||||
const MAX_DUMPS_PER_SESSION = 3;
|
||||
|
||||
/** Minimum interval between dumps (ms). */
|
||||
const MIN_DUMP_INTERVAL_MS = 30_000;
|
||||
|
||||
export interface MemoryDumpResult {
|
||||
filePath: string;
|
||||
trigger: string;
|
||||
}
|
||||
|
||||
export class MemoryDiagnosticsDumper {
|
||||
private dumpCount = 0;
|
||||
private lastDumpTime = 0;
|
||||
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
/**
|
||||
* Resets session-scoped state. Called when a new session starts.
|
||||
*/
|
||||
resetForNewSession(): void {
|
||||
this.dumpCount = 0;
|
||||
this.lastDumpTime = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a diagnostics snapshot to disk if within per-session limits.
|
||||
*
|
||||
* Uses a two-phase write strategy:
|
||||
* - Phase 1 (synchronous): writes a minimal JSON with process.memoryUsage()
|
||||
* and v8.getHeapStatistics() — no fork/exec, so it lands on disk even
|
||||
* under extreme memory pressure.
|
||||
* - Phase 2 (async): collects full diagnostics (may spawn subprocesses)
|
||||
* and overwrites the file with the complete payload. If Phase 2 crashes,
|
||||
* Phase 1's file still survives for debugging.
|
||||
*
|
||||
* Slot is reserved synchronously before any await to prevent concurrent
|
||||
* invocations from bypassing the cap/cooldown guards.
|
||||
*/
|
||||
async dump(
|
||||
trigger: 'hard' | 'critical',
|
||||
): Promise<MemoryDumpResult | undefined> {
|
||||
if (this.dumpCount >= MAX_DUMPS_PER_SESSION) {
|
||||
debugLogger.debug(
|
||||
`Skipping dump: session cap reached (${MAX_DUMPS_PER_SESSION})`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - this.lastDumpTime < MIN_DUMP_INTERVAL_MS) {
|
||||
debugLogger.debug('Skipping dump: cooldown not elapsed');
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Reserve slot synchronously to prevent race between concurrent dumps
|
||||
const dumpNumber = ++this.dumpCount;
|
||||
this.lastDumpTime = now;
|
||||
|
||||
try {
|
||||
const diagnosticsDir = this.ensureDiagnosticsDir();
|
||||
const sessionId = this.config.getSessionId();
|
||||
const timestamp = new Date()
|
||||
.toISOString()
|
||||
.replace(/:/g, '-')
|
||||
.replace(/\./g, '_');
|
||||
const fileName = `memory-${sessionId.slice(0, 8)}-${timestamp}.json`;
|
||||
const filePath = path.join(diagnosticsDir, fileName);
|
||||
|
||||
// Phase 1: synchronous minimal write — survives crash during Phase 2
|
||||
const minimalPayload = {
|
||||
trigger,
|
||||
dumpNumber,
|
||||
timestamp: new Date().toISOString(),
|
||||
memoryUsage: process.memoryUsage(),
|
||||
v8HeapStats: v8.getHeapStatistics(),
|
||||
session: this.collectSessionStats(),
|
||||
suggestion: this.getSuggestion(trigger),
|
||||
collectionComplete: false,
|
||||
};
|
||||
fs.writeFileSync(
|
||||
filePath,
|
||||
JSON.stringify(minimalPayload, null, 2),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
debugLogger.info(
|
||||
`Phase 1 diagnostics written to ${filePath} (trigger=${trigger}, dump #${dumpNumber})`,
|
||||
);
|
||||
|
||||
// Phase 2: full collection (may fork subprocesses — risky under pressure)
|
||||
const diagnostics = await collectMemoryDiagnostics({
|
||||
sessionId,
|
||||
qwenVersion: this.config.getCliVersion(),
|
||||
});
|
||||
|
||||
const fullPayload = {
|
||||
trigger,
|
||||
dumpNumber,
|
||||
...diagnostics,
|
||||
session: this.collectSessionStats(),
|
||||
suggestion: this.getSuggestion(trigger),
|
||||
collectionComplete: true,
|
||||
};
|
||||
fs.writeFileSync(filePath, JSON.stringify(fullPayload, null, 2), 'utf8');
|
||||
|
||||
debugLogger.info(
|
||||
`Phase 2 diagnostics written to ${filePath} (trigger=${trigger}, dump #${dumpNumber})`,
|
||||
);
|
||||
|
||||
return { filePath, trigger };
|
||||
} catch (err) {
|
||||
// Slot stays consumed — a failed write should not open the door to more
|
||||
// attempts that would likely also fail under the same pressure conditions.
|
||||
debugLogger.error(
|
||||
`Failed to write memory diagnostics: ${getErrorMessage(err)}`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private ensureDiagnosticsDir(): string {
|
||||
const projectDir = this.config.storage.getProjectDir();
|
||||
const diagnosticsDir = path.join(projectDir, 'diagnostics');
|
||||
fs.mkdirSync(diagnosticsDir, { recursive: true });
|
||||
return diagnosticsDir;
|
||||
}
|
||||
|
||||
private collectSessionStats(): Record<string, unknown> {
|
||||
try {
|
||||
const geminiClient = this.config.getGeminiClient?.();
|
||||
if (!geminiClient) return { available: false };
|
||||
const historyLength = geminiClient.getChat?.()?.getHistoryLength?.() ?? 0;
|
||||
return {
|
||||
historyEntries: historyLength,
|
||||
};
|
||||
} catch {
|
||||
return { available: false };
|
||||
}
|
||||
}
|
||||
|
||||
private getSuggestion(trigger: 'hard' | 'critical'): string {
|
||||
if (trigger === 'critical') {
|
||||
return 'Memory is critically high. Consider running /compress or starting a fresh session to avoid OOM.';
|
||||
}
|
||||
return 'Memory pressure detected. Running /compress may help reduce memory usage.';
|
||||
}
|
||||
}
|
||||
@ -11,6 +11,7 @@ import { getHeapStatistics } from 'node:v8';
|
||||
import { createDebugLogger } from '../utils/debugLogger.js';
|
||||
import { getErrorMessage } from '../utils/errors.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { MemoryDiagnosticsDumper } from './memoryDiagnosticsDumper.js';
|
||||
|
||||
// Types
|
||||
|
||||
@ -104,6 +105,7 @@ export class MemoryPressureMonitor extends EventEmitter {
|
||||
private consecutiveIneffectiveAggressiveCleanups = 0;
|
||||
private cleanupGeneration = 0;
|
||||
private readonly effectiveMemoryLimit: number;
|
||||
private readonly diagnosticsDumper: MemoryDiagnosticsDumper;
|
||||
|
||||
constructor(coreConfig: Config, pressureConfig?: MemoryPressureConfig) {
|
||||
super();
|
||||
@ -111,6 +113,7 @@ export class MemoryPressureMonitor extends EventEmitter {
|
||||
this.config = { ...(pressureConfig ?? DEFAULT_PRESSURE_CONFIG) };
|
||||
validateMemoryPressureConfig(this.config);
|
||||
this.effectiveMemoryLimit = this.computeEffectiveMemoryLimit();
|
||||
this.diagnosticsDumper = new MemoryDiagnosticsDumper(coreConfig);
|
||||
const heapSizeLimit = getHeapStatistics().heap_size_limit;
|
||||
debugLogger.info(
|
||||
`Effective memory limit: ${formatMiB(this.effectiveMemoryLimit)} MiB; ` +
|
||||
@ -142,6 +145,7 @@ export class MemoryPressureMonitor extends EventEmitter {
|
||||
resetForNewSession(): void {
|
||||
this.cleanupGeneration++;
|
||||
this.resetConsecutiveFailures();
|
||||
this.diagnosticsDumper.resetForNewSession();
|
||||
this.cleanupInProgress = false;
|
||||
this.activeCleanupAction = 'none';
|
||||
this.queuedCleanupRecommendation = undefined;
|
||||
@ -196,6 +200,16 @@ export class MemoryPressureMonitor extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
|
||||
// Write diagnostics to disk before cleanup. dump() uses a two-phase strategy:
|
||||
// Phase 1 (synchronous, before the first await) writes a minimal JSON via
|
||||
// writeFileSync — guaranteed to complete before executeCleanup starts because
|
||||
// async functions run synchronously up to the first await. Phase 2 enriches
|
||||
// the file asynchronously in the background; if it fails the minimal file
|
||||
// still survives for debugging.
|
||||
if (pressure === 'hard' || pressure === 'critical') {
|
||||
void this.diagnosticsDumper.dump(pressure);
|
||||
}
|
||||
|
||||
this.executeCleanup(recommendation);
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user