Compare commits

...

5 Commits

Author SHA1 Message Date
yiliang114
b3a6fbaa5c fix(core): point memory diagnostics suggestion at /compress (the actual command)
The suggestion text told users to run /compact, which does not exist in
this repository — the actual command is /compress (see compressCommand.ts).
Pointing users at a nonexistent slash command in a diagnostics report
makes the suggestion unactionable.
2026-06-01 17:01:53 +08:00
yiliang114
33de27e616 fix(core): two-phase memory diagnostics write to survive OOM
Two critical issues from review:

1. The async collectMemoryDiagnostics() runs before writeFileSync, but it
   spawns a `ps` subprocess and reads /proc — fork() under critical memory
   pressure can fail or be OOM-killed, leaving no file on disk despite the
   "cheap write first" design comment.

2. dumpCount and lastDumpTime were updated after the await, so concurrent
   dumps (e.g. hard→critical escalation) would both pass the cap/cooldown
   guards and overwrite each other.

Fix:

- Reserve the dump slot synchronously (++dumpCount, lastDumpTime) before
  any await, so concurrent calls correctly hit the cap.
- Phase 1: synchronously write a minimal JSON (process.memoryUsage +
  v8.getHeapStatistics, no fork/exec) with collectionComplete=false.
  Because async functions execute synchronously up to the first await,
  this is guaranteed on disk before the caller's next statement runs.
- Phase 2: enrich with full diagnostics asynchronously and overwrite the
  file with collectionComplete=true. If Phase 2 crashes, the minimal
  Phase 1 file still survives for debugging.

Tests updated for the two-phase write and gain two new cases covering the
sync-Phase-1 guarantee and the synchronous slot reservation.
2026-06-01 16:36:00 +08:00
yiliang114
8f776f7c7c test(core): use path.join in memoryDiagnosticsDumper test for cross-platform
The assertion hard-coded POSIX separators ('/tmp/test-project/diagnostics/'),
which fails on Windows where path.join produces backslashes. Build the
expected substring with path.join + path.sep so it matches the dumper's
actual output on every platform.
2026-06-01 13:41:23 +08:00
yiliang114
4aa1584c65 ci: retrigger CI after Windows flaky failure 2026-05-31 18:13:44 +08:00
yiliang114
5a8747332b feat(core): auto-dump memory diagnostics to disk on pressure detection
When the MemoryPressureMonitor (#4403) detects hard or critical pressure,
write a lightweight diagnostics JSON to .qwen/<project>/diagnostics/ before
running cleanup. The file survives even if a subsequent operation triggers
OOM, giving maintainers actionable data from bug reports without requiring
the user to manually run /doctor memory after a crash.

Design follows Claude Code's heapDumpService approach: write the cheap JSON
first (small write, won't OOM), heavy snapshot second. Diagnostics include
process memory stats, V8 heap stats, session history size, and an actionable
suggestion for the user.

Per-session limits: max 3 dumps, 30s cooldown between dumps.

Closes #4651
2026-05-31 16:01:25 +08:00
3 changed files with 439 additions and 0 deletions

View File

@ -0,0 +1,250 @@
/**
* @license
* Copyright 2026 Qwen Team
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { MemoryDiagnosticsDumper } from './memoryDiagnosticsDumper.js';
import type { Config } from '../config/config.js';
vi.mock('node:fs', () => ({
mkdirSync: vi.fn(),
writeFileSync: vi.fn(),
}));
vi.mock('node:v8', () => ({
getHeapStatistics: vi.fn().mockReturnValue({
heap_size_limit: 4_096_000_000,
total_heap_size: 2_048_000_000,
used_heap_size: 1_800_000_000,
total_available_size: 2_000_000_000,
}),
}));
vi.mock('../utils/memoryDiagnostics.js', () => ({
collectMemoryDiagnostics: vi.fn().mockResolvedValue({
timestamp: '2026-05-31T00:00:00.000Z',
memoryUsage: {
rss: 2_000_000_000,
heapUsed: 1_800_000_000,
heapTotal: 2_048_000_000,
external: 50_000_000,
arrayBuffers: 10_000_000,
},
v8HeapStats: {
heapSizeLimit: 4_096_000_000,
totalHeapSize: 2_048_000_000,
usedHeapSize: 1_800_000_000,
},
}),
}));
function createMockConfig(overrides: Partial<Record<string, unknown>> = {}) {
return {
getSessionId: vi.fn().mockReturnValue('test-session-id-12345678'),
getCliVersion: vi.fn().mockReturnValue('0.17.0'),
getGeminiClient: vi.fn().mockReturnValue({
getChat: () => ({
getHistoryLength: () => 500,
}),
}),
storage: {
getProjectDir: vi.fn().mockReturnValue('/tmp/test-project'),
},
...overrides,
} as unknown as Config;
}
describe('MemoryDiagnosticsDumper', () => {
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
it('writes diagnostics JSON on first dump', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
const result = await dumper.dump('hard');
expect(result).toBeDefined();
expect(result!.trigger).toBe('hard');
expect(result!.filePath).toContain(
path.join('/tmp/test-project', 'diagnostics') + path.sep,
);
expect(result!.filePath).toContain('memory-test-ses');
expect(fs.mkdirSync).toHaveBeenCalledWith(
expect.stringContaining('diagnostics'),
{ recursive: true },
);
// Two-phase write: Phase 1 (minimal) + Phase 2 (full)
expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
const phase1Content = JSON.parse(
vi.mocked(fs.writeFileSync).mock.calls[0][1] as string,
);
expect(phase1Content.trigger).toBe('hard');
expect(phase1Content.dumpNumber).toBe(1);
expect(phase1Content.collectionComplete).toBe(false);
expect(phase1Content.memoryUsage).toBeDefined();
expect(phase1Content.v8HeapStats).toBeDefined();
const phase2Content = JSON.parse(
vi.mocked(fs.writeFileSync).mock.calls[1][1] as string,
);
expect(phase2Content.trigger).toBe('hard');
expect(phase2Content.dumpNumber).toBe(1);
expect(phase2Content.collectionComplete).toBe(true);
expect(phase2Content.memoryUsage.rss).toBe(2_000_000_000);
expect(phase2Content.session.historyEntries).toBe(500);
expect(phase2Content.suggestion).toContain('/compress');
});
it('respects per-session cap of 3 dumps', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
// Bypass cooldown by mocking Date.now
let mockNow = 1000000;
vi.spyOn(Date, 'now').mockImplementation(() => {
mockNow += 60_000;
return mockNow;
});
const r1 = await dumper.dump('hard');
const r2 = await dumper.dump('critical');
const r3 = await dumper.dump('hard');
const r4 = await dumper.dump('critical');
expect(r1).toBeDefined();
expect(r2).toBeDefined();
expect(r3).toBeDefined();
expect(r4).toBeUndefined();
// 3 successful dumps × 2 writes each (Phase 1 + Phase 2)
expect(fs.writeFileSync).toHaveBeenCalledTimes(6);
});
it('respects cooldown between dumps', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
const mockNow = 1000000;
vi.spyOn(Date, 'now').mockReturnValue(mockNow);
const r1 = await dumper.dump('hard');
const r2 = await dumper.dump('hard');
expect(r1).toBeDefined();
expect(r2).toBeUndefined();
// 1 successful dump × 2 writes (Phase 1 + Phase 2)
expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
});
it('resets state on new session', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
let mockNow = 1000000;
vi.spyOn(Date, 'now').mockImplementation(() => {
mockNow += 60_000;
return mockNow;
});
await dumper.dump('hard');
await dumper.dump('hard');
await dumper.dump('hard');
// Cap reached
const r4 = await dumper.dump('hard');
expect(r4).toBeUndefined();
// Reset
dumper.resetForNewSession();
const r5 = await dumper.dump('critical');
expect(r5).toBeDefined();
expect(r5!.trigger).toBe('critical');
});
it('includes critical suggestion for critical pressure', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
await dumper.dump('critical');
// Phase 2 (full payload) is the second write
const writtenContent = JSON.parse(
vi.mocked(fs.writeFileSync).mock.calls[1][1] as string,
);
expect(writtenContent.suggestion).toContain('critically high');
expect(writtenContent.collectionComplete).toBe(true);
});
it('writes Phase 1 synchronously before any await (survives crash during Phase 2)', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
// Fire dump() but do not await — Phase 1 must have already written to disk
// because async functions execute synchronously up to the first await.
const promise = dumper.dump('hard');
// At this point Phase 2 has not run yet (its await is pending), but Phase 1
// must have completed its writeFileSync call.
expect(fs.writeFileSync).toHaveBeenCalledTimes(1);
const phase1Content = JSON.parse(
vi.mocked(fs.writeFileSync).mock.calls[0][1] as string,
);
expect(phase1Content.collectionComplete).toBe(false);
await promise;
// Phase 2 has now overwritten the file
expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
});
it('reserves slot synchronously to prevent concurrent dumps from bypassing cap', async () => {
const config = createMockConfig();
const dumper = new MemoryDiagnosticsDumper(config);
let mockNow = 1000000;
vi.spyOn(Date, 'now').mockImplementation(() => {
mockNow += 60_000;
return mockNow;
});
// Fire 4 concurrent dumps without awaiting between them. The synchronous
// slot reservation must enforce the cap of 3 even though all 4 calls happen
// before any of them complete their async Phase 2.
const results = await Promise.all([
dumper.dump('hard'),
dumper.dump('hard'),
dumper.dump('hard'),
dumper.dump('hard'),
]);
const successful = results.filter((r) => r !== undefined);
expect(successful).toHaveLength(3);
expect(results[3]).toBeUndefined();
});
it('handles missing geminiClient gracefully', async () => {
const config = createMockConfig({
getGeminiClient: vi.fn().mockReturnValue(null),
});
const dumper = new MemoryDiagnosticsDumper(config);
const result = await dumper.dump('hard');
expect(result).toBeDefined();
// Phase 2 (full payload) is the second write
const writtenContent = JSON.parse(
vi.mocked(fs.writeFileSync).mock.calls[1][1] as string,
);
expect(writtenContent.session.available).toBe(false);
});
});

View File

@ -0,0 +1,175 @@
/**
* @license
* Copyright 2026 Qwen Team
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Writes a lightweight memory diagnostics JSON to disk when the memory
* pressure monitor detects hard or critical pressure. The file survives
* a subsequent OOM crash, giving maintainers actionable data from bug
* reports without requiring the user to manually run `/doctor memory`.
*
* Design: diagnostics JSON is written BEFORE any expensive operation
* (like heap snapshots) so it lands on disk even if the process crashes
* during the heavier step.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as v8 from 'node:v8';
import { collectMemoryDiagnostics } from '../utils/memoryDiagnostics.js';
import { createDebugLogger } from '../utils/debugLogger.js';
import { getErrorMessage } from '../utils/errors.js';
import type { Config } from '../config/config.js';
const debugLogger = createDebugLogger('MEMORY_DUMP');
/** Maximum dumps per session to avoid disk flooding. */
const MAX_DUMPS_PER_SESSION = 3;
/** Minimum interval between dumps (ms). */
const MIN_DUMP_INTERVAL_MS = 30_000;
export interface MemoryDumpResult {
filePath: string;
trigger: string;
}
export class MemoryDiagnosticsDumper {
private dumpCount = 0;
private lastDumpTime = 0;
constructor(private readonly config: Config) {}
/**
* Resets session-scoped state. Called when a new session starts.
*/
resetForNewSession(): void {
this.dumpCount = 0;
this.lastDumpTime = 0;
}
/**
* Writes a diagnostics snapshot to disk if within per-session limits.
*
* Uses a two-phase write strategy:
* - Phase 1 (synchronous): writes a minimal JSON with process.memoryUsage()
* and v8.getHeapStatistics() — no fork/exec, so it lands on disk even
* under extreme memory pressure.
* - Phase 2 (async): collects full diagnostics (may spawn subprocesses)
* and overwrites the file with the complete payload. If Phase 2 crashes,
* Phase 1's file still survives for debugging.
*
* Slot is reserved synchronously before any await to prevent concurrent
* invocations from bypassing the cap/cooldown guards.
*/
async dump(
trigger: 'hard' | 'critical',
): Promise<MemoryDumpResult | undefined> {
if (this.dumpCount >= MAX_DUMPS_PER_SESSION) {
debugLogger.debug(
`Skipping dump: session cap reached (${MAX_DUMPS_PER_SESSION})`,
);
return undefined;
}
const now = Date.now();
if (now - this.lastDumpTime < MIN_DUMP_INTERVAL_MS) {
debugLogger.debug('Skipping dump: cooldown not elapsed');
return undefined;
}
// Reserve slot synchronously to prevent race between concurrent dumps
const dumpNumber = ++this.dumpCount;
this.lastDumpTime = now;
try {
const diagnosticsDir = this.ensureDiagnosticsDir();
const sessionId = this.config.getSessionId();
const timestamp = new Date()
.toISOString()
.replace(/:/g, '-')
.replace(/\./g, '_');
const fileName = `memory-${sessionId.slice(0, 8)}-${timestamp}.json`;
const filePath = path.join(diagnosticsDir, fileName);
// Phase 1: synchronous minimal write — survives crash during Phase 2
const minimalPayload = {
trigger,
dumpNumber,
timestamp: new Date().toISOString(),
memoryUsage: process.memoryUsage(),
v8HeapStats: v8.getHeapStatistics(),
session: this.collectSessionStats(),
suggestion: this.getSuggestion(trigger),
collectionComplete: false,
};
fs.writeFileSync(
filePath,
JSON.stringify(minimalPayload, null, 2),
'utf8',
);
debugLogger.info(
`Phase 1 diagnostics written to ${filePath} (trigger=${trigger}, dump #${dumpNumber})`,
);
// Phase 2: full collection (may fork subprocesses — risky under pressure)
const diagnostics = await collectMemoryDiagnostics({
sessionId,
qwenVersion: this.config.getCliVersion(),
});
const fullPayload = {
trigger,
dumpNumber,
...diagnostics,
session: this.collectSessionStats(),
suggestion: this.getSuggestion(trigger),
collectionComplete: true,
};
fs.writeFileSync(filePath, JSON.stringify(fullPayload, null, 2), 'utf8');
debugLogger.info(
`Phase 2 diagnostics written to ${filePath} (trigger=${trigger}, dump #${dumpNumber})`,
);
return { filePath, trigger };
} catch (err) {
// Slot stays consumed — a failed write should not open the door to more
// attempts that would likely also fail under the same pressure conditions.
debugLogger.error(
`Failed to write memory diagnostics: ${getErrorMessage(err)}`,
);
return undefined;
}
}
private ensureDiagnosticsDir(): string {
const projectDir = this.config.storage.getProjectDir();
const diagnosticsDir = path.join(projectDir, 'diagnostics');
fs.mkdirSync(diagnosticsDir, { recursive: true });
return diagnosticsDir;
}
private collectSessionStats(): Record<string, unknown> {
try {
const geminiClient = this.config.getGeminiClient?.();
if (!geminiClient) return { available: false };
const historyLength = geminiClient.getChat?.()?.getHistoryLength?.() ?? 0;
return {
historyEntries: historyLength,
};
} catch {
return { available: false };
}
}
private getSuggestion(trigger: 'hard' | 'critical'): string {
if (trigger === 'critical') {
return 'Memory is critically high. Consider running /compress or starting a fresh session to avoid OOM.';
}
return 'Memory pressure detected. Running /compress may help reduce memory usage.';
}
}

View File

@ -11,6 +11,7 @@ import { getHeapStatistics } from 'node:v8';
import { createDebugLogger } from '../utils/debugLogger.js';
import { getErrorMessage } from '../utils/errors.js';
import type { Config } from '../config/config.js';
import { MemoryDiagnosticsDumper } from './memoryDiagnosticsDumper.js';
// Types
@ -104,6 +105,7 @@ export class MemoryPressureMonitor extends EventEmitter {
private consecutiveIneffectiveAggressiveCleanups = 0;
private cleanupGeneration = 0;
private readonly effectiveMemoryLimit: number;
private readonly diagnosticsDumper: MemoryDiagnosticsDumper;
constructor(coreConfig: Config, pressureConfig?: MemoryPressureConfig) {
super();
@ -111,6 +113,7 @@ export class MemoryPressureMonitor extends EventEmitter {
this.config = { ...(pressureConfig ?? DEFAULT_PRESSURE_CONFIG) };
validateMemoryPressureConfig(this.config);
this.effectiveMemoryLimit = this.computeEffectiveMemoryLimit();
this.diagnosticsDumper = new MemoryDiagnosticsDumper(coreConfig);
const heapSizeLimit = getHeapStatistics().heap_size_limit;
debugLogger.info(
`Effective memory limit: ${formatMiB(this.effectiveMemoryLimit)} MiB; ` +
@ -142,6 +145,7 @@ export class MemoryPressureMonitor extends EventEmitter {
resetForNewSession(): void {
this.cleanupGeneration++;
this.resetConsecutiveFailures();
this.diagnosticsDumper.resetForNewSession();
this.cleanupInProgress = false;
this.activeCleanupAction = 'none';
this.queuedCleanupRecommendation = undefined;
@ -196,6 +200,16 @@ export class MemoryPressureMonitor extends EventEmitter {
return;
}
// Write diagnostics to disk before cleanup. dump() uses a two-phase strategy:
// Phase 1 (synchronous, before the first await) writes a minimal JSON via
// writeFileSync — guaranteed to complete before executeCleanup starts because
// async functions run synchronously up to the first await. Phase 2 enriches
// the file asynchronously in the background; if it fails the minimal file
// still survives for debugging.
if (pressure === 'hard' || pressure === 'critical') {
void this.diagnosticsDumper.dump(pressure);
}
this.executeCleanup(recommendation);
}