qwen-code/scripts/prepare-package.js

211 lines
6.1 KiB
JavaScript
Raw Permalink Normal View History

/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Prepares the bundled CLI package for npm publishing
* This script adds publishing metadata (package.json, README, LICENSE) to dist/
* All runtime assets (cli.js, vendor/, *.sb) are already in dist/ from the bundle step
*/
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, '..');
const distDir = path.join(rootDir, 'dist');
const cliBundlePath = path.join(distDir, 'cli.js');
const vendorDir = path.join(distDir, 'vendor');
// Verify dist directory and bundle exist
if (!fs.existsSync(distDir)) {
console.error('Error: dist/ directory not found');
console.error('Please run "npm run bundle" first');
process.exit(1);
}
if (!fs.existsSync(cliBundlePath)) {
console.error(`Error: Bundle not found at ${cliBundlePath}`);
console.error('Please run "npm run bundle" first');
process.exit(1);
}
if (!fs.existsSync(vendorDir)) {
console.error(`Error: Vendor directory not found at ${vendorDir}`);
console.error('Please run "npm run bundle" first');
process.exit(1);
}
const bundledDocsDir = path.join(distDir, 'bundled', 'qc-helper', 'docs');
if (!fs.existsSync(bundledDocsDir)) {
console.error(`Error: Bundled docs not found at ${bundledDocsDir}`);
console.error('Please run "npm run bundle" first');
process.exit(1);
}
// Copy README and LICENSE
console.log('Copying documentation files...');
const filesToCopy = ['README.md', 'LICENSE'];
for (const file of filesToCopy) {
const sourcePath = path.join(rootDir, file);
const destPath = path.join(distDir, file);
if (fs.existsSync(sourcePath)) {
fs.copyFileSync(sourcePath, destPath);
console.log(`Copied ${file}`);
} else {
console.warn(`Warning: ${file} not found at ${sourcePath}`);
}
}
// Copy locales folder
console.log('Copying locales folder...');
const localesSourceDir = path.join(
rootDir,
'packages',
'cli',
'src',
'i18n',
'locales',
);
const localesDestDir = path.join(distDir, 'locales');
if (fs.existsSync(localesSourceDir)) {
// Recursive copy function
function copyRecursiveSync(src, dest) {
const stats = fs.statSync(src);
if (stats.isDirectory()) {
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest, { recursive: true });
}
const entries = fs.readdirSync(src);
for (const entry of entries) {
const srcPath = path.join(src, entry);
const destPath = path.join(dest, entry);
copyRecursiveSync(srcPath, destPath);
}
} else {
fs.copyFileSync(src, dest);
}
}
copyRecursiveSync(localesSourceDir, localesDestDir);
console.log('Copied locales folder');
} else {
console.warn(`Warning: locales folder not found at ${localesSourceDir}`);
}
// Copy extensions folder
console.log('Copying extension examples folder...');
const extensionExamplesDir = path.join(
rootDir,
'packages',
'cli',
'src',
'commands',
'extensions',
'examples',
);
const extensionExamplesDestDir = path.join(distDir, 'examples');
if (fs.existsSync(extensionExamplesDir)) {
// Recursive copy function
function copyRecursiveSync(src, dest) {
const stats = fs.statSync(src);
if (stats.isDirectory()) {
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest, { recursive: true });
}
const entries = fs.readdirSync(src);
for (const entry of entries) {
const srcPath = path.join(src, entry);
const destPath = path.join(dest, entry);
copyRecursiveSync(srcPath, destPath);
}
} else {
fs.copyFileSync(src, dest);
}
}
copyRecursiveSync(extensionExamplesDir, extensionExamplesDestDir);
console.log('Copied extension examples folder');
} else {
console.warn(
`Warning: extension examples folder not found at ${extensionExamplesDir}`,
);
}
// Copy package.json from root and modify it for publishing
console.log('Creating package.json for distribution...');
const rootPackageJson = JSON.parse(
fs.readFileSync(path.join(rootDir, 'package.json'), 'utf-8'),
);
// Create a clean package.json for the published package
const distPackageJson = {
name: rootPackageJson.name,
version: rootPackageJson.version,
description:
rootPackageJson.description || 'Qwen Code - AI-powered coding assistant',
repository: rootPackageJson.repository,
type: 'module',
main: 'cli.js',
bin: {
qwen: 'cli.js',
},
files: [
'cli.js',
perf(cli): code-split lowlight to cut startup V8 parse cost (#4070) * perf(cli): code-split lowlight to cut startup V8 parse cost Move the syntax-highlight engine out of the synchronously-parsed cli.js entry into a separately-emitted chunk and load it via dynamic import on the first code-block render. Until the chunk arrives, code blocks render as plain text; the next React commit of the surrounding subtree picks up the highlighted version, so users never see incorrect highlighting โ€“ just an imperceptibly later transition for the very first code block. Mechanics: - esbuild config: switch entry to outdir + splitting:true so that `await import('lowlight')` produces an actual on-disk chunk that's only parsed by V8 when first needed. - esbuild-shims: rename injected __dirname/__filename to qwen-prefixed symbols + use `define` to redirect free references. Previous inject collided with vendored libraries (yargs) that ship their own `var __dirname` ESM-compat polyfill once splitting flattens chunks. - prepare-package: include the new chunks/ directory in the published package's files list. - CodeColorizer: keep the public colorize{Code,Line} signatures and HAST rendering identical; on first call when the chunk hasn't loaded it returns the plain line and fires the dynamic import via a tiny standalone loader module. - lowlightLoader (new): isolates the lazy-load surface to a module with zero transitive imports (no themeManager, settings, or core). This lets test-setup prime the cache without dragging the whole UI module graph into every test file, which was observed to perturb theme and settings test outcomes when CodeColorizer was imported directly. - test-setup: await loadLowlight() once via the standalone loader so synchronous snapshot tests see the highlighted output deterministically. Measurements (real $HOME, n=15 interleaved A/B vs main HEAD, macOS): | Metric | Before (meanยฑsd ms) | After (meanยฑsd ms) | ฮ” | t | p | | ------------------ | ------------------- | ------------------ | -------- | ------ | -------- | | firstByte (wall) | 1633.5 ยฑ 88.7 | 1475.8 ยฑ 73.3 | -157.7 | 5.31 | 1.33e-5 | | idle (wall) | 2048.7 ยฑ 93.6 | 1902.3 ยฑ 80.2 | -146.3 | 4.60 | 8.71e-5 | | cli.js size | 25 MB | 6.9 MB | -18.1 MB | โ€” | โ€” | Both metrics clear the +50ms-or-10% Welch's t-test bar by an order of magnitude. cli.js drops 72%; total payload (cli.js + chunks/) is similar but only cli.js is parsed at module-eval time, which is the phase that dominates the user-visible startup gap. How to validate: npm run bundle ls dist/ # cli.js + chunks/lowlight-*.js node dist/cli.js -y # interactive UI still renders Generated with AI Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(cli): resolve chunk-relative sibling paths under esbuild splitting With `splitting: true`, esbuild hoists modules with shared dependencies into `dist/chunks/`. Three modules derived runtime paths from `import.meta.url` assuming they were co-located with `cli.js`; once hoisted, `path.dirname(fileURLToPath(import.meta.url))` resolved to `dist/chunks/` and sibling-asset lookups silently missed: - `skill-manager.ts`: bundledSkillsDir โ†’ `dist/chunks/bundled` (actual `dist/bundled/`). The `existsSync` guard swallowed the miss, dropping all four bundled skills (`/review`, `/qc-helper`, `/batch`, `/loop`) with no user-visible signal. - `ripgrepUtils.ts`: `getBuiltinRipgrep()` โ†’ `dist/chunks/vendor/...`. Falls back to system rg if installed, otherwise null on minimal hosts โ€” degrading grep to the slow internal scanner. - `i18n/index.ts`: `getBuiltinLocalesDir()` โ†’ `dist/chunks/locales`. User-visible behavior survives via the static glob import in `tryImportBundledTranslations`, but the loose-on-disk override path is dead. Each module now strips a trailing `chunks` segment when present, so the lookup resolves under `dist/`. In source / transpiled modes the basename is never `chunks`, so the fallback is a no-op. Also: - Add `chunks` to `DIST_REQUIRED_PATHS` in `create-standalone-package.js` so a regressed bundle that produces only `cli.js` fails the pre-packaging check instead of shipping a broken archive. - Expand `esbuild-shims.js` header so future contributors understand that `__qwen_filename` / `__qwen_dirname` always resolve to the shim's chunk file (dist/chunks/) and that sibling-asset lookups must strip the `chunks` segment. Reported by claude-opus-4-7 via Qwen Code /qreview on #4070. Generated with AI Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * perf(cli): prefetch lowlight from AppContainer + harden loader Three follow-ups to the lowlight code-split: - AppContainer fires `loadLowlight()` from a mount effect so the dynamic import is already in flight before any code block needs colorizing. Without this, code blocks committed to ink's append-only `<Static>` region before the import resolves stay plain text for the rest of the session โ€” Static can only be re-rendered via `refreshStatic`, which is not wired to lowlight load completion. Common reachable paths: short `--prompt -p` runs that finalize quickly, Ctrl+C- cancelled first turns, and the first-paint history replay on `--resume`. The startup parse-cost win is preserved (V8 still parses off the critical path). - `lowlightLoader.ts` latches the first import failure so subsequent calls short-circuit to a rejected promise instead of re-attempting `import('lowlight')` on every keystroke. The colorizer already falls back to plain text on miss; recovery requires a fresh process anyway. - `test-setup.ts` wraps the top-level `await loadLowlight()` in try/catch. A transient import failure no longer crashes the entire vitest run โ€” tests that hit a code block render the plain-text fallback and surface a warning. - `CodeColorizer.tsx` header comment updated to point at the AppContainer prefetch instead of claiming first-paint always sees a loaded instance. Reported by DeepSeek/deepseek-v4-pro and claude-opus-4-7 via Qwen Code /review and /qreview on #4070. Generated with AI Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * refactor(bundle): extract resolveBundleDir helper, apply to extensions/new Centralises the `chunks/` strip pattern that three sites (`i18n/index.ts`, `skills/skill-manager.ts`, `utils/ripgrepUtils.ts`) each duplicated after the round-3 fix in d581da04d. The implicit coupling to `esbuild.config.js`'s `chunkNames: 'chunks/[name]-[hash]'` now lives in a single helper (`packages/core/src/utils/bundlePaths.ts`), so a future rename only needs updating in one place. Also applies the same anchor to `commands/extensions/new.ts:EXAMPLES_PATH`. That module is currently bundled into `cli.js` (so the strip is a no-op today), but `qwen extensions new --help` always reads the examples directory in its yargs `builder` โ€” confirmed against the built bundle that the lookup hits `dist/examples/` (sibling of `cli.js`). Using the helper future-proofs against esbuild later hoisting the module into a shared chunk, where the bare `__dirname`/`import.meta.url` lookup would silently break the command for every end user. While here, surface lowlight-load failures from `AppContainer`'s prefetch effect to the debug channel (`debugLogger.warn`) instead of swallowing them silently. The loader already latches failures permanently, so this fires at most once per session; `CodeColorizer` continues to fall back to plain text on miss, so user-visible behaviour is unchanged. Generated with AI Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(bundle): restore __filename shadow in ripgrepUtils; harden lowlight loader Round-4 review (wenshao 2026-05-13 13:12) flagged five issues in the recent code-split work. This commit addresses all of them. CRITICAL โ€” `packages/core/src/utils/ripgrepUtils.ts`: the round-3 `resolveBundleDir` refactor removed the local `__filename` declaration but `getBuiltinRipgrep` still references bare `__filename` to decide how many `..` segments to walk. In `npm run dev` (tsx, ESM) `__filename` is undefined so the function throws `ReferenceError`. In the bundle esbuild's `define` rewrites it to `__qwen_filename` (the shim chunk path), which is the wrong string but happens to short-circuit to `levelsUp = 0` โ€” accidentally correct only because the chunk-path string never contains `path.join('src', 'utils')`. Reproduced via tsx: `__filename is not defined`; fixed by re-introducing the explicit local shadow plus a comment explaining why centralising both helpers into `resolveBundleDir` cannot replace the per-file shadow. `packages/cli/src/ui/utils/lowlightLoader.ts`: the previous permanent `lowlightFailed` latch left syntax highlighting dead for the entire process lifetime on transient errors (EMFILE, antivirus locks, slow-disk-after-wake). Replaced with a 30-second cooldown โ€” within the window subsequent calls return the cached rejection synchronously (keeps the per-render short-circuit that protects against permanently-broken installs); after the cooldown the next call retries the dynamic import. Exposes `isLowlightCoolingDown()` so render-hot callers can also skip duplicate failure logging. `packages/cli/src/ui/utils/CodeColorizer.tsx`: hoisted `loadLowlight()` + log out of the per-line render loop into a single `ensureLowlightLoading()` call at the top of `colorizeCode`. In the failure case this collapses hundreds of duplicate debug entries (one per line) to one per block. The instance is now passed down to `highlightAndRenderLine` as a parameter. `packages/core/src/utils/bundlePaths.ts` + `esbuild.config.js`: exposed `BUNDLE_CHUNK_DIR = 'chunks'` as a named constant and updated `esbuild.config.js` to interpolate the same name into `chunkNames` (plus an explicit "MUST stay in sync" comment). Renaming on one side without the other now stands out at review time. Also expanded the `define` comment with a contributor-facing warning describing exactly why bare `__dirname` / `__filename` in source files becomes the shim chunk path, and pointing future contributors at the `fileURLToPath(import.meta.url)` shadow pattern (and `resolveBundleDir` for sibling-asset lookups). Verified: - typecheck (all 4 workspaces): clean - packages/core tests: 7747 passing (no regressions) - packages/cli tests: only the pre-existing `useAtCompletion.test.ts` filesystem-order failures remain (confirmed against `git stash`) - `npm run bundle` succeeds; `node dist/cli.js --version` returns `0.15.10`; `node dist/cli.js --help` renders normally - `npx tsx <call getBuiltinRipgrep>` now returns the vendored path instead of throwing `ReferenceError` Generated with AI Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(bundle): validate lowlight API shape; sync doc-comment drift; add tests - lowlightLoader: validate runtime shape of createLowlight() before the `as Lowlight` cast so an upstream API rename routes through the cooldown latch instead of silently degrading every code block to plain text. - bundlePaths: correct doc comment โ€” esbuild.config.js maintains its own `BUNDLE_CHUNK_DIR` constant rather than importing this one (it runs before any TS compile step). - AppContainer: update prefetch-failure comment to reference the cooldown symbols (`LOWLIGHT_RETRY_COOLDOWN_MS` / `lowlightLastFailureAt`) that replaced the removed `lowlightFailed` latch. - New unit tests covering the lowlightLoader state machine (success, in-flight dedup, shape mismatch, cooldown skip, post-cooldown retry) and `resolveBundleDir`'s strip-only-on-exact-match contract. * test(bundlePaths): use path.resolve for Windows-compatible absolute paths CI failure on Windows: the new `resolveBundleDir` tests built expected values with `path.join(path.sep, ...)` (e.g. `\tmp\dist`), but `pathToFileURL` resolves drive-less paths against the current drive on Windows. The URL -> `fileURLToPath` round-trip returned `D:\tmp\dist`, while the expectation stayed `\tmp\dist`, tripping all three new assertions. Switched both the URL source and the expected value to a single `path.resolve(path.sep, ...)` anchor per test so both sides absorb whatever the platform considers absolute. POSIX behaviour is unchanged (`/tmp/dist` -> `/tmp/dist`). --------- Co-authored-by: ็งฆๅฅ‡ <gary.gq@alibaba-inc.com> Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
2026-05-15 17:26:18 +08:00
'chunks',
'vendor',
'*.sb',
'README.md',
'LICENSE',
'locales',
'bundled',
],
config: rootPackageJson.config,
dependencies: {},
optionalDependencies: {
'@lydell/node-pty': '1.2.0-beta.10',
'@lydell/node-pty-darwin-arm64': '1.2.0-beta.10',
'@lydell/node-pty-darwin-x64': '1.2.0-beta.10',
'@lydell/node-pty-linux-x64': '1.2.0-beta.10',
'@lydell/node-pty-win32-arm64': '1.2.0-beta.10',
'@lydell/node-pty-win32-x64': '1.2.0-beta.10',
2026-02-10 14:38:01 +08:00
'@teddyzhu/clipboard': '0.0.5',
'@teddyzhu/clipboard-darwin-arm64': '0.0.5',
'@teddyzhu/clipboard-darwin-x64': '0.0.5',
'@teddyzhu/clipboard-linux-x64-gnu': '0.0.5',
'@teddyzhu/clipboard-linux-arm64-gnu': '0.0.5',
'@teddyzhu/clipboard-win32-x64-msvc': '0.0.5',
'@teddyzhu/clipboard-win32-arm64-msvc': '0.0.5',
},
engines: rootPackageJson.engines,
};
fs.writeFileSync(
path.join(distDir, 'package.json'),
JSON.stringify(distPackageJson, null, 2) + '\n',
);
console.log('\nโœ… Package prepared for publishing at dist/');
console.log('\nPackage structure:');
// Use Node.js to list directory contents (cross-platform)
const distFiles = fs.readdirSync(distDir);
for (const file of distFiles) {
const filePath = path.join(distDir, file);
const stats = fs.statSync(filePath);
const size = stats.isDirectory() ? '<DIR>' : formatBytes(stats.size);
console.log(` ${size.padEnd(12)} ${file}`);
}
function formatBytes(bytes) {
if (bytes < 1024) return `${bytes}B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
}