-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathcopy_bundle_assets.js
More file actions
456 lines (429 loc) · 16.8 KB
/
Copy pathcopy_bundle_assets.js
File metadata and controls
456 lines (429 loc) · 16.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { copyFileSync, existsSync, mkdirSync, statSync } from 'node:fs';
import {
dirname,
join,
basename,
resolve,
relative,
sep,
extname,
} from 'node:path';
import { createHash } from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { glob } from 'glob';
import fs from 'node:fs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const defaultRoot = join(__dirname, '..');
// Exported for `scripts/tests/review-source-digest.test.ts`, which holds the
// skill root's digest allowlist up to everything this rule lets the copier
// ship — a file the copier would carry but the digest cannot see is a
// staleness check with a blind spot.
export const BUNDLED_SKILL_TEST_FILE_RE =
/\.(?:test|spec)\.(?:d\.)?[cm]?[jt]sx?(?:\.map)?$/;
/**
* The digest of every review source this bundle was built from.
*
* A staleness hint, not an integrity control: an unsigned file beside the
* bundle, which anyone who can write `dist/` can write.
*
* Kept in step with `stale-bundle.ts`, which re-derives it the same way — and
* duplicated rather than shared, because this script runs before the package
* it would import has been built. `scripts/tests/review-source-digest.test.ts`
* is what holds the two equal; nothing here is imported from there.
*
* Tests and fixtures are excluded on both sides: esbuild follows imports from
* the CLI entry, neither is reachable that way, and a warning fired by an
* edit to a file the bundle cannot contain is the false positive this check
* exists not to produce. DESIGN.md is excluded for the same reason: the
* copier below deliberately does not ship it.
*/
// Mirrors NOT_BUNDLED_RE / NOT_BUNDLED_DIR / NOT_BUNDLED_FILE /
// NOT_BUNDLED_SKILL_FILE / DIGESTED_EXTENSIONS in stale-bundle.ts; the
// parity test keeps them equal.
const NOT_BUNDLED_RE = /\.(?:test|spec)\.(?:d\.)?[cm]?[jt]sx?$/;
const NOT_BUNDLED_DIR = new Set(['__fixtures__', '__snapshots__']);
const NOT_BUNDLED_FILE = new Set(['test-utils.ts']);
const NOT_BUNDLED_SKILL_FILE = new Set(['DESIGN.md']);
const DIGESTED_EXTENSIONS = {
code: new Set([
'.ts',
'.tsx',
'.mts',
'.cts',
'.js',
'.mjs',
'.json',
'.jsx',
]),
skill: new Set(['.md']),
};
function isDigestedFile(kind, name) {
if (!DIGESTED_EXTENSIONS[kind].has(extname(name))) return false;
if (kind !== 'code') return !NOT_BUNDLED_SKILL_FILE.has(name);
return !NOT_BUNDLED_RE.test(name) && !NOT_BUNDLED_FILE.has(name);
}
export function reviewSourceDigestForBuild(root) {
const cliCommands = join(root, 'packages', 'cli', 'src', 'commands');
const roots = [
{ path: join(cliCommands, 'review'), kind: 'code' },
{ path: join(cliCommands, 'review.ts'), kind: 'code' },
// Mirrors the lease root `reviewSourceRoots` (stale-bundle.ts) adds; the
// repo-tree case in review-source-digest.test.ts holds the two equal.
{
path: join(
root,
'packages',
'cli',
'src',
'services',
'review-worktree-lease.ts',
),
kind: 'code',
},
{
path: join(
root,
'packages',
'core',
'src',
'skills',
'bundled',
'review',
),
kind: 'skill',
},
];
const files = [];
// `listed` is true when the parent walk saw this path as a directory a
// moment ago: if it cannot be listed now, the tree changed underneath the
// walk and the measurement is incomplete, where a root that was never there
// has simply nothing to say.
const walk = (dir, kind, listed) => {
let entries;
try {
entries = fs.readdirSync(dir, { withFileTypes: true });
} catch {
// `readdirSync` failing says only "this path did not list" — the reason
// decides everything, and `statSync` states it directly, where inferring
// "this is a file" from `ENOTDIR` assumes every platform's libuv maps
// the case the same way, and a one-sided divergence would drop
// `review.ts` from one digest and not the other — a bundle that is
// byte-for-byte correct warning on every review, forever, on that
// platform alone. A directory that could not be listed (EACCES, or
// vanished after the parent listed it) throws instead of being skipped:
// a digest over the survivors would be stamped as the truth, and every
// review from then on would report stale against it.
let stats;
try {
stats = fs.statSync(dir);
} catch (error) {
if (listed) {
throw new Error(`${dir} vanished while being walked`);
}
// A root that was never there is silent only on ENOENT — EACCES/EPERM
// is a tree that IS there but cannot be measured, and skipping it
// would stamp a digest over the surviving roots, which the runtime
// twin (`sourceFilesUnder` in stale-bundle.ts) marks incomplete: a
// parity gap that would accuse a byte-for-byte correct bundle once
// the tree becomes readable again.
if (error?.code !== 'ENOENT') {
throw new Error(`${dir} could not be measured`);
}
return;
}
if (stats.isDirectory()) {
throw new Error(`${dir} could not be listed`);
}
if (stats.isFile() && isDigestedFile(kind, basename(dir))) {
files.push(dir);
}
return;
}
for (const e of entries) {
const full = join(dir, e.name);
if (e.isDirectory()) {
if (kind !== 'code' || !NOT_BUNDLED_DIR.has(e.name))
walk(full, kind, true);
} else if (e.isFile() && isDigestedFile(kind, e.name)) {
files.push(full);
}
}
};
for (const r of roots) walk(r.path, r.kind, false);
if (files.length === 0)
return { digest: undefined, count: 0, newest: undefined };
const hash = createHash('sha256');
let newest;
for (const file of files.sort()) {
const { mtimeMs } = fs.statSync(file);
if (!newest || mtimeMs > newest.mtimeMs) newest = { file, mtimeMs };
hash.update(relative(root, file).split(sep).join('/'));
hash.update('\0');
hash.update(fs.readFileSync(file));
hash.update('\0');
}
return { digest: hash.digest('hex'), count: files.length, newest };
}
function stampReviewSourceDigest(root, distDir) {
const stampPath = join(distDir, 'review-sources.sha256');
// Every refusal below removes an existing stamp first. Leaving an older
// stamp beside a newer bundle is a weaker form of the misdescription the
// refusal exists to avoid, and `unmeasured` is the state each refusal means.
const refuse = (why) => {
try {
fs.rmSync(stampPath, { recursive: true, force: true });
} catch (error) {
// `force` swallows only ENOENT; a permission or lock error on the old
// stamp must not kill the bundle step after every asset is in place.
// Leaving it is the lesser outcome: at worst a later "stale" notice
// whose rebuild advice is still correct.
console.warn(
`Could not remove the stale source digest at ${stampPath}: ` +
(error instanceof Error ? error.message : error),
);
}
// `warn`, not `log`: the runtime message sends its reader back to this
// build's output, and a refusal hidden among plain logs is not what they
// are being sent to find.
console.warn(why);
};
// Never fatal. This is the last step of the copier, so a file vanishing
// mid-walk would fail the bundle after every asset was already in place —
// and a missing stamp is only `unmeasured`, which the runtime check already
// treats as an acceptable answer.
let digest, count, newest;
try {
({ digest, count, newest } = reviewSourceDigestForBuild(root));
} catch (error) {
refuse(
`Could not read the review sources; skipped the source digest: ${
error instanceof Error ? error.message : error
}`,
);
return;
}
if (!digest) {
refuse('No review sources found; skipped the source digest.');
return;
}
// The digest describes the tree as the COPIER sees it, and the copier runs
// after esbuild — so a source edited after the bundle was written, or this
// script run on its own, would stamp a `cli.js` built from something else.
// That is the only direction where silence is affirmatively wrong instead
// of merely uninformative: every other gap here degrades to `unmeasured`.
// Timestamps are the wrong tool for judging staleness and the right one for
// judging whether this stamp can be honest at all, so refuse rather than
// stamp. The anchor is the bundle's WRITE time, not the build's start:
// esbuild reads the sources during the graph walk, before it writes, so an
// edit landing while the build is in flight is older than this anchor,
// passes the gate, and is what the copier digests — a stamp honest about
// the tree and not about the bundle. That window is not covered here.
const bundlePath = join(distDir, 'cli.js');
let builtAt;
try {
builtAt = fs.statSync(bundlePath).mtimeMs;
} catch {
refuse('No bundle found to stamp; skipped the source digest.');
return;
}
const newestSource = newest.mtimeMs;
if (newestSource > builtAt) {
refuse(
`A review source is newer than ${bundlePath}; skipped the source digest ` +
`rather than stamp a bundle it may not describe. Run \`npm run bundle\` ` +
`to rebuild the bundle and the stamp together.`,
);
return;
}
try {
fs.writeFileSync(stampPath, digest);
} catch (error) {
refuse(
`Could not write the source digest; skipped the stamp: ${
error instanceof Error ? error.message : error
}`,
);
return;
}
console.log(`Stamped the review source digest over ${count} files.`);
}
export function copyBundleAssets({ root = defaultRoot } = {}) {
const distDir = join(root, 'dist');
const coreVendorDir = join(root, 'packages', 'core', 'vendor');
// Create the dist directory if it doesn't exist
if (!existsSync(distDir)) {
mkdirSync(distDir);
}
// Find and copy all .sb files from packages to the root of the dist directory
const sbFiles = glob.sync('packages/**/*.sb', { cwd: root });
for (const file of sbFiles) {
copyFileSync(join(root, file), join(distDir, basename(file)));
}
console.log('Copied sandbox profiles to dist/');
// Copy vendor directory (contains ripgrep binaries)
console.log('Copying vendor directory...');
if (existsSync(coreVendorDir)) {
const destVendorDir = join(distDir, 'vendor');
copyRecursiveSync(coreVendorDir, destVendorDir);
console.log('Copied vendor directory to dist/');
} else {
console.warn(`Warning: Vendor directory not found at ${coreVendorDir}`);
}
// Copy bundled skills (e.g. /review) so they are available at runtime.
// In the esbuild bundle, import.meta.url resolves to dist/cli.js, so
// SkillManager looks for bundled skills at dist/bundled/.
const bundledSkillsDir = join(
root,
'packages',
'core',
'src',
'skills',
'bundled',
);
if (existsSync(bundledSkillsDir)) {
const destBundledDir = join(distDir, 'bundled');
fs.rmSync(destBundledDir, { recursive: true, force: true });
copyRecursiveSync(bundledSkillsDir, destBundledDir, {
// DESIGN.md files are maintainer design narratives, not runtime inputs;
// shipping one would hand a review a ~125 KB read_file target that
// outweighs the context the slimmed skill saves.
skipEntry: (entry) =>
isBundledSkillTestFile(entry) || entry === 'DESIGN.md',
});
console.log('Copied bundled skills to dist/bundled/');
} else {
console.warn(
`Warning: Bundled skills directory not found at ${bundledSkillsDir}`,
);
}
// Copy user docs into qc-helper bundled skill so it can reference them at runtime.
// The qc-helper skill reads docs from a `docs/` subdirectory relative to its own
// directory. In the esbuild bundle this becomes dist/bundled/qc-helper/docs/.
const userDocsDir = join(root, 'docs', 'users');
if (existsSync(userDocsDir)) {
const destDocsDir = join(distDir, 'bundled', 'qc-helper', 'docs');
copyRecursiveSync(userDocsDir, destDocsDir);
console.log('Copied docs/users/ to dist/bundled/qc-helper/docs/');
} else {
console.warn(`Warning: User docs directory not found at ${userDocsDir}`);
}
// Copy builtin locales so bundled dist/cli.js can load UI translations at runtime.
// Published packages already include these via prepare-package.js; bundle output
// should mirror that behavior for local `node dist/cli.js` runs.
const localesDir = join(root, 'packages', 'cli', 'src', 'i18n', 'locales');
if (existsSync(localesDir)) {
const destLocalesDir = join(distDir, 'locales');
copyRecursiveSync(localesDir, destLocalesDir);
console.log('Copied builtin locales to dist/locales/');
} else {
console.warn(`Warning: Locales directory not found at ${localesDir}`);
}
// Copy extension templates so bundled dist/cli.js can scaffold
// `/extensions new` from the runtime examples directory.
const extensionExamplesDir = join(
root,
'packages',
'cli',
'src',
'commands',
'extensions',
'examples',
);
if (existsSync(extensionExamplesDir)) {
const destExtensionExamplesDir = join(distDir, 'examples');
copyRecursiveSync(extensionExamplesDir, destExtensionExamplesDir);
console.log('Copied extension examples to dist/examples/');
} else {
console.warn(
`Warning: Extension examples directory not found at ${extensionExamplesDir}`,
);
}
// Copy the built Web Shell SPA (index.html + assets/) so the bundled
// `qwen serve` can serve the browser UI at its root path. The library
// build outputs (dist/index.js, dist/types) are for npm consumers and are
// intentionally NOT copied. Source only exists after the web-shell
// workspace is built (npm run build); when absent (e.g. a --cli-only
// build, or bundling without a prior full build) we warn and skip so the
// bundle step never fails — the daemon then runs API-only at runtime.
const webShellDistDir = join(root, 'packages', 'web-shell', 'dist');
const webShellIndexHtml = join(webShellDistDir, 'index.html');
const webShellAssetsDir = join(webShellDistDir, 'assets');
if (existsSync(webShellIndexHtml) && existsSync(webShellAssetsDir)) {
const destWebShellDir = join(distDir, 'web-shell');
mkdirSync(destWebShellDir, { recursive: true });
copyFileSync(webShellIndexHtml, join(destWebShellDir, 'index.html'));
copyRecursiveSync(webShellAssetsDir, join(destWebShellDir, 'assets'));
console.log('Copied Web Shell UI to dist/web-shell/');
} else {
console.warn(
`Warning: Web Shell assets not found at ${webShellDistDir}; ` +
'dist/web-shell/ will be absent and `qwen serve` runs API-only. ' +
'Run a full `npm run build` before bundling to include the UI.',
);
}
// Stamp what the review sources looked like at build time. `/review` drives
// the bundle, not the working tree, so a review command edited after this
// point takes no effect — and without a record of what was built, the run
// cannot tell and neither can its reader. Compared, not trusted: the check
// reads this and re-derives the digest from the tree.
stampReviewSourceDigest(root, distDir);
console.log('\n✅ All bundle assets copied to dist/');
}
if (isDirectRun()) {
copyBundleAssets();
}
function isDirectRun() {
return process.argv[1]
? fileURLToPath(import.meta.url) === resolve(process.argv[1])
: false;
}
/**
* Recursively copy directory
*/
function copyRecursiveSync(src, dest, options = {}) {
if (!existsSync(src)) {
return;
}
const stats = statSync(src);
if (stats.isDirectory()) {
if (!existsSync(dest)) {
mkdirSync(dest, { recursive: true });
}
const entries = fs.readdirSync(src);
for (const entry of entries) {
if (entry === '.DS_Store' || options.skipEntry?.(entry)) {
continue;
}
const srcPath = join(src, entry);
const destPath = join(dest, entry);
copyRecursiveSync(srcPath, destPath, options);
}
} else {
copyFileSync(src, dest);
// Preserve execute permissions for binaries
const srcStats = statSync(src);
if (srcStats.mode & 0o111) {
fs.chmodSync(dest, srcStats.mode);
}
}
}
function isBundledSkillTestFile(fileName) {
return BUNDLED_SKILL_TEST_FILE_RE.test(fileName);
}