Skip to content

Commit df68f31

Browse files
lukekarryswraithgar
lukekarrys
authored and
wraithgar
committed
chore: update scripts
1 parent bd92aa0 commit df68f31

24 files changed

+851
-414
lines changed

‎scripts/bundle-and-gitignore-deps.js‎

Lines changed: 212 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,22 @@
11
const Arborist = require('@npmcli/arborist')
2-
const { resolve } = require('path')
3-
const ignore = resolve(__dirname, '../node_modules/.gitignore')
4-
const { writeFileSync } = require('fs')
5-
const pj = resolve(__dirname, '../package.json')
6-
const pkg = require(pj)
7-
const bundle = []
8-
const arb = new Arborist({ path: resolve(__dirname, '..') })
9-
const shouldIgnore = []
10-
11-
// disabling to get linting to pass, this file is going away soon
12-
// eslint-disable-next-line
13-
arb.loadVirtual().then(tree => {
14-
// eslint-disable-next-line
15-
for (const node of tree.children.values()) {
16-
const has = (obj, key) => Object.prototype.hasOwnProperty.call(obj, key)
17-
const nonProdWorkspace =
18-
node.isWorkspace && !(has(tree.package.dependencies, node.name))
19-
if (node.dev || nonProdWorkspace) {
20-
console.error('ignore', node.name)
21-
shouldIgnore.push(node.name)
22-
} else if (tree.edgesOut.has(node.name)) {
23-
console.error('BUNDLE', node.name)
24-
bundle.push(node.name)
25-
}
26-
}
27-
pkg.bundleDependencies = bundle.sort((a, b) => a.localeCompare(b, 'en'))
2+
const packlist = require('npm-packlist')
3+
const { join, relative } = require('path')
4+
const localeCompare = require('@isaacs/string-locale-compare')('en')
5+
const PackageJson = require('@npmcli/package-json')
6+
const { run, CWD, git, fs } = require('./util')
287

29-
const ignores = shouldIgnore.sort((a, b) => a.localeCompare(b, 'en'))
30-
.map(i => `/${i}`)
31-
.join('\n')
32-
const ignoreData = `# Automatically generated to ignore dev deps
33-
/.package-lock.json
8+
const ALWAYS_IGNORE = `
9+
.bin/
10+
.cache/
3411
package-lock.json
3512
CHANGELOG*
3613
changelog*
14+
ChangeLog*
15+
Changelog*
3716
README*
3817
readme*
18+
ReadMe*
19+
Readme*
3920
__pycache__
4021
.editorconfig
4122
.idea/
@@ -55,9 +36,204 @@ __pycache__
5536
.babelrc*
5637
.nyc_output
5738
.gitkeep
58-
59-
${ignores}
6039
`
61-
writeFileSync(ignore, ignoreData)
62-
writeFileSync(pj, JSON.stringify(pkg, 0, 2) + '\n')
63-
})
40+
41+
const lsIgnored = async (dir, { removeIgnoredFiles }) => {
42+
const files = await git(
43+
'ls-files',
44+
'--cached',
45+
'--ignored',
46+
`--exclude-standard`,
47+
dir,
48+
{ lines: true }
49+
)
50+
51+
if (removeIgnoredFiles) {
52+
for (const file of files) {
53+
await git('rm', file)
54+
}
55+
return []
56+
}
57+
58+
return files
59+
}
60+
61+
const getAllowedPaths = (files) => {
62+
// Get all files within node_modules and remove
63+
// the node_modules/ portion of the path for processing
64+
// since this list will go inside a gitignore at the
65+
// root of the node_modules dir
66+
const nmFiles = files
67+
.filter(f => f.startsWith('node_modules/'))
68+
.map(f => f.replace(/^node_modules\//, ''))
69+
.sort(localeCompare)
70+
71+
class AllowSegments {
72+
#segments
73+
#usedSegments
74+
75+
constructor (pathSegments, rootSegments = []) {
76+
// Copy strings with spread operator since we mutate these arrays
77+
this.#segments = [...pathSegments]
78+
this.#usedSegments = [...rootSegments]
79+
}
80+
81+
get next () {
82+
return this.#segments[0]
83+
}
84+
85+
get remaining () {
86+
return this.#segments
87+
}
88+
89+
get used () {
90+
return this.#usedSegments
91+
}
92+
93+
use () {
94+
const segment = this.#segments.shift()
95+
this.#usedSegments.push(segment)
96+
return segment
97+
}
98+
99+
allowContents ({ use = true, isDirectory = true } = {}) {
100+
if (use) {
101+
this.use()
102+
}
103+
// Allow a previously ignored directy
104+
// Important: this should NOT have a trailing
105+
// slash if we are not sure it is a directory.
106+
// Since a dep can be a directory or a symlink and
107+
// a trailing slash in a .gitignore file
108+
// tells git to treat it only as a directory
109+
return [`!/${this.used.join('/')}${isDirectory ? '/' : ''}`]
110+
}
111+
112+
allow ({ use = true } = {}) {
113+
if (use) {
114+
this.use()
115+
}
116+
// Allow a previously ignored directory but ignore everything inside
117+
return [
118+
...this.allowContents({ use: false, isDirectory: true }),
119+
`/${this.used.join('/')}/*`,
120+
]
121+
}
122+
}
123+
124+
const gatherAllows = (pathParts, usedParts) => {
125+
const ignores = []
126+
const segments = new AllowSegments(pathParts, usedParts)
127+
128+
if (segments.next) {
129+
// 1) Process scope segment of the path, if it has one
130+
if (segments.next.startsWith('@')) {
131+
// For scoped deps we need to allow the entire scope dir
132+
// due to how gitignore works. Without this the gitignore will
133+
// never look to allow our bundled dep since the scope dir was ignored.
134+
// It ends up looking like this for `@colors/colors`:
135+
//
136+
// # Allow @colors dir
137+
// !/@colors/
138+
// # Ignore everything inside. This is safe because there is
139+
// # nothing inside a scope except other packages
140+
// !/colors/*
141+
//
142+
// Then later we will allow the specific dep inside that scope.
143+
// This way if a scope includes bundled AND unbundled deps,
144+
// we only allow the bundled ones.
145+
ignores.push(...segments.allow())
146+
}
147+
148+
// 2) Now we process the name segment of the path
149+
// and allow the dir and everything inside of it (like source code, etc)
150+
ignores.push(...segments.allowContents({ isDirectory: false }))
151+
152+
// 3) If we still have remaining segments and the next segment
153+
// is a nested node_modules directory...
154+
if (segments.next && segments.use() === 'node_modules') {
155+
ignores.push(
156+
// Allow node_modules and ignore everything inside of it
157+
// Set false here since we already "used" the node_modules path segment
158+
...segments.allow({ use: false }),
159+
// Repeat the process with the remaining path segments to include whatever is left
160+
...gatherAllows(segments.remaining, segments.used)
161+
)
162+
}
163+
}
164+
165+
return ignores
166+
}
167+
168+
const allowPaths = new Set()
169+
for (const file of nmFiles) {
170+
for (const allow of gatherAllows(file.split('/'))) {
171+
allowPaths.add(allow)
172+
}
173+
}
174+
175+
return [...allowPaths]
176+
}
177+
178+
const setBundleDeps = async () => {
179+
const pkg = await PackageJson.load(CWD)
180+
181+
pkg.update({
182+
bundleDependencies: Object.keys(pkg.content.dependencies).sort(localeCompare),
183+
})
184+
185+
await pkg.save()
186+
187+
return pkg.content.bundleDependencies
188+
}
189+
190+
/*
191+
This file sets what is checked in to node_modules. The root .gitignore file
192+
includes node_modules and this file writes an ignore file to
193+
node_modules/.gitignore. We ignore everything and then use a query to find all
194+
the bundled deps and allow each one of those explicitly.
195+
196+
Since node_modules can be nested we have to process each portion of the path and
197+
allow it while also ignoring everything inside of it, with the exception of a
198+
deps source. We have to do this since everything is ignored by default, and git
199+
will not allow a nested path if its parent has not also been allowed. BUT! We
200+
also have to ignore other things in those directories.
201+
*/
202+
const main = async ({ removeIgnoredFiles }) => {
203+
await setBundleDeps()
204+
205+
const arb = new Arborist({ path: CWD })
206+
const files = await arb.loadActual().then(packlist)
207+
208+
const ignoreFile = [
209+
'# Automatically generated to ignore everything except bundled deps',
210+
'# Ignore everything by default except this file',
211+
'/*',
212+
'!/.gitignore',
213+
'# Allow all bundled deps',
214+
...getAllowedPaths(files),
215+
'# Always ignore some specific patterns within any allowed package',
216+
...ALWAYS_IGNORE.trim().split('\n'),
217+
]
218+
219+
const NODE_MODULES = join(CWD, 'node_modules')
220+
const res = await fs.writeFile(join(NODE_MODULES, '.gitignore'), ignoreFile.join('\n'))
221+
222+
// After we write the file we have to check if any of the paths already checked in
223+
// inside node_modules are now going to be ignored. If we find any then fail with
224+
// a list of paths that will need to have `git rm` run on them.
225+
const trackedAndIgnored = await lsIgnored(NODE_MODULES, { removeIgnoredFiles })
226+
227+
if (trackedAndIgnored.length) {
228+
const message = [
229+
'The following files are checked in to git but will now be ignored.',
230+
`Rerun this script with \`--remove-ignored-files\` to remove them.`,
231+
...trackedAndIgnored.map(p => relative(NODE_MODULES, p)),
232+
].join('\n')
233+
throw new Error(message)
234+
}
235+
236+
return res
237+
}
238+
239+
run(main)

‎scripts/create-node-pr.js‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
const { join } = require('path')
2+
const fsp = require('fs/promises')
3+
const hgi = require('hosted-git-info')
4+
const semver = require('semver')
5+
const pacote = require('pacote')
6+
const log = require('proc-log')
7+
const tar = require('tar')
8+
const { cp, withTempDir } = require('@npmcli/fs')
9+
const { CWD, run, spawn, git, fs, gh } = require('./util.js')
10+
11+
// this script expects node to already be cloned to a directory at the cli root named "node"
12+
const NODE_DIR = join(CWD, 'node')
13+
const gitNode = spawn.create('git', { cwd: NODE_DIR })
14+
15+
const createNodeTarball = async ({ mani, registryOnly, tag, dir: extractDir }) => {
16+
const tarball = join(extractDir, 'npm-node.tgz')
17+
await pacote.tarball.file(mani._from, tarball, { resolved: mani._resolved })
18+
19+
if (registryOnly) {
20+
// a future goal is to only need files from the published tarball for
21+
// inclusion in node. in that case, we'd be able to remove everything after
22+
// this line since we have already fetched the tarball
23+
return tarball
24+
}
25+
26+
// extract tarball to current dir and delete original tarball
27+
await tar.x({ strip: 1, file: tarball, cwd: extractDir })
28+
await fs.rimraf(tarball)
29+
30+
// checkout the tag since we need to get files from source.
31+
await git.dirty()
32+
tag && await git('checkout', tag)
33+
for (const path of ['.npmrc', 'tap-snapshots/', 'test/']) {
34+
await cp(join(CWD, path), join(extractDir, path), { recursive: true })
35+
}
36+
37+
await tar.c({
38+
...pacote.DirFetcher.tarCreateOptions(mani),
39+
cwd: extractDir,
40+
file: tarball,
41+
}, ['.'])
42+
43+
return tarball
44+
}
45+
46+
const main = async (spec, opts) => withTempDir(CWD, async (tmpDir) => {
47+
const { dryRun, registryOnly, skipCheckout } = opts
48+
49+
const mani = await pacote.manifest(`npm@${spec}`, { preferOnline: true })
50+
51+
const head = {
52+
tag: `v${mani.version}`,
53+
branch: `npm-v${mani.version}`,
54+
host: hgi.fromUrl('npm/node'),
55+
message: `deps: upgrade npm to ${mani.version}`,
56+
}
57+
log.silly(head)
58+
59+
const tarball = await createNodeTarball({
60+
mani,
61+
dir: tmpDir,
62+
registryOnly,
63+
// the only reason this is optional is for testing when updating this script.
64+
// if we checkout an older tag, it won't have the updates we are testing.
65+
tag: skipCheckout ? null : head.tag,
66+
})
67+
68+
await fsp.access(NODE_DIR, fsp.constants.F_OK).catch(() => {
69+
throw new Error(`node repo must be checked out to \`${NODE_DIR}\` to continue`)
70+
})
71+
72+
const base = {
73+
// we used to send PRs sometimes for old versions to the 14.x staging
74+
// branch. this might not be needed anymore, but this is how we
75+
// would do it, if we needed to send a PR for backport fixes
76+
branch: semver.major(mani.version) <= 8 ? '14.x-staging' : 'main',
77+
remote: 'origin',
78+
host: hgi.fromUrl(await gitNode('remote', 'get-url', 'origin', { out: true })),
79+
}
80+
log.silly(base)
81+
82+
await gh('repo', 'fork', base.host.path(), '--org', head.host.user, { quiet: true, ok: true })
83+
await gitNode('fetch', base.remote)
84+
await gitNode('checkout', base.branch)
85+
await gitNode('reset', '--hard', `${base.remote}/${base.branch}`)
86+
await gitNode('branch', '-D', head.branch, { ok: true })
87+
await gitNode('checkout', '-b', head.branch)
88+
89+
const npmPath = join('deps', 'npm')
90+
const npmDir = join(NODE_DIR, npmPath)
91+
await fs.clean(npmDir)
92+
await tar.x({ strip: 1, file: tarball, cwd: npmDir })
93+
94+
await gitNode('add', '-A', npmPath)
95+
await gitNode('commit', '-m', head.message)
96+
await gitNode('rebase', '--whitespace', 'fix', base.branch)
97+
98+
await gitNode('remote', 'add', head.host.user, head.host.ssh(), { ok: true })
99+
await gitNode('push', head.host.user, head.branch, '--force')
100+
101+
const notes = await gh.json('release', 'view', head.tag, 'body')
102+
log.silly('body', notes)
103+
104+
const prArgs = [
105+
'pr', 'create',
106+
'-R', base.host.path(),
107+
'-B', base.branch,
108+
'-H', `${head.host.user}:${head.branch}`,
109+
'-t', head.message,
110+
]
111+
112+
if (dryRun) {
113+
log.info(`gh ${prArgs.join(' ')}`)
114+
const url = new URL(base.host.browse())
115+
const compare = `${base.branch}...${head.host.user}:${head.host.project}:${head.branch}`
116+
url.pathname += `/compare/${compare}`
117+
url.searchParams.set('expand', '1')
118+
return url.toString()
119+
}
120+
121+
return gh(...prArgs, '-F', '-', { cwd: NODE_DIR, input: notes, out: true })
122+
})
123+
124+
run(({ argv, ...opts }) => main(argv.remain[0], opts))

0 commit comments

Comments
 (0)