Skip to content

Commit bbe7eb5

Browse files
author
Nicholas Pape
authored
Rush: reduce how often we need to run generate (microsoft#543)
* Update the shrinkwrap file to be able to reuse dependencies from a different temp package * Modify the shrinkwrap file to add a reused dependency * Changefile * Rename API * Fixup API * PR Feedback * Fix lint error * PR Feedback
1 parent 5911186 commit bbe7eb5

File tree

7 files changed

+201
-79
lines changed

7 files changed

+201
-79
lines changed

‎apps/rush-lib/src/cli/logic/InstallManager.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,11 +268,6 @@ export default class InstallManager {
268268
}
269269
}
270270

271-
// Either way, resync the temporary shrinkwrap file.
272-
// Copy (or delete) common\npm-shrinkwrap.json --> common\temp\npm-shrinkwrap.json
273-
this.syncFile(this._rushConfiguration.committedShrinkwrapFilename,
274-
this._rushConfiguration.tempShrinkwrapFilename);
275-
276271
// Also copy down the committed .npmrc file, if there is one
277272
// "common\config\rush\.npmrc" --> "common\temp\.npmrc"
278273
const committedNpmrcPath: string = path.join(this._rushConfiguration.commonRushConfigFolder, '.npmrc');
@@ -391,7 +386,7 @@ export default class InstallManager {
391386
tempPackageJson.dependencies![pair.packageName] = pair.packageVersion;
392387

393388
if (shrinkwrapFile) {
394-
if (!shrinkwrapFile.hasCompatibleDependency(pair.packageName, pair.packageVersion,
389+
if (!shrinkwrapFile.tryEnsureCompatibleDependency(pair.packageName, pair.packageVersion,
395390
rushProject.tempProjectName)) {
396391
console.log(colors.yellow(
397392
wrap(`${os.EOL}The NPM shrinkwrap file is missing "${pair.packageName}"`
@@ -469,6 +464,14 @@ export default class InstallManager {
469464
const commonPackageJsonFilename: string = path.join(this._rushConfiguration.commonTempFolder,
470465
RushConstants.packageJsonFilename);
471466

467+
if (shrinkwrapFile) {
468+
// Resync the temporary shrinkwrap file.
469+
// Copy (or delete) common\npm-shrinkwrap.json --> common\temp\npm-shrinkwrap.json
470+
shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapFilename);
471+
} else {
472+
fsx.removeSync(this._rushConfiguration.tempShrinkwrapFilename);
473+
}
474+
472475
// Don't update the file timestamp unless the content has changed, since "rush install"
473476
// will consider this timestamp
474477
JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true });

‎apps/rush-lib/src/cli/logic/base/BaseShrinkwrapFile.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// See LICENSE in the project root for license information.
33

44
import * as colors from 'colors';
5+
import * as fsx from 'fs-extra';
56
import * as semver from 'semver';
67
import npmPackageArg = require('npm-package-arg');
78

@@ -21,6 +22,13 @@ export abstract class BaseShrinkwrapFile {
2122
return undefined;
2223
}
2324

25+
/**
26+
* Serializes and saves the shrinkwrap file to specified location
27+
*/
28+
public save(filePath: string): void {
29+
fsx.writeFileSync(filePath, this.serialize());
30+
}
31+
2432
/**
2533
* Returns true if the shrinkwrap file includes a top-level package that would satisfy the specified
2634
* package name and SemVer version range
@@ -37,10 +45,24 @@ export abstract class BaseShrinkwrapFile {
3745

3846
/**
3947
* Returns true if the shrinkwrap file includes a package that would satisfiying the specified
40-
* package name and SemVer version range for a given temp project.
48+
* package name and SemVer version range. By default, the dependencies are resolved by looking
49+
* at the root of the node_modules folder described by the shrinkwrap file. However, if
50+
* tempProjectName is specified, then the resolution will start in that subfolder.
51+
*
52+
* Consider this example:
53+
*
54+
* - node_modules\
55+
* - temp-project\
56+
* - lib-a@1.2.3
57+
* - lib-b@1.0.0
58+
* - lib-b@2.0.0
59+
*
60+
* In this example, hasCompatibleDependency("lib-b", ">= 1.1.0", "temp-project") would fail
61+
* because it finds lib-b@1.0.0 which does not satisfy the pattern ">= 1.1.0".
4162
*/
42-
public hasCompatibleDependency(dependencyName: string, versionRange: string, tempProjectName: string): boolean {
43-
const dependencyVersion: string | undefined = this.getDependencyVersion(dependencyName, tempProjectName);
63+
public tryEnsureCompatibleDependency(dependencyName: string, versionRange: string, tempProjectName: string): boolean {
64+
const dependencyVersion: string | undefined =
65+
this.tryEnsureDependencyVersion(dependencyName, tempProjectName, versionRange);
4466
if (!dependencyVersion) {
4567
return false;
4668
}
@@ -54,8 +76,10 @@ export abstract class BaseShrinkwrapFile {
5476
*/
5577
public abstract getTempProjectNames(): ReadonlyArray<string>;
5678

57-
protected abstract getDependencyVersion(dependencyName: string, tempProjectName?: string): string | undefined;
79+
protected abstract tryEnsureDependencyVersion(dependencyName: string,
80+
tempProjectName: string, versionRange: string): string | undefined;
5881
protected abstract getTopLevelDependencyVersion(dependencyName: string): string | undefined;
82+
protected abstract serialize(): string;
5983

6084
protected _getTempProjectNames(dependencies: { [key: string]: {} } ): ReadonlyArray<string> {
6185
const result: string[] = [];

‎apps/rush-lib/src/cli/logic/npm/NpmShrinkwrapFile.ts‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import * as fsx from 'fs-extra';
22
import * as os from 'os';
33

4+
import {
5+
JsonFile
6+
} from '@microsoft/node-core-library';
7+
48
import {
59
BaseShrinkwrapFile
610
} from '../base/BaseShrinkwrapFile';
@@ -45,47 +49,43 @@ export class NpmShrinkwrapFile extends BaseShrinkwrapFile {
4549
return this._getTempProjectNames(this._shrinkwrapJson.dependencies);
4650
}
4751

52+
protected serialize(): string {
53+
return JsonFile.stringify(this._shrinkwrapJson);
54+
}
55+
4856
protected getTopLevelDependencyVersion(dependencyName: string): string | undefined {
49-
return this.getDependencyVersion(dependencyName);
57+
// First, check under tempProjectName, as this is the first place "rush link" looks.
58+
const dependencyJson: IShrinkwrapDependencyJson | undefined =
59+
NpmShrinkwrapFile.tryGetValue(this._shrinkwrapJson.dependencies, dependencyName);
60+
61+
if (!dependencyJson) {
62+
return undefined;
63+
}
64+
65+
return dependencyJson.version;
5066
}
5167

5268
/**
53-
* Returns true if the shrinkwrap file includes a package that would satisfiying the specified
54-
* package name and SemVer version range. By default, the dependencies are resolved by looking
55-
* at the root of the node_modules folder described by the shrinkwrap file. However, if
56-
* tempProjectName is specified, then the resolution will start in that subfolder.
57-
*
58-
* Consider this example:
59-
*
60-
* - node_modules\
61-
* - temp-project\
62-
* - lib-a@1.2.3
63-
* - lib-b@1.0.0
64-
* - lib-b@2.0.0
65-
*
66-
* In this example, hasCompatibleDependency("lib-b", ">= 1.1.0", "temp-project") would fail
67-
* because it finds lib-b@1.0.0 which does not satisfy the pattern ">= 1.1.0".
69+
* @param dependencyName the name of the dependency to get a version for
70+
* @param tempProjectName the name of the temp project to check for this dependency
71+
* @param versionRange Not used, just exists to satisfy abstract API contract
6872
*/
69-
protected getDependencyVersion(dependencyName: string, tempProjectName?: string): string | undefined {
73+
protected tryEnsureDependencyVersion(dependencyName: string,
74+
tempProjectName: string,
75+
versionRange: string): string | undefined {
7076

7177
// First, check under tempProjectName, as this is the first place "rush link" looks.
7278
let dependencyJson: IShrinkwrapDependencyJson | undefined = undefined;
7379

74-
if (tempProjectName) {
75-
const tempDependency: IShrinkwrapDependencyJson | undefined = NpmShrinkwrapFile.tryGetValue(
76-
this._shrinkwrapJson.dependencies, tempProjectName);
77-
if (tempDependency && tempDependency.dependencies) {
78-
dependencyJson = NpmShrinkwrapFile.tryGetValue(tempDependency.dependencies, dependencyName);
79-
}
80+
const tempDependency: IShrinkwrapDependencyJson | undefined = NpmShrinkwrapFile.tryGetValue(
81+
this._shrinkwrapJson.dependencies, tempProjectName);
82+
if (tempDependency && tempDependency.dependencies) {
83+
dependencyJson = NpmShrinkwrapFile.tryGetValue(tempDependency.dependencies, dependencyName);
8084
}
8185

8286
// Otherwise look at the root of the shrinkwrap file
8387
if (!dependencyJson) {
84-
dependencyJson = NpmShrinkwrapFile.tryGetValue(this._shrinkwrapJson.dependencies, dependencyName);
85-
}
86-
87-
if (!dependencyJson) {
88-
return undefined;
88+
return this.getTopLevelDependencyVersion(dependencyName);
8989
}
9090

9191
return dependencyJson.version;

‎apps/rush-lib/src/cli/logic/pnpm/PnpmShrinkwrapFile.ts‎

Lines changed: 104 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as fsx from 'fs-extra';
22
import * as yaml from 'js-yaml';
33
import * as os from 'os';
4+
import * as semver from 'semver';
45

56
import Utilities from '../../../utilities/Utilities';
67
import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile';
@@ -114,7 +115,15 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
114115
}
115116

116117
/**
117-
* abstract
118+
* Serializes the PNPM Shrinkwrap file
119+
*/
120+
protected serialize(): string {
121+
return yaml.safeDump(this._shrinkwrapJson, {
122+
sortKeys: true
123+
});
124+
}
125+
126+
/**
118127
* Gets the version number from the list of top-level dependencies in the "dependencies" section
119128
* of the shrinkwrap file
120129
*/
@@ -123,10 +132,13 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
123132
}
124133

125134
/**
126-
* abstract
127-
* Gets the resolved version number of a dependency for a specific temp project
135+
* Gets the resolved version number of a dependency for a specific temp project.
136+
* For PNPM, we can reuse the version that another project is using.
137+
* Note that this function modifies the shrinkwrap data.
128138
*/
129-
protected getDependencyVersion(dependencyName: string, tempProjectName: string): string | undefined {
139+
protected tryEnsureDependencyVersion(dependencyName: string,
140+
tempProjectName: string,
141+
versionRange: string): string | undefined {
130142
// PNPM doesn't have the same advantage of NPM, where we can skip generate as long as the
131143
// shrinkwrap file puts our dependency in either the top of the node_modules folder
132144
// or underneath the package we are looking at.
@@ -135,22 +147,103 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
135147
// Because of this, we actually need to check for a version that this package is directly
136148
// linked to.
137149

138-
// Example: "project1"
139-
const unscopedTempProjectName: string = Utilities.parseScopedPackageName(tempProjectName).name;
140-
const tempProjectDependencyKey: string = `file:projects/${unscopedTempProjectName}.tgz`;
150+
const tempProjectDependencyKey: string = this._getTempProjectKey(tempProjectName);
151+
const packageDescription: IShrinkwrapDependencyJson | undefined =
152+
this._getPackageDescription(tempProjectDependencyKey);
153+
if (!packageDescription) {
154+
return undefined;
155+
}
141156

142-
const packageDescription: IShrinkwrapDependencyJson | undefined
143-
= BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.packages, tempProjectDependencyKey);
157+
if (!packageDescription.dependencies.hasOwnProperty(dependencyName)) {
158+
if (versionRange) {
159+
// this means the current temp project doesn't provide this dependency,
160+
// however, we may be able to use a different version. we prefer the latest version
161+
let latestVersion: string | undefined = undefined;
144162

145-
if (!packageDescription || !packageDescription.dependencies) {
163+
this.getTempProjectNames().forEach((otherTempProject: string) => {
164+
const otherVersion: string | undefined = this._getDependencyVersion(dependencyName, otherTempProject);
165+
if (otherVersion && semver.satisfies(otherVersion, versionRange)) {
166+
if (!latestVersion || semver.gt(otherVersion, latestVersion)) {
167+
latestVersion = otherVersion;
168+
}
169+
}
170+
});
171+
172+
if (latestVersion) {
173+
// go ahead and fixup the shrinkwrap file to point at this
174+
const dependencies: { [key: string]: string } | undefined =
175+
this._shrinkwrapJson.packages[tempProjectDependencyKey].dependencies || {};
176+
dependencies[dependencyName] = latestVersion;
177+
this._shrinkwrapJson.packages[tempProjectDependencyKey].dependencies = dependencies;
178+
179+
return latestVersion;
180+
}
181+
}
182+
183+
return undefined;
184+
}
185+
186+
return this._normalizeDependencyVersion(dependencyName, packageDescription.dependencies[dependencyName]);
187+
}
188+
189+
private constructor(shrinkwrapJson: IShrinkwrapYaml) {
190+
super();
191+
this._shrinkwrapJson = shrinkwrapJson;
192+
193+
// Normalize the data
194+
if (!this._shrinkwrapJson.registry) {
195+
this._shrinkwrapJson.registry = '';
196+
}
197+
if (!this._shrinkwrapJson.dependencies) {
198+
this._shrinkwrapJson.dependencies = { };
199+
}
200+
if (!this._shrinkwrapJson.specifiers) {
201+
this._shrinkwrapJson.specifiers = { };
202+
}
203+
if (!this._shrinkwrapJson.packages) {
204+
this._shrinkwrapJson.packages = { };
205+
}
206+
}
207+
208+
/**
209+
* Returns the version of a dependency being used by a given project
210+
*/
211+
private _getDependencyVersion(dependencyName: string, tempProjectName: string): string | undefined {
212+
const tempProjectDependencyKey: string = this._getTempProjectKey(tempProjectName);
213+
const packageDescription: IShrinkwrapDependencyJson | undefined =
214+
this._getPackageDescription(tempProjectDependencyKey);
215+
if (!packageDescription) {
146216
return undefined;
147217
}
148218

149219
if (!packageDescription.dependencies.hasOwnProperty(dependencyName)) {
150220
return undefined;
151221
}
152222

153-
const version: string = packageDescription.dependencies[dependencyName];
223+
return this._normalizeDependencyVersion(dependencyName, packageDescription.dependencies[dependencyName]);
224+
}
225+
226+
/**
227+
* Gets the package description for a tempProject from the shrinkwrap file.
228+
*/
229+
private _getPackageDescription(tempProjectDependencyKey: string): IShrinkwrapDependencyJson | undefined {
230+
const packageDescription: IShrinkwrapDependencyJson | undefined
231+
= BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.packages, tempProjectDependencyKey);
232+
233+
if (!packageDescription || !packageDescription.dependencies) {
234+
return undefined;
235+
}
236+
237+
return packageDescription;
238+
}
239+
240+
private _getTempProjectKey(tempProjectName: string): string {
241+
// Example: "project1"
242+
const unscopedTempProjectName: string = Utilities.parseScopedPackageName(tempProjectName).name;
243+
return `file:projects/${unscopedTempProjectName}.tgz`;
244+
}
245+
246+
private _normalizeDependencyVersion(dependencyName: string, version: string): string | undefined {
154247
// version will be either:
155248
// A - the version (e.g. "0.0.5")
156249
// B - a peer dep version (e.g. "/gulp-karma/0.0.5/karma@0.13.22"
@@ -175,23 +268,4 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile {
175268
return undefined;
176269
}
177270
}
178-
179-
private constructor(shrinkwrapJson: IShrinkwrapYaml) {
180-
super();
181-
this._shrinkwrapJson = shrinkwrapJson;
182-
183-
// Normalize the data
184-
if (!this._shrinkwrapJson.registry) {
185-
this._shrinkwrapJson.registry = '';
186-
}
187-
if (!this._shrinkwrapJson.dependencies) {
188-
this._shrinkwrapJson.dependencies = { };
189-
}
190-
if (!this._shrinkwrapJson.specifiers) {
191-
this._shrinkwrapJson.specifiers = { };
192-
}
193-
if (!this._shrinkwrapJson.packages) {
194-
this._shrinkwrapJson.packages = { };
195-
}
196-
}
197271
}

0 commit comments

Comments
 (0)