Skip to content

Move ReleasesApiConsumer to a separate file and do simple refactors #3188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
robertbrignull merged 11 commits into from
Jan 3, 2024
Merged

Move ReleasesApiConsumer to a separate file and do simple refactors #3188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2f9c8d1
Move Release and ReleaseAsset to their own files
robertbrignull Jan 3, 2024
90d72ce
Add more comments to Release and ReleaseAsset
robertbrignull Jan 3, 2024
bc7c956
Pull out GithubApiError and GithubRateLimitedError to a separate file
robertbrignull Jan 3, 2024
6d1cf08
Pull out ReleasesApiConsumer to its own file
robertbrignull Jan 3, 2024
337e5f8
Adjust comments in releases-api-consumer.ts
robertbrignull Jan 3, 2024
c575ede
Move releases-api-consumer.test.ts from no-workspace to unit-tests
robertbrignull Jan 3, 2024
b8cbd75
Use "private readonly" modifier on constructor args
robertbrignull Jan 3, 2024
9fd8c1c
Convert other fields to not use underscore prefix
robertbrignull Jan 3, 2024
426ab98
Move fields to top of class definition
robertbrignull Jan 3, 2024
6746b07
Merge ownerName and repositoryName into one field because we no longe…
robertbrignull Jan 3, 2024
1ce2b36
Move files to codeql-cli/distribution directory
robertbrignull Jan 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
328 changes: 11 additions & 317 deletions extensions/ql-vscode/src/codeql-cli/distribution.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import * as fetch from "node-fetch";
import { pathExists, mkdtemp, createWriteStream, remove } from "fs-extra";
import { tmpdir } from "os";
import { delimiter, dirname, join } from "path";
import * as semver from "semver";
import { URL } from "url";
import { ExtensionContext, Event } from "vscode";
import { DistributionConfig } from "../config";
import { extLogger } from "../common/logging/vscode";
Expand All @@ -27,6 +25,8 @@ import {
} from "../common/logging";
import { unzipToDirectoryConcurrently } from "../common/unzip-concurrently";
import { reportUnzipProgress } from "../common/vscode/unzip-progress";
import { Release } from "./distribution/release";
import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";

/**
* distribution.ts
Expand All @@ -36,30 +36,14 @@ import { reportUnzipProgress } from "../common/vscode/unzip-progress";
*/

/**
* Default value for the owner name of the extension-managed distribution on GitHub.
*
* We set the default here rather than as a default config value so that this default is invoked
* upon blanking the setting.
*/
const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";

/**
* Default value for the repository name of the extension-managed distribution on GitHub.
*
* We set the default here rather than as a default config value so that this default is invoked
* upon blanking the setting.
* Repository name with owner of the stable version of the extension-managed distribution on GitHub.
*/
const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-binaries";
const STABLE_DISTRIBUTION_REPOSITORY_NWO = "github/codeql-cli-binaries";

/**
* Owner name of the nightly version of the extension-managed distribution on GitHub.
* Repository name with owner of the nightly version of the extension-managed distribution on GitHub.
*/
const NIGHTLY_DISTRIBUTION_OWNER_NAME = "dsp-testing";

/**
* Repository name of the nightly version of the extension-managed distribution on GitHub.
*/
const NIGHTLY_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-nightlies";
const NIGHTLY_DISTRIBUTION_REPOSITORY_NWO = "dsp-testing/codeql-cli-nightlies";

/**
* Range of versions of the CLI that are compatible with the extension.
Expand Down Expand Up @@ -505,32 +489,22 @@ class ExtensionSpecificDistributionManager {

private createReleasesApiConsumer(): ReleasesApiConsumer {
return new ReleasesApiConsumer(
this.distributionOwnerName,
this.distributionRepositoryName,
this.distributionRepositoryNwo,
this.config.personalAccessToken,
);
}

private get distributionOwnerName(): string {
private get distributionRepositoryNwo(): string {
if (this.config.channel === "nightly") {
return NIGHTLY_DISTRIBUTION_OWNER_NAME;
return NIGHTLY_DISTRIBUTION_REPOSITORY_NWO;
} else {
return DEFAULT_DISTRIBUTION_OWNER_NAME;
}
}

private get distributionRepositoryName(): string {
if (this.config.channel === "nightly") {
return NIGHTLY_DISTRIBUTION_REPOSITORY_NAME;
} else {
return DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
return STABLE_DISTRIBUTION_REPOSITORY_NWO;
}
}

private get usingNightlyReleases(): boolean {
return (
this.distributionOwnerName === NIGHTLY_DISTRIBUTION_OWNER_NAME &&
this.distributionRepositoryName === NIGHTLY_DISTRIBUTION_REPOSITORY_NAME
this.distributionRepositoryNwo === NIGHTLY_DISTRIBUTION_REPOSITORY_NWO
);
}

Expand Down Expand Up @@ -588,173 +562,6 @@ class ExtensionSpecificDistributionManager {
private static readonly _codeQlExtractedFolderName = "codeql";
}

export class ReleasesApiConsumer {
constructor(
ownerName: string,
repoName: string,
personalAccessToken?: string,
) {
// Specify version of the GitHub API
this._defaultHeaders["accept"] = "application/vnd.github.v3+json";

if (personalAccessToken) {
this._defaultHeaders["authorization"] = `token ${personalAccessToken}`;
}

this._ownerName = ownerName;
this._repoName = repoName;
}

public async getLatestRelease(
versionRange: semver.Range | undefined,
orderBySemver = true,
includePrerelease = false,
additionalCompatibilityCheck?: (release: GithubRelease) => boolean,
): Promise<Release> {
const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases`;
const allReleases: GithubRelease[] = await (
await this.makeApiCall(apiPath)
).json();
const compatibleReleases = allReleases.filter((release) => {
if (release.prerelease && !includePrerelease) {
return false;
}

if (versionRange !== undefined) {
const version = semver.parse(release.tag_name);
if (
version === null ||
!semver.satisfies(version, versionRange, { includePrerelease })
) {
return false;
}
}

return (
!additionalCompatibilityCheck || additionalCompatibilityCheck(release)
);
});
// Tag names must all be parsable to semvers due to the previous filtering step.
const latestRelease = compatibleReleases.sort((a, b) => {
const versionComparison = orderBySemver
? semver.compare(semver.parse(b.tag_name)!, semver.parse(a.tag_name)!)
: b.id - a.id;
if (versionComparison !== 0) {
return versionComparison;
}
return b.created_at.localeCompare(a.created_at, "en-US");
})[0];
if (latestRelease === undefined) {
throw new Error(
"No compatible CodeQL CLI releases were found. " +
"Please check that the CodeQL extension is up to date.",
);
}
const assets: ReleaseAsset[] = latestRelease.assets.map((asset) => {
return {
id: asset.id,
name: asset.name,
size: asset.size,
};
});

return {
assets,
createdAt: latestRelease.created_at,
id: latestRelease.id,
name: latestRelease.name,
};
}

public async streamBinaryContentOfAsset(
asset: ReleaseAsset,
): Promise<fetch.Response> {
const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases/assets/${asset.id}`;

return await this.makeApiCall(apiPath, {
accept: "application/octet-stream",
});
}

protected async makeApiCall(
apiPath: string,
additionalHeaders: { [key: string]: string } = {},
): Promise<fetch.Response> {
const response = await this.makeRawRequest(
ReleasesApiConsumer._apiBase + apiPath,
Object.assign({}, this._defaultHeaders, additionalHeaders),
);

if (!response.ok) {
// Check for rate limiting
const rateLimitResetValue = response.headers.get("X-RateLimit-Reset");
if (response.status === 403 && rateLimitResetValue) {
const secondsToMillisecondsFactor = 1000;
const rateLimitResetDate = new Date(
parseInt(rateLimitResetValue, 10) * secondsToMillisecondsFactor,
);
throw new GithubRateLimitedError(
response.status,
await response.text(),
rateLimitResetDate,
);
}
throw new GithubApiError(response.status, await response.text());
}
return response;
}

private async makeRawRequest(
requestUrl: string,
headers: { [key: string]: string },
redirectCount = 0,
): Promise<fetch.Response> {
const response = await fetch.default(requestUrl, {
headers,
redirect: "manual",
});

const redirectUrl = response.headers.get("location");
if (
isRedirectStatusCode(response.status) &&
redirectUrl &&
redirectCount < ReleasesApiConsumer._maxRedirects
) {
const parsedRedirectUrl = new URL(redirectUrl);
if (parsedRedirectUrl.protocol !== "https:") {
throw new Error("Encountered a non-https redirect, rejecting");
}
if (parsedRedirectUrl.host !== "api.github.com") {
// Remove authorization header if we are redirected outside of the GitHub API.
//
// This is necessary to stream release assets since AWS fails if more than one auth
// mechanism is provided.
delete headers["authorization"];
}
return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
}

return response;
}

private readonly _defaultHeaders: { [key: string]: string } = {};
private readonly _ownerName: string;
private readonly _repoName: string;

private static readonly _apiBase = "https://api.github.com";
private static readonly _maxRedirects = 20;
}

function isRedirectStatusCode(statusCode: number): boolean {
return (
statusCode === 301 ||
statusCode === 302 ||
statusCode === 303 ||
statusCode === 307 ||
statusCode === 308
);
}

/*
* Types and helper functions relating to those types.
*/
Expand Down Expand Up @@ -905,116 +712,3 @@ function warnDeprecatedLauncher() {
`Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`,
);
}

/**
* A release on GitHub.
*/
interface Release {
assets: ReleaseAsset[];

/**
* The creation date of the release on GitHub.
*/
createdAt: string;

/**
* The id associated with the release on GitHub.
*/
id: number;

/**
* The name associated with the release on GitHub.
*/
name: string;
}

/**
* An asset corresponding to a release on GitHub.
*/
interface ReleaseAsset {
/**
* The id associated with the asset on GitHub.
*/
id: number;

/**
* The name associated with the asset on GitHub.
*/
name: string;

/**
* The size of the asset in bytes.
*/
size: number;
}

/**
* The json returned from github for a release.
*/
export interface GithubRelease {
assets: GithubReleaseAsset[];

/**
* The creation date of the release on GitHub, in ISO 8601 format.
*/
created_at: string;

/**
* The id associated with the release on GitHub.
*/
id: number;

/**
* The name associated with the release on GitHub.
*/
name: string;

/**
* Whether the release is a prerelease.
*/
prerelease: boolean;

/**
* The tag name. This should be the version.
*/
tag_name: string;
}

/**
* The json returned by github for an asset in a release.
*/
export interface GithubReleaseAsset {
/**
* The id associated with the asset on GitHub.
*/
id: number;

/**
* The name associated with the asset on GitHub.
*/
name: string;

/**
* The size of the asset in bytes.
*/
size: number;
}

export class GithubApiError extends Error {
constructor(
public status: number,
public body: string,
) {
super(`API call failed with status code ${status}, body: ${body}`);
}
}

export class GithubRateLimitedError extends GithubApiError {
constructor(
public status: number,
public body: string,
public rateLimitResetDate: Date,
) {
super(status, body);
}
}
Loading