-
Notifications
You must be signed in to change notification settings - Fork 66.6k
Expand file tree
/
Copy pathsync.ts
More file actions
executable file
·611 lines (547 loc) · 20.4 KB
/
sync.ts
File metadata and controls
executable file
·611 lines (547 loc) · 20.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
import { existsSync } from 'fs'
import { mkdirp } from 'mkdirp'
import { readFile, writeFile } from 'fs/promises'
import path from 'path'
import { slug } from 'github-slugger'
import yaml from 'js-yaml'
import walk from 'walk-sync'
import { getContents, getDirectoryContents } from '@/workflows/git-utils'
import permissionSchema from './permission-list-schema'
import enabledSchema from './enabled-list-schema'
import { validateJson } from '@/tests/lib/validate-json-schema'
const ENABLED_APPS_DIR = 'src/github-apps/data'
const CONFIG_FILE = 'src/github-apps/lib/config.json'
// Actor type mapping from generic names to actual YAML values
export const actorTypeMap: Record<string, string> = {
fine_grained_pat: 'fine_grained_personal_access_token',
server_to_server: 'github_app',
user_to_server: 'user_access_token',
}
interface AppDataOperation {
slug: string
subcategory: string
verb: string
requestPath: string
}
interface AppDataOperationWithCategory extends AppDataOperation {
category: string
}
interface PermissionData {
title: string
displayTitle: string
permissions: Array<
AppDataOperationWithCategory & {
access: string
'user-to-server'?: boolean
'server-to-server'?: boolean
'additional-permissions'?: boolean
}
>
}
interface GitHubAppsData {
[pageType: string]: {
[category: string]: AppDataOperation[] | PermissionData
}
}
interface ProgAccessOperationData {
userToServerRest: boolean
serverToServer: boolean
fineGrainedPat: boolean
permissions: Array<Record<string, string>>
allowPermissionlessAccess?: boolean
allowsPublicRead?: boolean
basicAuth?: boolean
disabledForPatV2?: boolean
}
interface ProgAccessData {
[operationId: string]: ProgAccessOperationData
}
interface ProgActorResource {
title?: string
resource_group?: string
visibility?: string
excluded_actors?: string[]
}
interface ProgActorResources {
[key: string]: ProgActorResource
}
interface OpenApiOperation {
operationId: string
summary: string
'x-github': {
category: string
subcategory: string
}
}
interface OpenApiData {
paths: {
[path: string]: {
[verb: string]: OpenApiOperation
}
}
}
interface AppsDataConfig {
pages: {
[pageType: string]: unknown
}
}
interface ProgAccessRawOperation {
operation_ids: string
user_to_server: {
enabled: boolean
}
server_to_server: {
enabled: boolean
}
disabled_for_patv2?: boolean
permission_sets?: Array<Record<string, string>>
allows_permissionless_access?: boolean
allows_public_read?: boolean
basic_auth?: boolean
}
export async function syncGitHubAppsData(
openApiSource: string,
sourceSchemas: string[],
progAccessSource: string,
): Promise<void> {
console.log(
`Generating GitHub Apps data from ${openApiSource}, ${sourceSchemas} and ${progAccessSource}`,
)
const { progAccessData, progActorResources } = await getProgAccessData(progAccessSource)
for (const schemaName of sourceSchemas) {
const schemaData = JSON.parse(
await readFile(path.join(openApiSource, schemaName), 'utf8'),
) as OpenApiData
const appsDataConfig = JSON.parse(await readFile(CONFIG_FILE, 'utf8')) as AppsDataConfig
// Initialize the data structure with keys for each page type
const githubAppsData: GitHubAppsData = {}
for (const pageType of Object.keys(appsDataConfig.pages)) {
githubAppsData[pageType] = {}
}
// Because the information used on the apps page doesn't require any
// rendered content we can parse the dereferenced files directly
for (const [requestPath, operationsAtPath] of Object.entries(schemaData.paths)) {
for (const [verb, operation] of Object.entries(operationsAtPath)) {
// We only want to process operations that have programmatic access data
if (!progAccessData[operation.operationId]) continue
const isInstallationAccessToken = progAccessData[operation.operationId].serverToServer
const isUserAccessToken = progAccessData[operation.operationId].userToServerRest
const isFineGrainedPat =
isUserAccessToken && !progAccessData[operation.operationId].disabledForPatV2
const { category, subcategory } = operation['x-github']
const appDataOperation: AppDataOperation = {
slug: slug(operation.summary),
subcategory,
verb,
requestPath,
}
const appDataOperationWithCategory: AppDataOperationWithCategory = Object.assign(
{ category },
appDataOperation,
)
// server-to-server
if (isInstallationAccessToken) {
addAppData(githubAppsData['server-to-server-rest'], category, appDataOperation)
}
// user-to-server
if (isUserAccessToken) {
addAppData(githubAppsData['user-to-server-rest'], category, appDataOperation)
}
// fine-grained pat
if (isFineGrainedPat) {
// Check if all permission sets for this operation are excluded for fine-grained PATs
const allPermissionSetsExcluded = progAccessData[operation.operationId].permissions.every(
(permissionSet) =>
Object.keys(permissionSet).every((permissionName) =>
isActorExcluded(
progActorResources[permissionName]?.excluded_actors,
'fine_grained_pat',
actorTypeMap,
),
),
)
if (!allPermissionSetsExcluded) {
addAppData(githubAppsData['fine-grained-pat'], category, appDataOperation)
}
}
// permissions
for (const permissionSet of progAccessData[operation.operationId].permissions) {
for (const [permissionName, readOrWrite] of Object.entries(permissionSet)) {
const { title, displayTitle } = getDisplayTitle(permissionName, progActorResources)
if (progActorResources[permissionName]?.['visibility'] === 'private') continue
const excludedActors = progActorResources[permissionName]?.['excluded_actors']
const additionalPermissions = calculateAdditionalPermissions(
progAccessData[operation.operationId].permissions,
)
// Filter out metadata permissions when combined with other permissions
// The metadata permission is automatically granted with any other repository permission,
// so documenting it for operations that require additional permissions is misleading.
// This fixes the issue where mutating operations (PUT, DELETE) incorrectly appeared
// to only need metadata access when they actually require write permissions.
// See: https://github.com/github/docs-engineering/issues/5212
if (
shouldFilterMetadataPermission(
permissionName,
progAccessData[operation.operationId].permissions,
)
) {
continue
}
// github app permissions
if (!isActorExcluded(excludedActors, 'server_to_server', actorTypeMap)) {
const serverToServerPermissions = githubAppsData['server-to-server-permissions']
if (!serverToServerPermissions[permissionName]) {
serverToServerPermissions[permissionName] = {
title,
displayTitle,
permissions: [],
}
}
const worksWithData = {
'user-to-server': Boolean(
isUserAccessToken &&
!isActorExcluded(excludedActors, 'user_to_server', actorTypeMap),
),
'server-to-server': Boolean(
isInstallationAccessToken &&
!isActorExcluded(excludedActors, 'server_to_server', actorTypeMap),
),
'additional-permissions': additionalPermissions,
}
const permissionsArray = (serverToServerPermissions[permissionName] as PermissionData)
.permissions
permissionsArray.push(
Object.assign(
{},
appDataOperationWithCategory,
{ access: readOrWrite },
worksWithData,
),
)
}
// fine-grained pats
const isExcluded = isActorExcluded(excludedActors, 'fine_grained_pat', actorTypeMap)
if (isFineGrainedPat && !isExcluded) {
// Hardcoded exception: exclude repository_projects from fine-grained PAT permissions
// This is because fine-grained PATs can only operate on organization-level Projects (classic),
// not repository-level Projects (classic). Users cannot grant the repository Projects (classic)
// fine-grained permission in the fine-grained PAT UI.
// See: https://github.com/github/docs-engineering/issues/4613
if (permissionName === 'repository_projects') {
continue
}
const findGrainedPatPermissions = githubAppsData['fine-grained-pat-permissions']
if (!findGrainedPatPermissions[permissionName]) {
findGrainedPatPermissions[permissionName] = {
title,
displayTitle,
permissions: [],
}
}
const permissionsArray = (findGrainedPatPermissions[permissionName] as PermissionData)
.permissions
permissionsArray.push(
Object.assign({}, appDataOperationWithCategory, {
'additional-permissions': additionalPermissions,
access: readOrWrite,
}),
)
}
}
}
}
}
const versionName = path.basename(schemaName, '.json')
const targetDirectory = path.join(ENABLED_APPS_DIR, versionName)
// When a new version is added, we need to create the directory for it
if (!existsSync(targetDirectory)) {
await mkdirp(targetDirectory)
}
for (const pageType of Object.keys(githubAppsData)) {
const data = githubAppsData[pageType]
await validateAppData(data, pageType)
const filename = `${pageType}.json`
if (Object.keys(data).length === 0) {
throw new Error(
`Generating GitHub Apps data failed for ${openApiSource}/${schemaName}. The generated data file was empty.`,
)
}
const sortedOperations = pageType.includes('permissions')
? sortObjectByTitle(data)
: sortObjectByKeys(data)
const targetPath = path.join(targetDirectory, filename)
await writeFile(targetPath, JSON.stringify(sortedOperations, null, 2))
console.log(`✅ Wrote ${targetPath}`)
}
}
}
export async function getProgAccessData(
progAccessSource: string,
isRest = false,
): Promise<{ progAccessData: ProgAccessData; progActorResources: ProgActorResources }> {
const useRemoteGitHubFiles = progAccessSource === 'rest-api-description'
// check for required PAT
if (useRemoteGitHubFiles && !process.env.GITHUB_TOKEN) {
throw new Error(
'Error! You must have the GITHUB_TOKEN environment variable set to access the programmatic access and resource files via the GitHub REST API.',
)
}
let progAccessDataRaw: ProgAccessRawOperation[]
let progActorResources: ProgActorResources
const progAccessFilepath = 'config/access_control/programmatic_access.yaml'
const progActorDirectory =
'config/access_control/fine_grained_permissions/programmatic_actor_fine_grained_resources'
if (!useRemoteGitHubFiles) {
progAccessDataRaw = yaml.load(
await readFile(path.join(progAccessSource, progAccessFilepath), 'utf8'),
) as ProgAccessRawOperation[]
progActorResources = await getProgActorResourceContent({
gitHubSourceDirectory: path.join(progAccessSource, progActorDirectory),
})
} else {
progAccessDataRaw = yaml.load(
await getContents('github', 'github', 'master', progAccessFilepath),
) as ProgAccessRawOperation[]
progActorResources = await getProgActorResourceContent({
owner: 'github',
repo: 'github',
branch: 'master',
path: progActorDirectory,
})
}
const progAccessData: ProgAccessData = {}
for (const operation of progAccessDataRaw) {
const operationData: ProgAccessOperationData = {
userToServerRest: operation.user_to_server.enabled,
serverToServer: operation.server_to_server.enabled,
fineGrainedPat: operation.user_to_server.enabled && !operation.disabled_for_patv2,
permissions: isRest
? getDisplayPermissions(operation.permission_sets || [], progActorResources)
: operation.permission_sets || [],
allowPermissionlessAccess: operation.allows_permissionless_access,
allowsPublicRead: operation.allows_public_read,
basicAuth: operation.basic_auth,
disabledForPatV2: operation.disabled_for_patv2,
}
// Handle comma-separated operation IDs
const operationIds = operation.operation_ids.split(',').map((id) => id.trim())
for (const operationId of operationIds) {
progAccessData[operationId] = operationData
}
}
return { progAccessData, progActorResources }
}
function getDisplayPermissions(
permissionSets: Array<Record<string, string>>,
progActorResources: ProgActorResources,
): Array<Record<string, string>> {
const displayPermissions = permissionSets.map((permissionSet) => {
const displayPermissionSet: Record<string, string> = {}
for (const [key, value] of Object.entries(permissionSet)) {
const { displayTitle } = getDisplayTitle(key, progActorResources, true)
displayPermissionSet[displayTitle] = value
}
return displayPermissionSet
})
return displayPermissions
}
function sortObjectByKeys<T>(obj: Record<string, T>): Record<string, T> {
return Object.keys(obj)
.sort()
.reduce(
(acc, key) => {
acc[key] = obj[key]
return acc
},
{} as Record<string, T>,
)
}
function sortObjectByTitle(obj: Record<string, PermissionData | unknown>): Record<string, unknown> {
return Object.keys(obj)
.sort((a, b) => {
const aData = obj[a] as PermissionData
const bData = obj[b] as PermissionData
if (aData.displayTitle > bData.displayTitle) {
return 1
}
if (aData.displayTitle < bData.displayTitle) {
return -1
}
return 0
})
.reduce(
(acc, key) => {
acc[key] = obj[key]
return acc
},
{} as Record<string, unknown>,
)
}
function getDisplayTitle(
permissionName: string,
progActorResources: ProgActorResources,
isRest = false,
): { title: string; displayTitle: string } {
const tempTitle = permissionName.replace(/_/g, ' ')
const permissionNameExists = progActorResources[permissionName]
if (!permissionNameExists) {
console.warn(
`The permission ${permissionName} is missing from the definitions in the config/access_control/fine_grained_permissions/programmatic_actor_fine_grained_resources directory. Creating a placeholder value of ${tempTitle} until it's added.`,
)
}
const title = progActorResources[permissionName]?.title || tempTitle
let resourceGroup = progActorResources[permissionName]?.resource_group || ''
if (resourceGroup === 'business') {
resourceGroup = 'enterprise'
}
if (!title) {
console.warn(`No title found for title ${title} resource group ${resourceGroup}`)
return { title: '', displayTitle: '' }
}
const displayTitle = isRest
? !resourceGroup
? `${sentenceCase(title)} permissions`
: `"${sentenceCase(title)}" ${resourceGroup} permissions`
: !resourceGroup
? `${sentenceCase(title)} permissions`
: `${sentenceCase(resourceGroup)} permissions for "${title}"`
return { title, displayTitle }
}
function sentenceCase(str: string): string {
return str.charAt(0).toUpperCase() + str.slice(1)
}
/**
* Calculates whether an operation has additional permissions beyond a single permission.
*/
export function calculateAdditionalPermissions(
permissionSets: Array<Record<string, string>>,
): boolean {
return (
permissionSets.length > 1 ||
permissionSets.some((permissionSet) => Object.keys(permissionSet).length > 1)
)
}
/**
* Determines whether a metadata permission should be filtered out when it has additional permissions.
* Prevents misleading documentation where mutating operations appear to only need metadata access.
*/
export function shouldFilterMetadataPermission(
permissionName: string,
permissionSets: Array<Record<string, string>>,
): boolean {
if (permissionName !== 'metadata') {
return false
}
return calculateAdditionalPermissions(permissionSets)
}
export function isActorExcluded(
excludedActors: string[] | undefined | null | unknown,
actorType: string,
actorMapping: Record<string, string> = {},
): boolean {
if (!excludedActors || !Array.isArray(excludedActors)) {
return false
}
// Map generic actor type to actual YAML value if mapping exists
const mappedActorType = actorMapping[actorType] || actorType
// Check if the mapped actor type is excluded
if (excludedActors.includes(mappedActorType)) {
return true
}
// Also check for the original actor type (before mapping)
if (excludedActors.includes(actorType)) {
return true
}
// Check for known aliases - the source data might use different values
// than what we expect in our mapping
if (actorType === 'fine_grained_pat' && excludedActors.includes('UserProgrammaticAccess')) {
return true
}
return false
}
function addAppData(
storage: Record<string, AppDataOperation[] | PermissionData>,
category: string,
data: AppDataOperation,
): void {
if (!storage[category]) {
storage[category] = []
}
;(storage[category] as AppDataOperation[]).push(data)
}
async function validateAppData(
data: Record<string, AppDataOperation[] | PermissionData>,
pageType: string,
): Promise<void> {
if (pageType.includes('permissions')) {
for (const value of Object.values(data)) {
const { isValid, errors } = validateJson(permissionSchema, value)
if (!isValid) {
console.error(JSON.stringify(errors, null, 2))
throw new Error('GitHub Apps permission schema validation failed')
}
}
} else {
for (const arrayItems of Object.values(data)) {
for (const item of arrayItems as AppDataOperation[]) {
const { isValid, errors } = validateJson(enabledSchema, item)
if (!isValid) {
console.error(JSON.stringify(errors, null, 2))
throw new Error('GitHub Apps enabled apps schema validation failed')
}
}
}
}
}
interface ProgActorResourceContentOptions {
owner?: string
repo?: string
branch?: string
path?: string
gitHubSourceDirectory?: string | null
}
// When getting files from the GitHub repo locally (or in a Codespace)
// you can pass the full or relative path to the `github` repository
// directory on disk.
// When the source directory is `rest-api-description` (which is more common)
// you can pass the `owner`, `repo`, `branch`, and `path` (repository path)
async function getProgActorResourceContent({
owner,
repo,
branch,
path: resourcePath,
gitHubSourceDirectory = null,
}: ProgActorResourceContentOptions): Promise<ProgActorResources> {
// Get files either locally from disk or from the GitHub remote repo
let files: string[]
if (gitHubSourceDirectory) {
files = await getProgActorContentFromDisk(gitHubSourceDirectory)
} else {
files = (await getDirectoryContents(owner!, repo!, branch!, resourcePath!)).map(
(file) => file.content,
)
}
// We need to format the file content into a single object. Each file
// contains a single key and a single value that needs to be added
// to the object.
const progActorResources: ProgActorResources = {}
for (const file of files) {
const fileContent = yaml.load(file) as Record<string, ProgActorResource>
// Each file should only contain a single key and value.
if (Object.keys(fileContent).length !== 1) {
throw new Error(`Error: The file ${JSON.stringify(fileContent)} must only have one key.`)
}
for (const [key, value] of Object.entries(fileContent)) {
progActorResources[key] = value
}
}
return progActorResources
}
async function getProgActorContentFromDisk(directory: string): Promise<string[]> {
const files = walk(directory, {
includeBasePath: true,
directories: false,
})
const promises = files.map(async (file) => await readFile(file, 'utf8'))
return await Promise.all(promises)
}