Repository navigation
Expand file tree
/
Copy pathtest-generators-e2e.ts
More file actions
221 lines (215 loc) · 9.42 KB
/
Copy pathtest-generators-e2e.ts
File metadata and controls
221 lines (215 loc) · 9.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { access, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'
import { dirname, resolve } from 'node:path'
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { compareStrings } from '@sim/utils/string'
import type { ContentCatalog } from '@/lib/content/types'
const logger = createLogger('DocsGeneratorsE2E')
const reportSetting = process.env.DOCS_GENERATORS_REPORT_PATH
if (!reportSetting || process.env.CI !== 'true')
throw new Error('Run in an isolated CI checkout with DOCS_GENERATORS_REPORT_PATH configured')
const reportPath: string = reportSetting
const root = resolve(import.meta.dirname, '../../..')
const startedAt = new Date().toISOString()
const commands = [
'scripts/generate-openapi.ts',
'scripts/generate-v2-cli-api.ts',
'scripts/generate-cli-docs.ts',
'scripts/generate-v2-mcp-operations.ts',
'scripts/generate-v2-route-table.ts',
'scripts/sync-agent-stream-docs.ts',
'scripts/generate-docs.ts',
'scripts/sync-docs-manifest.ts',
'apps/docs/scripts/compile-content.ts',
] as const
const results: Array<{
name: string
status: 'passed' | 'failed'
durationMs: number
error?: string
}> = []
const git = (...args: string[]) =>
execFileSync('git', args, { cwd: root, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 })
const hash = (value: string) => createHash('sha256').update(value).digest('hex')
const changedFiles: string[][] = []
let failure: string | undefined
const fixturePath = resolve(root, 'apps/docs/content/docs/ci-cache-fixture.mdx')
let createdFixture = false
await mkdir(dirname(reportPath), { recursive: true })
async function runGenerator(command: string, logName: string) {
const output = execFileSync(
'bun',
[command, ...(command === 'apps/docs/scripts/compile-content.ts' ? ['--prune'] : [])],
{
cwd: root,
encoding: 'utf8',
timeout: 180_000,
maxBuffer: 16 * 1024 * 1024,
env: { ...process.env, DATABASE_URL: '', OPENAI_API_KEY: '' },
}
)
await writeFile(resolve(dirname(reportPath), logName), output)
}
async function contentArtifacts() {
const artifacts: Record<string, string> = {}
for (const file of ['catalog.json', 'search.json', 'routes.json', 'registry.ts', 'cache.json'])
artifacts[file] = hash(await readFile(resolve(root, 'apps/docs/.content', file), 'utf8'))
artifacts.fullMarkdown = hash(
await readFile(resolve(root, 'apps/docs/public/llms-full.txt'), 'utf8')
)
const modules = createHash('sha256')
const directory = resolve(root, 'apps/docs/.content/pages')
for (const file of (await readdir(directory)).sort(compareStrings))
modules.update(`${file}\0`).update(await readFile(resolve(directory, file)))
artifacts.modules = modules.digest('hex')
return artifacts
}
async function authoredRegions() {
const regions: Record<string, string> = {}
const files = git('ls-files', 'apps/docs/content/docs/**/*.mdx').trim().split('\n')
for (const file of files) {
const text = await readFile(resolve(root, file), 'utf8')
for (const match of text.matchAll(
/\{\/\*\s*MANUAL-CONTENT-START:(\w+)\s*\*\/\}([\s\S]*?)\{\/\*\s*MANUAL-CONTENT-END\s*\*\/\}/g
))
regions[`${file}:${match[1]}`] = hash(match[2])
}
return regions
}
async function check(name: string, action: () => Promise<void>) {
const started = performance.now()
try {
await action()
results.push({ name, status: 'passed', durationMs: performance.now() - started })
} catch (error) {
results.push({
name,
status: 'failed',
durationMs: performance.now() - started,
error: getErrorMessage(error),
})
throw error
}
}
try {
if (git('status', '--porcelain', '--untracked-files=all').trim())
throw new Error('Generator verification requires a clean checkout')
const regionsBefore = await authoredRegions()
let firstArtifacts: Record<string, string> | undefined
let missingDeclaration: string | undefined
for (const pass of [1, 2]) {
if (pass === 2) {
const directory = resolve(root, 'apps/docs/.content/pages')
const declaration = (await readdir(directory)).find((file) => file.endsWith('.d.ts'))
if (!declaration) throw new Error('No compiled declaration is available for recovery')
missingDeclaration = resolve(directory, declaration)
await rm(missingDeclaration)
for (const suffix of ['js', 'd.ts'])
await writeFile(
resolve(root, 'apps/docs/.content/pages', `obsolete-page.${suffix}`),
'obsolete compiled page'
)
}
for (const command of commands)
await check(`Pass ${pass}: ${command}`, async () => {
await runGenerator(command, `pass-${pass}-${command.split('/').at(-1)}.log`)
})
if (missingDeclaration)
await check('A partial content cache regenerates its missing declaration', async () => {
if (!missingDeclaration) throw new Error('The missing declaration fixture is unavailable')
await access(missingDeclaration)
})
const diff = git('diff', '--binary')
changedFiles.push([
...git('diff', '--name-only').trim().split('\n').filter(Boolean),
...git('ls-files', '--others', '--exclude-standard').trim().split('\n').filter(Boolean),
])
await writeFile(resolve(dirname(reportPath), `pass-${pass}.patch`), diff)
await check(`Pass ${pass}: authored regions preserved`, async () => {
const regionsAfter = await authoredRegions()
for (const [name, digest] of Object.entries(regionsBefore))
if (regionsAfter[name] !== digest) throw new Error(`Authored region changed: ${name}`)
})
const artifacts = await contentArtifacts()
const previousArtifacts = firstArtifacts
if (previousArtifacts)
await check('Unchanged second content generation is byte-identical', async () => {
for (const [file, digest] of Object.entries(previousArtifacts))
if (artifacts[file] !== digest) throw new Error(`Second generation changed ${file}`)
})
else firstArtifacts = artifacts
}
await check('Content generation removes obsolete cached page modules', async () => {
for (const suffix of ['js', 'd.ts']) {
const obsoletePage = resolve(root, 'apps/docs/.content/pages', `obsolete-page.${suffix}`)
if (
await access(obsoletePage).then(
() => true,
() => false
)
)
throw new Error(`Obsolete compiled page remains: ${obsoletePage}`)
}
})
await check('Generated source is current on both passes', async () => {
if (changedFiles.some((files) => files.length))
throw new Error(`Unexpected generated changes: ${JSON.stringify(changedFiles)}`)
})
const originalModules = (await readdir(resolve(root, 'apps/docs/.content/pages'))).sort(
compareStrings
)
const readCatalog = async () =>
JSON.parse(
await readFile(resolve(root, 'apps/docs/.content/catalog.json'), 'utf8')
) as ContentCatalog
const fixture = (version: string) =>
`---\ntitle: CI cache fixture\ndescription: Incremental compilation fixture\n---\n\n${version}\n`
await check('New authored content is published from a warm cache', async () => {
if (
await access(fixturePath).then(
() => true,
() => false
)
)
throw new Error('The incremental compilation fixture already exists')
createdFixture = true
await writeFile(fixturePath, fixture('First fixture revision.'))
await runGenerator('apps/docs/scripts/compile-content.ts', 'cache-create.log')
const page = (await readCatalog()).pages.find((page) => page.id === '/ci-cache-fixture')
if (!page?.markdown.includes('First fixture revision.'))
throw new Error('The new authored page was not published')
})
await check('Changed authored content invalidates its cached output', async () => {
await writeFile(fixturePath, fixture('Second fixture revision.'))
await runGenerator('apps/docs/scripts/compile-content.ts', 'cache-edit.log')
const page = (await readCatalog()).pages.find((page) => page.id === '/ci-cache-fixture')
if (
!page?.markdown.includes('Second fixture revision.') ||
page.markdown.includes('First fixture revision.')
)
throw new Error('The old authored content survived recompilation')
})
await check('Deleted authored content leaves no catalog entry or cached modules', async () => {
await rm(fixturePath)
await runGenerator('apps/docs/scripts/compile-content.ts', 'cache-delete.log')
const modules = (await readdir(resolve(root, 'apps/docs/.content/pages'))).sort(compareStrings)
if (JSON.stringify(modules) !== JSON.stringify(originalModules))
throw new Error('The compiled module inventory changed after the fixture was removed')
const restored = await contentArtifacts()
if (JSON.stringify(restored) !== JSON.stringify(firstArtifacts))
throw new Error('Deleting the fixture did not restore the original generated artifacts')
})
} catch (error) {
failure = getErrorMessage(error)
logger.error('Generator verification failed', { error: failure })
process.exitCode = 1
} finally {
if (createdFixture) await rm(fixturePath, { force: true })
await writeFile(
reportPath,
`${JSON.stringify({ startedAt, completedAt: new Date().toISOString(), revision: git('rev-parse', 'HEAD').trim(), status: failure ? 'failed' : 'passed', failure, changedFiles, results }, null, 2)}\n`
)
logger.info('Generator verification complete', { reportPath, checks: results.length })
}