Repository navigation
Expand file tree
/
Copy pathgenerate-docs.ts
More file actions
executable file
·5054 lines (4428 loc) · 182 KB
/
Copy pathgenerate-docs.ts
File metadata and controls
executable file
·5054 lines (4428 loc) · 182 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env ts-node
import fs from 'fs'
import path from 'path'
import { fileURLToPath, pathToFileURL } from 'url'
import { isVersionedType, stripVersionSuffix } from '@sim/utils/string'
import { glob } from 'glob'
import type { BlockCategory } from '../apps/sim/blocks/types'
import { IntegrationType } from '../apps/sim/blocks/types'
import type { ToolOutputProperty } from '../apps/sim/tools/types'
/**
* Cache for resolved const definitions from types files.
* Key: "toolPrefix:constName" (e.g., "calcom:SCHEDULE_DATA_OUTPUT_PROPERTIES")
* Value: The resolved properties object
*/
const constResolutionCache = new Map<string, Record<string, any>>()
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
const rootDir = path.resolve(__dirname, '..')
const BLOCKS_PATH = path.join(rootDir, 'apps/sim/blocks/blocks')
export const DOCS_OUTPUT_PATH = path.join(rootDir, 'apps/docs/content/docs/integrations')
export const DOCS_ORIGIN = 'https://docs.sim.ai/'
/**
* The docs URL a block gets when it declares no `docsLink` — one generated page
* per service, named for the block's base type. Exported so the catalog checker
* validates the same contract this generator emits rather than a second copy of
* it that can silently drift.
*/
export function defaultIntegrationDocsUrl(blockType: string): string {
return `${DOCS_ORIGIN}integrations/${stripVersionSuffix(blockType)}`
}
const ICONS_PATH = path.join(rootDir, 'apps/sim/components/icons.tsx')
const DOCS_ICONS_PATH = path.join(rootDir, 'apps/docs/components/icons.tsx')
const INTEGRATIONS_DATA_PATH = path.join(rootDir, 'apps/sim/lib/integrations')
const INTEGRATIONS_CATALOG_PATH = path.join(rootDir, 'packages/deployment-config/src')
const LANDING_INTEGRATIONS_DATA_PATH = path.join(
rootDir,
'apps/sim/app/(landing)/integrations/data'
)
const TRIGGERS_PATH = path.join(rootDir, 'apps/sim/triggers')
const sourceFileCache = new Map<string, string>()
const sourceGlobCache = new Map<string, Promise<string[]>>()
const blockConfigCache = new Map<string, ReturnType<typeof extractAllBlockConfigs>>()
function readSourceFile(filePath: string): string {
const cached = sourceFileCache.get(filePath)
if (cached !== undefined) return cached
const source = fs.readFileSync(filePath, 'utf-8')
sourceFileCache.set(filePath, source)
return source
}
async function sourceGlob(pattern: string): Promise<string[]> {
let pending = sourceGlobCache.get(pattern)
if (!pending) {
pending = glob(pattern)
sourceGlobCache.set(pattern, pending)
}
return [...(await pending)]
}
function blockConfigsForFile(filePath: string): ReturnType<typeof extractAllBlockConfigs> {
const cached = blockConfigCache.get(filePath)
if (cached) return cached
const configs = extractAllBlockConfigs(readSourceFile(filePath))
blockConfigCache.set(filePath, configs)
return configs
}
// Integration triggers are merged into the same per-service page as the service's
// actions (one block per integration: actions + an optional Trigger).
const TRIGGER_DOCS_OUTPUT_PATH = DOCS_OUTPUT_PATH
/**
* Hand-written integration pages in DOCS_OUTPUT_PATH that the generator must
* never clobber. Every hand-authored `*-service-account` credential guide has
* to be listed here — these pages carry no `MANUAL-CONTENT` markers and no
* backing block, so the stale-doc cleanup deletes any that go unregistered.
*/
const HANDWRITTEN_INTEGRATION_DOCS = new Set([
'index',
'a2a',
'airtable-service-account',
'asana-service-account',
'atlassian-service-account',
'attio-service-account',
'box-service-account',
'calcom-service-account',
'clickup-service-account',
'google-service-account',
'hubspot-service-account',
'hubspot-setup',
'linear-service-account',
'monday-service-account',
'netsuite-service-account',
'notion-service-account',
'pipedrive-service-account',
'salesforce-service-account',
'shopify-service-account',
'snowflake-service-account',
'trello-service-account',
'wealthbox-service-account',
'webflow-service-account',
'zoho-desk-service-account',
'zoom-service-account',
])
/**
* Native Sim resource blocks (category 'blocks') that still get a generated
* integration page. The writer's filter, the stale-doc cleanup, and the icon
* map must all honor this set: cleanup would otherwise delete what the writer
* emits (losing manual content), and an icon map that omits these types leaves
* their pages rendering the two-letter text fallback instead of the icon.
*/
const NATIVE_RESOURCE_BLOCK_TYPES = new Set([
'memory',
'knowledge',
'table',
'enrichment',
'logs',
'deployments',
])
/** Trigger doc pages that are hand-written and must never be overwritten. */
const HANDWRITTEN_TRIGGER_DOCS = new Set([
'index',
'start',
'schedule',
'webhook',
'rss',
'table',
'sim',
])
/** Omits hand-written providers and Slack's superseded legacy webhook trigger. */
const SKIP_TRIGGER_PROVIDERS = new Set(['generic', 'rss', 'table', 'sim', 'slack'])
/**
* Maps trigger provider names (from TriggerConfig.provider) to their
* corresponding block type when the two differ. Used to resolve icon
* colours from the block registry.
*/
const PROVIDER_TO_BLOCK_TYPE: Record<string, string> = {
'microsoft-teams': 'microsoft_teams',
'google-calendar': 'google_calendar',
'google-drive': 'google_drive',
'google-sheets': 'google_sheets',
jsm: 'jira_service_management',
slack_app: 'slack',
}
/** Human-readable display names for trigger providers. */
const TRIGGER_PROVIDER_DISPLAY_NAMES: Record<string, string> = {
airtable: 'Airtable',
ashby: 'Ashby',
attio: 'Attio',
calcom: 'Cal.com',
calendly: 'Calendly',
circleback: 'Circleback',
confluence: 'Confluence',
fathom: 'Fathom',
fireflies: 'Fireflies',
github: 'GitHub',
gmail: 'Gmail',
gong: 'Gong',
'google-calendar': 'Google Calendar',
'google-drive': 'Google Drive',
'google-sheets': 'Google Sheets',
google_forms: 'Google Forms',
grain: 'Grain',
greenhouse: 'Greenhouse',
hubspot: 'HubSpot',
imap: 'IMAP',
intercom: 'Intercom',
jira: 'Jira',
lemlist: 'Lemlist',
linear: 'Linear',
'microsoft-teams': 'Microsoft Teams',
notion: 'Notion',
outlook: 'Outlook',
resend: 'Resend',
salesforce: 'Salesforce',
servicenow: 'ServiceNow',
slack: 'Slack',
stripe: 'Stripe',
telegram: 'Telegram',
tiktok: 'TikTok',
twilio_voice: 'Twilio Voice',
typeform: 'Typeform',
vercel: 'Vercel',
webflow: 'Webflow',
whatsapp: 'WhatsApp',
zoom: 'Zoom',
}
if (!fs.existsSync(DOCS_OUTPUT_PATH)) {
fs.mkdirSync(DOCS_OUTPUT_PATH, { recursive: true })
}
const docsComponentsDir = path.dirname(DOCS_ICONS_PATH)
if (!fs.existsSync(docsComponentsDir)) {
fs.mkdirSync(docsComponentsDir, { recursive: true })
}
/** Runtime set of valid `IntegrationType` values, derived from the canonical enum. */
const INTEGRATION_CATEGORY_VALUES: ReadonlySet<IntegrationType> = new Set(
Object.values(IntegrationType)
)
/**
* Defensive shape for blocks parsed out of source files. Fields stay loose
* (`string`) so the AST-style extractor can populate them progressively; the
* canonical taxonomy is enforced at the JSON-write boundary inside
* `writeIntegrationsJson`.
*/
interface BlockConfig {
type: string
name: string
description: string
longDescription?: string
category: string
integrationType?: string
bgColor?: string
outputs?: Record<string, any>
tools?: {
access?: string[]
}
operations?: OperationInfo[]
/**
* Param names the block itself supplies — via a `subBlocks` field (id or
* `canonicalParamId`) or via its `tools.config.params` mapper.
*
* `null` means the block's `subBlocks` array could not be read, so which params it supplies
* is UNKNOWN and the hidden-param filter is skipped for it. Never conflate that with `[]`,
* which asserts the block supplies nothing and strips every hidden param from its page.
*/
userSettableParamIds?: string[] | null
docsLink?: string
[key: string]: any
}
/**
* True when a block's source text marks it as an unreleased `preview: true`
* block. THE single preview gate for this script — every surface it emits
* (docs .mdx, integrations.json, icon mapping) must consult this, because a
* missed gate publishes an unreleased block to docs.sim.ai, the catalog, the
* sitemap, and OG images. Mirrors the `hideFromToolbar` source-text checks.
*/
function isPreviewSource(blockContent: string): boolean {
return /preview\s*:\s*true/.test(blockContent)
}
/**
* Blank out `//` and block comments so source-text property probes match real
* code only. Without this, prose that quotes a property — e.g. slack.ts's
* "At v2 GA this becomes `hideFromToolbar: true`" — reads as the property
* itself and silently drops the block from every generated surface.
*
* Comment bodies are replaced with spaces rather than removed so byte offsets
* stay aligned with the original content. Deliberately not applied to
* {@link isPreviewSource}: that gate is fail-closed on purpose, and a
* false positive there only over-hides an unreleased block.
*/
function stripSourceComments(content: string): string {
return content
.replace(/\/\*[\s\S]*?\*\//g, (m) => m.replace(/[^\n]/g, ' '))
.replace(/(^|[^:])\/\/[^\n]*/g, (m, prefix) => prefix + ' '.repeat(m.length - prefix.length))
}
/**
* Find the position after the matching close delimiter for an opening delimiter.
* Assumes `content[openPos]` is the opening char (e.g. `{` or `[`).
* Returns the index one past the matching close char, or -1 if unbalanced.
*/
function findMatchingClose(
content: string,
openPos: number,
openChar = '{',
closeChar = '}'
): number {
let count = 1
let pos = openPos + 1
while (pos < content.length && count > 0) {
if (content[pos] === openChar) count++
else if (content[pos] === closeChar) count--
pos++
}
return count === 0 ? pos : -1
}
interface TriggerInfo {
id: string
name: string
description: string
}
interface TriggerConfigField {
id: string
title: string
type: string
required: boolean
description?: string
placeholder?: string
}
/** The subset of `SubBlockConfig` the generated configuration table reads. */
interface RegistrySubBlock {
id?: string
type?: string
title?: string
description?: string
placeholder?: unknown
/** `true`, or a condition object making the field required only for some configurations. */
required?: unknown
readOnly?: boolean
}
interface RegistryTrigger {
subBlocks?: RegistrySubBlock[]
outputs?: Record<string, any>
}
/** Present for the operator, not part of a trigger's configuration surface. */
const TRIGGER_UI_ONLY_IDS = new Set([
'webhookUrlDisplay',
'triggerInstructions',
'selectedTriggerId',
])
/**
* Loads the evaluated trigger registry.
*
* Imported by absolute path so Bun resolves the `@/` aliases against `apps/sim`'s tsconfig
* rather than this script's.
*/
async function loadTriggerRegistry(): Promise<Record<string, RegistryTrigger>> {
const module = await import(path.join(rootDir, 'apps/sim/triggers/registry.ts'))
return module.TRIGGER_REGISTRY as Record<string, RegistryTrigger>
}
interface ToolMetadataParam {
type?: string
required?: boolean
description?: string
visibility?: string
}
interface ToolMetadataEntry {
name?: string
description?: string
params?: Record<string, ToolMetadataParam>
}
/** Client-safe tool metadata, keyed by tool id and kept in sync with the registry by CI. */
let toolMetadata: Record<string, ToolMetadataEntry> | null = null
async function loadToolMetadata(): Promise<Record<string, ToolMetadataEntry>> {
if (toolMetadata) return toolMetadata
const module = await import(path.join(rootDir, 'apps/sim/tools/generated/tool-metadata.ts'))
toolMetadata = module.default as Record<string, ToolMetadataEntry>
return toolMetadata
}
/** Evaluated tool output schemas, keyed by tool id and kept in sync with the registry by CI. */
let toolOutputs: Record<string, Record<string, ToolOutputProperty>> | null = null
async function loadToolOutputs(): Promise<Record<string, Record<string, ToolOutputProperty>>> {
if (toolOutputs) return toolOutputs
const module = await import(path.join(rootDir, 'apps/sim/tools/generated/tool-outputs.ts'))
toolOutputs = module.default as Record<string, Record<string, ToolOutputProperty>>
return toolOutputs
}
/** Human-facing tool names, keyed by tool id. Kept in sync with the registry by CI. */
let toolDisplayNames: Map<string, string> | null = null
async function loadToolDisplayNames(): Promise<Map<string, string>> {
if (toolDisplayNames) return toolDisplayNames
const metadata = await loadToolMetadata()
toolDisplayNames = new Map(
Object.entries(metadata).flatMap(([id, entry]) =>
entry?.name ? [[id, entry.name] as const] : []
)
)
return toolDisplayNames
}
interface TriggerFullInfo {
id: string
name: string
description: string
provider: string
polling: boolean
outputs: Record<string, any>
configFields: TriggerConfigField[]
}
interface OperationInfo {
name: string
description: string
}
interface IntegrationEntry {
type: string
slug: string
name: string
description: string
longDescription: string
bgColor: string
iconName: string
docsUrl: string
operations: OperationInfo[]
operationCount: number
triggers: TriggerInfo[]
triggerCount: number
authType: 'oauth' | 'api-key' | 'none'
oauthServiceId?: string
category: BlockCategory
integrationType: IntegrationType
tags?: string[]
landingContent?: Record<string, unknown>
}
/** A block icon component together with the module it must be imported from. */
interface IconRef {
name: string
source: string
}
/**
* Check mode (`--check`): render every generated artifact in memory and compare
* it against the committed file instead of writing, so CI can fail on docs
* drift the same way `tool-metadata:check` fails on stale tool metadata. Check
* mode performs no filesystem mutations.
*
* The pipeline writes some pages twice per run — the block pass writes the base
* page, then the trigger pass reads it back and appends/merges the Triggers
* section — so check mode keeps an in-memory overlay of everything "written"
* this run (`emittedByPath`), readers consult the overlay before disk
* (`readGeneratedFile`), and staleness is judged once at the end against each
* artifact's FINAL content. Comparing at emit time would flag the intermediate
* block-pass content of every trigger-owning page as a false positive.
*
* Known limitation: `updateMetaJson` derives the sidebar from the mdx files on
* disk, so in check mode a brand-new block's missing page is reported directly
* while the corresponding meta.json entry is not — regenerating fixes both.
*/
let CHECK_ONLY = false
const staleArtifacts: string[] = []
const emittedByPath = new Map<string, string>()
/**
* Deletion candidates recorded by cleanup in check mode. Judged at the end of
* the run, not at cleanup time: generate mode deletes a non-canonical page and
* lets the trigger pass recreate it in the same run, so a candidate that was
* re-emitted this run is that delete-then-recreate dance — content drift (if
* any) is already covered by the overlay comparison — while a candidate nothing
* re-emitted is a genuinely stale page regeneration would remove.
*/
const wouldDeletePaths: string[] = []
/** Writes a generated artifact, or in check mode records its final content for the end-of-run comparison. */
function emitGeneratedFile(filePath: string, content: string): void {
if (CHECK_ONLY) {
emittedByPath.set(filePath, content)
return
}
fs.writeFileSync(filePath, content)
}
/** Reads a generated artifact as the pipeline would see it mid-run: overlay first in check mode, then disk. */
function readGeneratedFile(filePath: string): string | null {
const emitted = emittedByPath.get(filePath)
if (emitted !== undefined) return emitted
return fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : null
}
/** Compares every overlay entry against the committed file; returns repo-relative stale paths. */
function collectStaleEmissions(): string[] {
const stale: string[] = []
for (const [filePath, content] of emittedByPath) {
const committed = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : null
if (committed !== content) stale.push(path.relative(rootDir, filePath))
}
return stale
}
/**
* Copy the icons.tsx file from the main sim app to the docs app
* This ensures icons are rendered consistently across both apps
*/
function copyIconsFile(): void {
try {
if (!CHECK_ONLY) console.log('Copying icons from sim app to docs app...')
if (!fs.existsSync(ICONS_PATH)) {
console.error(`Source icons file not found: ${ICONS_PATH}`)
return
}
const iconsContent = readSourceFile(ICONS_PATH)
emitGeneratedFile(DOCS_ICONS_PATH, iconsContent)
if (!CHECK_ONLY) console.log('✓ Icons successfully copied to docs app')
} catch (error) {
console.error('Error copying icons file:', error)
}
}
/**
* Some trigger providers have no block of their own (`slack_app`, `twilio`) yet
* still get a generated page keyed by the provider id. Seed those provider ids
* from the trigger definitions' own `icon` so their pages render the brand mark
* instead of the two-letter fallback. Never overwrites a block-derived entry —
* the block is the canonical icon source when one exists.
*/
async function addTriggerProviderIcons(
iconMappings: readonly Record<string, IconRef>[]
): Promise<void> {
const triggerFiles = (await sourceGlob(`${TRIGGERS_PATH}/**/*.ts`)).filter(
(f) => !f.includes('.test.')
)
const previewOnly = await collectPreviewOnlyTriggerIds()
for (const file of triggerFiles) {
const fileContent = readSourceFile(file)
const source = stripSourceComments(fileContent)
// Pair each trigger's `id` with the `provider` that follows it in the same
// config, so files holding several trigger configs attribute each provider
// (and its icon) to the right trigger.
const configRegex =
/\bid\s*:\s*['"]([^'"]+)['"][\s\S]{0,600}?\bprovider\s*:\s*['"]([^'"]+)['"]/g
for (const match of source.matchAll(configRegex)) {
const [, triggerId, provider] = match
if (iconMappings.every((iconMapping) => iconMapping[provider])) continue
// Preview-only triggers get no page, so they need no provider icon.
if (previewOnly.has(triggerId)) continue
const iconName = extractIconNameFromContent(source.slice(match.index))
if (!iconName) continue
const iconRef = { name: iconName, source: resolveIconSource(fileContent, iconName) }
for (const iconMapping of iconMappings) {
if (!iconMapping[provider]) iconMapping[provider] = iconRef
}
}
}
}
/**
* Generate icon mapping from block definitions.
* Docs need hidden historical version keys so old BlockInfoCard references and
* versioned docs links still render icons, while landing only needs visible blocks.
*/
async function generateIconMappings(): Promise<{
docs: Record<string, IconRef>
visible: Record<string, IconRef>
}> {
try {
console.log('Generating icon mapping from block definitions...')
const docs: Record<string, IconRef> = {}
const visible: Record<string, IconRef> = {}
const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort()
for (const blockFile of blockFiles) {
const fileContent = readSourceFile(blockFile)
// For icon mapping, we need ALL blocks including hidden ones
// because V2 blocks inherit icons from legacy blocks via spread
// First, extract the primary icon from the file (usually the legacy block's icon)
const primaryIcon = extractIconNameFromContent(fileContent)
const exportRegex = /export\s+const\s+(\w+)Block\s*:\s*BlockConfig[^=]*=\s*\{/g
let match
while ((match = exportRegex.exec(fileContent)) !== null) {
const blockName = match[1]
const startIndex = match.index + match[0].length - 1
const endIndex = findMatchingClose(fileContent, startIndex)
if (endIndex !== -1) {
const blockContent = fileContent.substring(startIndex, endIndex)
// Check hideFromToolbar - skip hidden blocks for docs but NOT for icon mapping
const hideFromToolbar = /hideFromToolbar\s*:\s*true/.test(
stripSourceComments(blockContent)
)
// Unreleased preview blocks never reach any public surface, icon map included.
if (isPreviewSource(blockContent)) {
continue
}
const blockType =
extractStringPropertyFromContent(blockContent, 'type') || blockName.toLowerCase()
const iconName = extractIconNameFromContent(blockContent) || primaryIcon
if (!blockType || !iconName) {
continue
}
if (
blockType.includes('_trigger') ||
blockType.includes('_webhook') ||
blockType.includes('rss')
) {
continue
}
const category = extractStringPropertyFromContent(blockContent, 'category') || 'misc'
// Exclude first-party `blocks`-category primitives (except the native
// resource blocks that still get a generated docs page) and
// core/plumbing types. Keying the exception off
// `NATIVE_RESOURCE_BLOCK_TYPES` — the same set the docs writer uses —
// keeps the icon map from drifting behind the pages that consume it.
const baseType = stripVersionSuffix(blockType)
if (
(category === 'blocks' &&
!NATIVE_RESOURCE_BLOCK_TYPES.has(baseType) &&
!HANDWRITTEN_INTEGRATION_DOCS.has(baseType)) ||
ICON_MAP_EXCLUDED_TYPES.has(blockType)
) {
continue
}
const isVersionedBlockType = isVersionedType(blockType)
/**
* A sunset block keeps its docs page — `docsLink` is baked into every
* placed instance — so it still needs an icon there, exactly like a
* hidden versioned block. Without this it renders as a text tile.
*/
const isSunsetBlockType = /sunset\s*:\s*\{/.test(stripSourceComments(blockContent))
const iconRef = {
name: iconName,
source: resolveIconSource(fileContent, iconName),
}
if (!hideFromToolbar) {
docs[blockType] = iconRef
visible[blockType] = iconRef
} else if (isVersionedBlockType || isSunsetBlockType) {
docs[blockType] = iconRef
}
}
}
}
await addTriggerProviderIcons([docs, visible])
console.log(
`✓ Generated icon mappings for ${Object.keys(docs).length} docs blocks and ` +
`${Object.keys(visible).length} visible blocks`
)
return { docs, visible }
} catch (error) {
console.error('Error generating icon mapping:', error)
return { docs: {}, visible: {} }
}
}
/**
* Write the icon mapping to the docs app
* This file is imported by BlockInfoCard to resolve icons automatically
*/
/**
* Sort strings to match Biome's organizeImports order:
* case-insensitive character-by-character, uppercase before lowercase as tiebreaker.
*/
function biomeSortCompare(a: string, b: string): number {
const minLen = Math.min(a.length, b.length)
for (let i = 0; i < minLen; i++) {
const al = a[i].toLowerCase()
const bl = b[i].toLowerCase()
if (al !== bl) return al < bl ? -1 : 1
if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1
}
return a.length - b.length
}
function writeIconMapping(iconMapping: Record<string, IconRef>): void {
try {
const iconMappingPath = path.join(rootDir, 'apps/docs/components/ui/icon-mapping.ts')
// Add bare-name aliases for versioned block types so trigger provider names resolve correctly.
// e.g. github_v2 → github, fireflies_v2 → fireflies, gmail_v2 → gmail
const withAliases: Record<string, IconRef> = { ...iconMapping }
for (const [blockType, iconRef] of Object.entries(iconMapping)) {
const baseType = stripVersionSuffix(blockType)
if (baseType !== blockType && !withAliases[baseType]) {
withAliases[baseType] = iconRef
}
}
const imports = renderIconImports(Object.values(withAliases))
// Generate mapping with direct references (no dynamic access for tree shaking)
const mappingEntries = Object.entries(withAliases)
.sort(([a], [b]) => compareCatalogNames(a, b))
.map(([blockType, iconRef]) => ` ${formatIconMapKey(blockType)}: ${iconRef.name},`)
.join('\n')
const content = `// Auto-generated file - do not edit manually
// Generated by scripts/generate-docs.ts
// Maps block types to their icon component references
import type { ComponentType, SVGProps } from 'react'
${imports}
type IconComponent = ComponentType<SVGProps<SVGSVGElement>>
export const blockTypeToIconMap: Record<string, IconComponent> = {
${mappingEntries}
}
`
emitGeneratedFile(iconMappingPath, content)
if (!CHECK_ONLY) console.log('✓ Icon mapping file written to docs app')
} catch (error) {
console.error('Error writing icon mapping:', error)
}
}
/**
* Raised when a block's `subBlocks` array is present but cannot be read. Distinguishes
* a parse failure from a block that genuinely exposes no fields — both used to surface
* as an empty array, and the empty array silently strips documented rows.
*/
class SubBlockParseError extends Error {
override name = 'SubBlockParseError'
}
/** Blocks already warned about. The same block is re-parsed by the page pass, the icon pass and
* each spread-base recursion, so without this the same warning prints several times. */
const subBlockParseWarnings = new Set<string>()
/**
* Collects the param names a block exposes to the user through its own `subBlocks`.
*
* A subBlock's `id` is the param it writes, unless it declares `canonicalParamId`,
* which is how a differently-named field maps onto a tool param. A tool param marked
* `visibility: 'hidden'` is not an LLM-settable tool argument, but when the block
* declares a matching field the value is still typed by the user (e.g. Mailchimp's
* `apiKey`) and must stay documented. Params with no matching field are genuinely
* server-derived (Jira's `cloudId`, Salesforce's `idToken`) and stay filtered out.
*
* Brace matching runs on a blanked copy so braces inside string literals and comments
* cannot skew it; only depth-1 properties of each subBlock are read, so `id` fields on
* nested `options`/`condition` objects are never mistaken for the subBlock's own id.
*
* Returns `null` for UNKNOWN — an array whose elements are all spreads of fields arrays this
* scanner cannot follow (`...NotionBlock.subBlocks`, `...getTrigger('x').subBlocks`). `[]` is
* reserved for a block that genuinely exposes no fields, because `[]` strips every hidden param
* from the page. Throws {@link SubBlockParseError} when the array is there but unreadable.
*/
export function extractUserSettableParamIds(
blockContent: string,
blockName = 'block'
): string[] | null {
const scannable = blankStringsAndComments(blockContent)
if (scannable === null) return null
const keyMatch = /\bsubBlocks\s*:/.exec(scannable)
if (!keyMatch) return []
const afterKey = keyMatch.index + keyMatch[0].length
const literalMatch = /^\s*\[/.exec(scannable.slice(afterKey))
if (!literalMatch) {
throw new SubBlockParseError(
`${blockName}: subBlocks is built by an expression rather than an array literal, so the fields it contributes cannot be read`
)
}
const arrayStart = afterKey + literalMatch[0].length - 1
const arrayEnd = findMatchingClose(scannable, arrayStart, '[', ']')
if (arrayEnd === -1) {
throw new SubBlockParseError(
`${blockName}: found a subBlocks array but could not locate its closing bracket`
)
}
const ids = new Set<string>()
let elementsWithoutIds = 0
/**
* Text of the array's own elements with every object literal, call argument and nested
* bracket elided, so each remaining comma-separated segment is one element's head. Used to
* tell an element that names an existing fields array from one that hides its fields behind
* a helper call.
*/
let elementHeads = ''
let nesting = 0
let i = arrayStart + 1
while (i < arrayEnd - 1) {
const char = scannable[i]
if (nesting === 0 && char === '{') {
const objectEnd = findMatchingClose(scannable, i)
if (objectEnd === -1) break
let depth = 0
let topLevel = ''
const sourceIndices: number[] = []
for (let k = i; k < objectEnd; k++) {
const inner = scannable[k]
if (inner === '{' || inner === '[') {
depth++
continue
}
if (inner === '}' || inner === ']') {
depth--
continue
}
if (depth === 1) {
topLevel += inner
sourceIndices.push(k)
}
}
/**
* Matching runs on the blanked characters, so an `id:` sitting inside a string value or a
* `//` comment cannot be mistaken for the subBlock's own id. Blanking keeps a string's
* quotes and its length, so the matched literal's value is read back character by character
* from the original content at the indices the blanked copy matched at.
*/
const readLiteral = (match: RegExpExecArray): string => {
const valueStart = match.index + match[0].length - 1 - match[1].length
let value = ''
for (let offset = 0; offset < match[1].length; offset++) {
value += blockContent[sourceIndices[valueStart + offset]]
}
return value
}
const idMatch = /\bid\s*:\s*['"]([^'"]+)['"]/.exec(topLevel)
if (idMatch) ids.add(readLiteral(idMatch))
const canonicalMatch = /\bcanonicalParamId\s*:\s*['"]([^'"]+)['"]/.exec(topLevel)
if (canonicalMatch) ids.add(readLiteral(canonicalMatch))
/**
* An object that spreads an existing subBlock to override one property
* (`{ ...sb, required: true }`) legitimately carries no id of its own — the id comes from
* the spread source. Only an object with neither an id nor a spread means the scan failed.
*/
if (!idMatch && !canonicalMatch && !topLevel.includes('...')) elementsWithoutIds++
i = objectEnd
continue
}
if (char === '(' || char === '[') {
nesting++
i++
continue
}
if (char === ')' || char === ']') {
nesting--
i++
continue
}
if (nesting === 0) elementHeads += char
i++
}
/**
* Any id at all means the array was read and the page keeps a populated Input table. An
* opaque element alongside real ids can only omit extra rows — the long-standing limitation
* that a spread contributes ids this scanner never sees — and is not this guard's business.
* The guard exists solely to stop an empty result, because empty is what strips every hidden
* param from the page.
*/
if (ids.size > 0) return [...ids]
if (elementsWithoutIds > 0) {
throw new SubBlockParseError(
`${blockName}: subBlocks array holds object literals but no id was extracted`
)
}
/**
* Zero ids is a legitimate answer only when every element names an existing fields array
* (`...NotionBlock.subBlocks`, `...getTrigger('x').subBlocks`, `...Base.subBlocks.filter(…)`),
* because those fields reach the page through the spread base instead. An element that is a
* bare helper call (`...getSlackV2ActionSubBlocks()`) hides whatever fields the helper builds,
* and used to yield a silent empty array indistinguishable from a spread-only block.
*/
const segments = elementHeads
.split(',')
.map((segment) => segment.trim())
.filter((segment) => segment.length > 0)
const opaque = segments.filter((segment) => !segment.includes('.subBlocks'))
if (opaque.length > 0) {
throw new SubBlockParseError(
`${blockName}: subBlocks array yielded no ids and element${
opaque.length > 1 ? 's' : ''
} ${opaque.map((segment) => `\`${segment}\``).join(', ')} do not name a fields array`
)
}
/**
* Every element named a fields array this scanner cannot follow, so the block's fields are
* UNKNOWN, not empty. Returning `[]` here would assert the block supplies nothing and strip
* every hidden param from its tools' Input tables with no warning — the silent false-drop the
* `null` state exists to prevent. Only a genuinely empty array (`subBlocks: []`) reaches the
* `[]` below.
*/
if (segments.length > 0) return null
return []
}
/**
* Locates the bodies of every `tools.config.params` mapper in `scannable`.
*
* Returns `[start, end)` index pairs into `scannable` (a length-preserving blanked copy, so
* the same indices address the original content).
*
* In production this only ever runs on a single block's slice, which holds at most one
* `subBlocks:` key — the loop over every `tools` object is defensive rather than required, and
* the multi-block file it was once justified by (Textract's v1 and v2) is split before it gets
* here. The tests do pass whole files, so the loop is exercised on wider input than production
* ever supplies.
*
* Handles `params: (params) => { ... }`, the concise `params: (params) => ({ ... })` form, the
* `async` and generic-annotated variants, and method shorthand. Candidates are tried in order
* rather than only the first, because a decoy key that is not a mapper at all
* (`params: (GitHubBlock.tools?.config as any)?.params`) would otherwise mask the real one.
*/
function findMapperBodyRanges(scannable: string): [number, number][] {
const ranges: [number, number][] = []
const toolsRegex = /\btools\s*:\s*\{/g
let toolsMatch: RegExpExecArray | null
while ((toolsMatch = toolsRegex.exec(scannable)) !== null) {
const toolsEnd = findMatchingClose(scannable, toolsMatch.index + toolsMatch[0].length - 1)
if (toolsEnd === -1) continue
toolsRegex.lastIndex = toolsEnd
const toolsRegion = scannable.slice(toolsMatch.index, toolsEnd)
const configMatch = /\bconfig\s*:\s*\{/.exec(toolsRegion)
if (!configMatch) continue
const configStart = toolsMatch.index + configMatch.index + configMatch[0].length - 1
const configEnd = findMatchingClose(scannable, configStart)
if (configEnd === -1) continue
const configRegion = scannable.slice(configStart, configEnd)
const paramsRegex = /\bparams\s*(?::\s*(?:async\s*)?(?:<[^<>]*>\s*)?)?\(/g
let paramsMatch: RegExpExecArray | null
while ((paramsMatch = paramsRegex.exec(configRegion)) !== null) {
const argsStart = configStart + paramsMatch.index + paramsMatch[0].length - 1
const argsEnd = findMatchingClose(scannable, argsStart, '(', ')')
if (argsEnd === -1) continue
const afterArgs = scannable.slice(argsEnd, configEnd)
const bodyMatch = /^\s*(?::[^=({]*)?(?:=>\s*)?([({])/.exec(afterArgs)
if (!bodyMatch) continue
const open = bodyMatch[1] as '(' | '{'
const bodyStart = argsEnd + bodyMatch[0].length - 1
const bodyEnd = findMatchingClose(scannable, bodyStart, open, open === '(' ? ')' : '}')
if (bodyEnd === -1) continue
ranges.push([bodyStart, bodyEnd])
break
}
}
return ranges
}
/**
* Adds the shorthand property names of every object literal in `body` to `into`.
*
* `{ file }` names the `file` param exactly as `{ file: value }` does, but carries no colon,
* so the key scan below cannot see it — a mapper written in the idiomatic shorthand form used
* to drop the param from the docs silently, which is the one failure mode this whole filter
* exists to prevent.
*
* Only the depth-1 comma segments of a brace-matched region are read, and a segment that
* opens a call or an index is marked so it can no longer look like a bare identifier. That
* keeps argument lists (`fn(a, b, c)`), calls (`{ doWork() }`) and nested values
* (`{ a: { b: 1 }, file }`) from contributing names, while `{ ...rest, file }` still yields
* `file` because `...rest` is not an identifier on its own.
*/
function collectShorthandPropertyNames(body: string, into: Set<string>): void {
for (let i = 0; i < body.length; i++) {
if (body[i] !== '{') continue
const objectEnd = findMatchingClose(body, i)
if (objectEnd === -1) continue
const segments: string[] = []
let current = ''