Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions js-packages/web-console/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"@fontsource-variable/dm-sans": "5.1.1",
"@fontsource/dm-mono": "5.1.1",
"@fortawesome/fontawesome-free": "6.7.2",
"@githubnext/tiny-svelte-fsm": "^1.0.0",
"@hey-api/client-fetch": "0.1.10",
"@hey-api/openapi-ts": "0.47.2",
"@monaco-editor/loader": "1.6.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
import {
programStatusOf,
type ExtendedPipeline,
type Pipeline,
type PipelineAction,
type PipelineThumb
} from '$lib/services/pipelineManager'
import {
Expand Down Expand Up @@ -95,17 +93,11 @@
const pipelineAction = usePipelineAction()

const api = usePipelineManager()
const pipelineActionCallbacks = usePipelineActionCallbacks()
const handleActionSuccess = async (pipelineName: string, action: PipelineAction) => {
const cbs = pipelineActionCallbacks.getAll(pipelineName, action)
await Promise.allSettled(cbs.map((x) => x(pipelineName)))
}
// Initialize the callback system to start reactive monitoring
usePipelineActionCallbacks(preloaded)

const handleDeletePipeline = async (pipelineName: string) => {
updatePipelines((pipelines) => pipelines.filter((p) => p.name !== pipelineName))
const cbs = pipelineActionCallbacks
.getAll('', 'delete')
.concat(pipelineActionCallbacks.getAll(pipelineName, 'delete'))
cbs.map((x) => x(pipelineName))
}

const programErrors = $derived(
Expand Down Expand Up @@ -298,11 +290,7 @@ example = "1.0"`
{changes}
onCancel={() => (contextDrawer.content = null)}
onApprove={async () => {
const { waitFor } = await pipelineAction.postPipelineAction(
pipeline.current.name,
'approve_changes'
)
await waitFor()
await pipelineAction.postPipelineAction(pipeline.current.name, 'approve_changes')
}}
>
{#snippet titleEnd()}
Expand All @@ -323,7 +311,6 @@ example = "1.0"`
onDeletePipeline={handleDeletePipeline}
{editConfigDisabled}
unsavedChanges={downstreamChanged}
onActionSuccess={handleActionSuccess}
{saveFile}
></PipelineActions>
{/snippet}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
}
})

const pipelineActionCallbacks = usePipelineActionCallbacks()
const pipelineActionCallbacks = getPipelineActionCallbacks()
const dropOpenedFile = async (pipelineName: string) => {
const files = ['program.sql', 'stubs.rs', 'udf.rs', 'udf.toml'].map(
(file) => `${pipelineName}/${file}`
Expand Down Expand Up @@ -63,7 +63,7 @@
import { pipelineFileNameRegex } from '$lib/compositions/health/systemErrors'
import { effectMonacoContentPlaceholder } from '$lib/components/monacoEditor/effectMonacoContentPlaceholder.svelte'
import { GenericOverlayWidget } from '$lib/components/monacoEditor/GenericOverlayWidget'
import { usePipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { getPipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { useCodeEditorSettings } from '$lib/compositions/pipelines/useCodeEditorSettings.svelte'
import { rgbToHex } from '$lib/functions/common/color'
import type { $ } from 'bun'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<script lang="ts" module>
const pipelineActionCallbacks = usePipelineActionCallbacks()
const pipelineActionCallbacks = getPipelineActionCallbacks()
</script>

<script lang="ts">
Expand All @@ -13,7 +13,7 @@
import { tuple } from '$lib/functions/common/tuple'
import type { ExtendedPipeline } from '$lib/services/pipelineManager'
import type { PipelineMetrics } from '$lib/functions/pipelineMetrics'
import { usePipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { getPipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { count } from '$lib/functions/common/array'
import { untrack } from 'svelte'
import ClipboardCopyButton from '$lib/components/other/ClipboardCopyButton.svelte'
Expand Down Expand Up @@ -80,7 +80,7 @@
$effect(() => {
untrack(() => pipelineActionCallbacks.add('', 'delete', forgetCurrentTab))
return () => {
pipelineActionCallbacks.remove('', 'start_paused', forgetCurrentTab)
pipelineActionCallbacks.remove('', 'delete', forgetCurrentTab)
}
})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
let pipelinesRelations = $state<
Record<string, Record<string, Record<string, ExtraType & { type: 'tables' | 'views' }>>>
>({})
const pipelineActionCallbacks = usePipelineActionCallbacks()
const pipelineActionCallbacks = getPipelineActionCallbacks()
let changeStream: Record<string, Record<string, ChangeStreamData>> = {} // Initialize row array nested by tenant and pipeline
// Separate getRows as a $state avoids burdening rows array itself with reactivity overhead
let getChangeStream = $state(() => changeStream)
Expand Down Expand Up @@ -220,7 +220,7 @@
</script>

<script lang="ts">
import { usePipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { getPipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'

import {
getCaseIndependentName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
}
> = {}
let getStreams = $state(() => streams)
const pipelineActionCallbacks = usePipelineActionCallbacks()
const pipelineActionCallbacks = getPipelineActionCallbacks()
const dropLogHistory = async (pipelineName: string) => {
if ('open' in streams[pipelineName].stream) {
streams[pipelineName].stream.stop()
Expand All @@ -32,7 +32,7 @@
SplitNewlineTransformStream
} from '$lib/functions/pipelines/changeStream'
import { type ExtendedPipeline, type PipelineStatus } from '$lib/services/pipelineManager'
import { usePipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { getPipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import { untrack } from 'svelte'
import WarningBanner from '$lib/components/pipelines/editor/WarningBanner.svelte'
import { useInterval } from '$lib/compositions/common/useInterval.svelte'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,23 +62,20 @@ groups related actions into multi-action dropdowns when multiple options are ava
import { slide } from 'svelte/transition'
import { useIsMobile } from '$lib/compositions/layout/useIsMobile.svelte'
import { usePipelineAction } from '$lib/compositions/usePipelineAction.svelte'
import { usePipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte'
import type { WritablePipeline } from '$lib/compositions/useWritablePipeline.svelte'

let {
pipeline,
onDeletePipeline,
editConfigDisabled,
unsavedChanges,
onActionSuccess,
saveFile,
class: _class = ''
}: {
pipeline: WritablePipeline
onDeletePipeline?: (pipelineName: string) => void
editConfigDisabled: boolean
unsavedChanges: boolean
onActionSuccess?: (pipelineName: string, action: PipelineAction) => void
saveFile: () => void
class?: string
} = $props()
Expand Down Expand Up @@ -330,28 +327,13 @@ groups related actions into multi-action dropdowns when multiple options are ava
const importantBtnColor = 'preset-filled-primary-500'

const { postPipelineAction } = usePipelineAction()
const pipelineActionCallbacks = usePipelineActionCallbacks()

const performStartAction = async (
action: PipelineAction,
pipelineName: string
// nextStatus: PipelineStatus
) => {
const callbacks =
action === 'start'
? {
onPausedReady: async (pipelineName: string) => {
const cbs = pipelineActionCallbacks.getAll(pipelineName, 'start_paused')
await Promise.allSettled(cbs.map((x) => x(pipelineName)))
}
}
: undefined

const { waitFor } = await postPipelineAction(pipeline.current.name, action, callbacks)
waitFor().then(
(shouldContinue) => shouldContinue && onActionSuccess?.(pipelineName, action),
toastError
)
const performStartAction = async (action: PipelineAction) => {
try {
await postPipelineAction(pipeline.current.name, action)
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
}

// Static multi-action dropdown configurations
Expand All @@ -372,8 +354,7 @@ groups related actions into multi-action dropdowns when multiple options are ava
label: 'Start',
description: 'Start the pipeline normally',
onclick: () => {
const pipelineName = pipeline.current.name
performStartAction('start', pipelineName)
performStartAction('start')
},
disabled: () => unsavedChanges,
disabledText: 'Save First',
Expand All @@ -383,8 +364,7 @@ groups related actions into multi-action dropdowns when multiple options are ava
label: 'Start as Paused',
description: 'Start the pipeline in a paused state',
onclick: () => {
const pipelineName = pipeline.current.name
performStartAction('start_paused', pipelineName)
performStartAction('start_paused')
},
disabled: () => unsavedChanges,
disabledText: 'Save First',
Expand All @@ -394,8 +374,7 @@ groups related actions into multi-action dropdowns when multiple options are ava
label: 'Resume',
description: 'Resume the paused pipeline',
onclick: () => {
const pipelineName = pipeline.current.name
performStartAction('resume', pipelineName)
performStartAction('resume')
},
disabled: () => unsavedChanges,
disabledText: 'Save First',
Expand All @@ -420,9 +399,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
label: 'Pause',
description: 'Pause the running pipeline',
onclick: async () => {
const pipelineName = pipeline.current.name
const { waitFor } = await postPipelineAction(pipelineName, 'pause')
waitFor().then(() => onActionSuccess?.(pipelineName, 'pause'), toastError)
try {
await postPipelineAction(pipeline.current.name, 'pause')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
},
disabled: () => false,
standaloneButton: _pause
Expand All @@ -431,9 +412,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
label: 'Start in Standby',
description: 'Put the pipeline in standby mode',
onclick: async () => {
const pipelineName = pipeline.current.name
const { waitFor } = await postPipelineAction(pipelineName, 'standby')
waitFor().then(() => onActionSuccess?.(pipelineName, 'standby'), toastError)
try {
await postPipelineAction(pipeline.current.name, 'standby')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
},
disabled: () => false,
standaloneButton: _standby
Expand All @@ -442,9 +425,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
label: 'Activate',
description: 'Activate the pipeline to start data ingress and processing',
onclick: async () => {
const pipelineName = pipeline.current.name
const { waitFor } = await postPipelineAction(pipelineName, 'activate')
waitFor().then(() => onActionSuccess?.(pipelineName, 'activate'), toastError)
try {
await postPipelineAction(pipeline.current.name, 'activate')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
},
disabled: () => false,
standaloneButton: _activate
Expand All @@ -458,8 +443,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
'Clear',
(name) => `Clear ${name} pipeline storage?`,
async (pipelineName: string) => {
const { waitFor } = await postPipelineAction(pipelineName, 'clear')
waitFor().then(() => {}, toastError)
try {
await postPipelineAction(pipelineName, 'clear')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
},
'This will delete all checkpoints.'
)(pipeline.current.name)}
Expand All @@ -486,8 +474,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
'Force stop',
(name) => `Force stop ${name} pipeline?`,
async (pipelineName: string) => {
const { waitFor } = await postPipelineAction(pipelineName, 'kill')
waitFor().then(() => onActionSuccess?.(pipelineName, 'kill'), toastError)
try {
await postPipelineAction(pipelineName, 'kill')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
},
'The pipeline will stop processing inputs without making a checkpoint, leaving only a previous one, if any.'
)(pipeline.current.name)}
Expand All @@ -501,8 +492,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
'Stop',
(name) => `Stop ${name} pipeline?`,
async (pipelineName: string) => {
const { waitFor } = await postPipelineAction(pipelineName, 'stop')
waitFor().then(() => onActionSuccess?.(pipelineName, 'stop'), toastError)
try {
await postPipelineAction(pipelineName, 'stop')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
},
'The pipeline will stop processing inputs and make a checkpoint of its state.'
)(pipeline.current.name)}
Expand Down Expand Up @@ -661,8 +655,7 @@ groups related actions into multi-action dropdowns when multiple options are ava
return
}
const action = getAction(e.ctrlKey || e.shiftKey || e.metaKey)
const pipelineName = pipeline.current.name
performStartAction(action, pipelineName)
performStartAction(action)
}}
>
<span class="fd fd-play {iconClass}"></span>
Expand Down Expand Up @@ -748,9 +741,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
<button
class="hidden sm:flex {buttonClass} {longClass} {basicBtnColor}"
onclick={async () => {
const pipelineName = pipeline.current.name
const { waitFor } = await postPipelineAction(pipelineName, 'pause')
waitFor().then(() => onActionSuccess?.(pipelineName, 'pause'), toastError)
try {
await postPipelineAction(pipeline.current.name, 'pause')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
}}
>
<span class="fd fd-pause {iconClass}"></span>
Expand All @@ -760,9 +755,11 @@ groups related actions into multi-action dropdowns when multiple options are ava
<button
class="flex sm:hidden {buttonClass} {shortClass} {basicBtnColor} {iconClass}"
onclick={async () => {
const pipelineName = pipeline.current.name
const { waitFor } = await postPipelineAction(pipelineName, 'pause')
waitFor().then(() => onActionSuccess?.(pipelineName, 'pause'), toastError)
try {
await postPipelineAction(pipeline.current.name, 'pause')
} catch (error) {
toastError(error instanceof Error ? error : new Error(String(error)))
}
}}
>
<span class="fd fd-pause {iconClass}"></span>
Expand Down
Loading