Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/test-web-console-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ jobs:
- name: Verify pipeline-manager is reachable
run: curl -fsSL --retry 5 --retry-delay 2 --retry-connrefused http://localhost:8080/healthz

- name: Run vitest integration tests
if: ${{ vars.CI_DRY_RUN != 'true' }}
run: bun run test-integration
working-directory: js-packages/web-console
env:
FELDERA_API_URL: http://localhost:8080

- name: Run Playwright e2e tests
if: ${{ vars.CI_DRY_RUN != 'true' }}
run: bun run test-e2e
Expand Down
3 changes: 2 additions & 1 deletion js-packages/web-console/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@
"test-prepare": "git clone --depth 1 https://github.com/feldera/playwright-snapshots.git || true && mkdir -p playwright-snapshots/e2e playwright-snapshots/component",
"test-update-snapshots": "bun run test -- --update && bun playwright test --update-snapshots",
"test-unit": "vitest",
"test": "bun run test-unit -- --run"
"test-integration": "vitest --run --project integration --project integration-client",
"test": "bun run test-unit -- --run --project client --project server"
},
"trustedDependencies": ["@axa-fr/oidc-client", "sk-oidc-oauth", "svelte-preprocess"],
"type": "module"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
type OutputEndpointStatus
} from '$lib/services/manager'
import { getInputConnectorStatus, getOutputConnectorStatus } from '$lib/services/pipelineManager'
import { getCaseDependentName } from '$lib/functions/felderaRelation'

export type ConnectorErrorFilter = 'all' | 'parse' | 'transport' | 'encode'

Expand Down Expand Up @@ -61,6 +62,8 @@
tagsFilter = filter
})

const strippedConnectorName = $derived(connectorName.slice(getCaseDependentName(relationName).name.length + 1))

$effect(() => {
pipelineName
relationName
Expand All @@ -69,11 +72,10 @@
loading = true
status = null

const stripped = connectorName.slice(connectorName.indexOf('.') + 1)
const request =
direction === 'input'
? getInputConnectorStatus(pipelineName, relationName, stripped)
: getOutputConnectorStatus(pipelineName, relationName, stripped)
? getInputConnectorStatus(pipelineName, relationName, strippedConnectorName)
: getOutputConnectorStatus(pipelineName, relationName, strippedConnectorName)

request.then((s) => {
status = s
Expand Down Expand Up @@ -137,7 +139,7 @@
<div class="bg-white-dark flex h-full flex-col gap-2 rounded p-4">
<div class="flex items-start justify-between">
<div>
<div class="font-medium">{connectorName.replace('.', ' · ')}</div>
<div class="font-medium">{relationName} · {strippedConnectorName}</div>
</div>
<button class="fd fd-x text-[20px]" onclick={() => (open = false)} aria-label="Close"
></button>
Expand Down
5 changes: 1 addition & 4 deletions js-packages/web-console/src/lib/functions/felderaRelation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@ export const normalizeCaseIndependentName = ({
export const getCaseDependentName = (caseIndependentName: string) => {
const case_sensitive = isCaseSensitive(caseIndependentName)
return {
name: normalizeCaseIndependentName({
name: caseIndependentName.replaceAll('"', ''),
case_sensitive
}),
name: caseIndependentName.replaceAll('"', ''),
case_sensitive
}
}
Expand Down
125 changes: 125 additions & 0 deletions js-packages/web-console/src/lib/services/pipelineManager.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/**
* Integration tests for pipelineManager.ts that require a running Feldera instance.
* Run via the 'integration' vitest project:
*
* FELDERA_API_URL=http://localhost:8080 bun run test-integration
*/
import { afterAll, beforeAll, describe, expect, it } from 'vitest'

const BASE_URL = process.env.FELDERA_API_URL ?? 'http://localhost:8080'

// Pipeline name is URL-safe by convention, but table/view/connector names are not.
const PIPELINE_NAME = 'test-special-chars-connector-status'

// Names with dots and URL-unsafe characters
const TABLE_NAME = 'my.input.table'
const VIEW_NAME = 'my.output.view'
const INPUT_CONNECTOR_NAME = 'http.input/special&name'
const OUTPUT_CONNECTOR_NAME = 'http.output/special&name'

const api = (path: string, init?: RequestInit) =>
fetch(`${BASE_URL}${path}`, {
...init,
headers: { 'Content-Type': 'application/json', ...init?.headers }
})

describe('pipelineManager connector status with special characters', () => {
beforeAll(async () => {
// Clean up any leftover pipeline from a previous run
const del = await api(`/v0/pipelines/${PIPELINE_NAME}`, { method: 'DELETE' })
if (del.ok || del.status === 404) {
// ok
} else {
throw new Error(`Cleanup failed: ${del.status} ${await del.text()}`)
}

// SQL program with quoted identifiers containing dots
const programCode = `
CREATE TABLE "${TABLE_NAME}" (id INT NOT NULL, val VARCHAR)
WITH (
'connectors' = '[{
"name": "${INPUT_CONNECTOR_NAME}",
"transport": { "name": "http_input" },
"format": { "name": "json", "config": { "update_format": "raw" } }
}]'
);
CREATE VIEW "${VIEW_NAME}" AS SELECT * FROM "${TABLE_NAME}"
WITH (
'connectors' = '[{
"name": "${OUTPUT_CONNECTOR_NAME}",
"transport": { "name": "http_output" },
"format": { "name": "json", "config": { "update_format": "raw" } }
}]'
);
`

// Create the pipeline
const put = await api(`/v0/pipelines/${PIPELINE_NAME}`, {
method: 'PUT',
body: JSON.stringify({
name: PIPELINE_NAME,
description: 'Integration test for special character handling',
program_code: programCode,
runtime_config: {}
})
})
expect(put.ok, `PUT pipeline: ${put.status} ${await put.clone().text()}`).toBe(true)

// Wait for compilation
for (let i = 0; i < 120; i++) {
const res = await api(`/v0/pipelines/${PIPELINE_NAME}`)
const info = await res.json()
if (info.program_status === 'Success') break
if (info.program_status?.SqlError || info.program_status?.RustError || info.program_status?.SystemError) {
throw new Error(`Compilation failed: ${JSON.stringify(info.program_status)}`)
}
await new Promise((r) => setTimeout(r, 1000))
}

// Start the pipeline
const start = await api(`/v0/pipelines/${PIPELINE_NAME}/start`, { method: 'POST' })
expect(start.ok, `POST start: ${start.status} ${await start.clone().text()}`).toBe(true)

// Wait for pipeline to be running
for (let i = 0; i < 60; i++) {
const res = await api(`/v0/pipelines/${PIPELINE_NAME}`)
const info = await res.json()
if (info.deployment_status === 'Running') break
if (info.deployment_error) {
throw new Error(`Deployment failed: ${JSON.stringify(info.deployment_error)}`)
}
await new Promise((r) => setTimeout(r, 1000))
}
}, 180_000)

afterAll(async () => {
// Shutdown and delete
await api(`/v0/pipelines/${PIPELINE_NAME}/shutdown`, { method: 'POST' })
// Wait for shutdown
for (let i = 0; i < 30; i++) {
const res = await api(`/v0/pipelines/${PIPELINE_NAME}`)
const info = await res.json()
if (info.deployment_status === 'Shutdown') break
await new Promise((r) => setTimeout(r, 1000))
}
await api(`/v0/pipelines/${PIPELINE_NAME}`, { method: 'DELETE' })
}, 60_000)

it('getInputConnectorStatus succeeds with dot and URL-unsafe characters in table and connector name', async () => {
const url = `/v0/pipelines/${PIPELINE_NAME}/tables/${encodeURIComponent(TABLE_NAME)}/connectors/${encodeURIComponent(INPUT_CONNECTOR_NAME)}/status`
const res = await api(url)
expect(res.ok, `GET input connector status: ${res.status} ${await res.clone().text()}`).toBe(true)
const body = await res.json()
expect(body).toHaveProperty('num_parse_errors')
expect(body).toHaveProperty('num_transport_errors')
})

it('getOutputConnectorStatus succeeds with dot and URL-unsafe characters in view and connector name', async () => {
const url = `/v0/pipelines/${PIPELINE_NAME}/views/${encodeURIComponent(VIEW_NAME)}/connectors/${encodeURIComponent(OUTPUT_CONNECTOR_NAME)}/status`
const res = await api(url)
expect(res.ok, `GET output connector status: ${res.status} ${await res.clone().text()}`).toBe(true)
const body = await res.json()
expect(body).toHaveProperty('num_encode_errors')
expect(body).toHaveProperty('num_transport_errors')
})
})
14 changes: 7 additions & 7 deletions js-packages/web-console/src/lib/services/pipelineManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ export const getExtendedPipeline = async (
) => {
return mapResponse(
_getPipeline({
path: { pipeline_name: encodeURIComponent(pipeline_name) },
path: { pipeline_name },
...options
}),
toExtendedPipeline,
Expand Down Expand Up @@ -398,7 +398,7 @@ export const putPipeline = async (
await mapResponse(
_putPipeline({
body: newPipeline,
path: { pipeline_name: encodeURIComponent(pipeline_name) },
path: { pipeline_name },
...options
}),
(v) => v
Expand All @@ -412,7 +412,7 @@ export const patchPipeline = async (
) => {
return mapResponse(
_patchPipeline({
path: { pipeline_name: encodeURIComponent(pipeline_name) },
path: { pipeline_name },
body: fromPipeline(pipeline),
...options
}),
Expand All @@ -433,7 +433,7 @@ export const getPipelines = async (options?: FetchOptions): Promise<PipelineThum
export const getPipelineStatus = async (pipeline_name: string, options?: FetchOptions) => {
return mapResponse(
_getPipeline({
path: { pipeline_name: encodeURIComponent(pipeline_name) },
path: { pipeline_name },
query: { selector: 'status' },
...options
}),
Expand All @@ -450,7 +450,7 @@ export const getPipelineStatus = async (pipeline_name: string, options?: FetchOp
export const getPipelineStats = async (pipeline_name: string, options?: FetchOptions) => {
return mapResponse(
_getPipelineStats({
path: { pipeline_name: encodeURIComponent(pipeline_name) },
path: { pipeline_name },
...options
}),
(status) => ({
Expand Down Expand Up @@ -578,7 +578,7 @@ export const getClusterEvent = (eventId: string) =>
const getSamplyProfileStream = (pipelineName: string, latest: boolean) => {
const result = streamingFetch(
getAuthenticatedFetch(),
`${felderaEndpoint}/v0/pipelines/${encodeURIComponent(pipelineName)}/samply_profile${latest ? '?latest=true' : ''}`,
`${felderaEndpoint}/v0/pipelines/${pipelineName}/samply_profile${latest ? '?latest=true' : ''}`,
{
method: 'GET'
}
Expand Down Expand Up @@ -967,5 +967,5 @@ export const getPipelineSupportBundleUrl = (
for (const [key, value] of Object.entries(options)) {
query.append(key, String(value))
}
return `${felderaEndpoint}/v0/pipelines/${encodeURIComponent(pipelineName)}/support_bundle?${query.toString()}`
return `${felderaEndpoint}/v0/pipelines/${pipelineName}/support_bundle?${query.toString()}`
}
68 changes: 45 additions & 23 deletions js-packages/web-console/vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { playwright } from '@vitest/browser-playwright'
import { type PluginOption } from 'vite'
import devtoolsJson from 'vite-plugin-devtools-json'
import virtual from 'vite-plugin-virtual'
import { defineConfig, type ViteUserConfigExport } from 'vitest/config'
import { defineConfig, type TestProjectInlineConfiguration, type ViteUserConfigExport } from 'vitest/config'
import { felderaApiJsonSchemas } from './src/lib/functions/felderaApiJsonSchemas'
import { svelteCssVirtualModuleFallback } from './src/lib/vite-plugins/svelte-css-virtual-module-fallback'

Expand Down Expand Up @@ -61,6 +61,36 @@ const testOptimizeDepsInclude = [
'virtua/svelte'
]

const browserTestProject = ({
name,
include,
exclude
}: {
name: string
include: string[]
exclude?: string[]
}): TestProjectInlineConfiguration => ({
extends: './vite.config.ts',
test: {
name,
browser: {
enabled: true,
provider: playwright({ contextOptions: {} }),
instances: [{ browser: 'chromium', headless: true }],
expect: {
toMatchScreenshot: {
resolveScreenshotPath({ testFileName, arg, ext }) {
return path.join(snapshotsDir, 'component', testFileName, `${arg}${ext}`)
}
}
}
},
setupFiles: ['src/lib/vitest-browser-setup.ts'],
include,
exclude: exclude ?? ['src/lib/server/**']
}
})

// TODO: remove Prettier
export default defineConfig(async () => {
return {
Expand Down Expand Up @@ -179,39 +209,31 @@ export default defineConfig(async () => {
)
},
projects: [
// Unit tests: *.spec.ts (run with `bun run test`)
browserTestProject({ name: 'client', include: ['src/**/*.svelte.spec.{js,ts}'] }),

{
extends: './vite.config.ts',
test: {
name: 'client',
browser: {
enabled: true,
provider: playwright({
contextOptions: {}
}),
instances: [{ browser: 'chromium', headless: true }],
expect: {
toMatchScreenshot: {
resolveScreenshotPath({ testFileName, arg, ext }) {
return path.join(snapshotsDir, 'component', testFileName, `${arg}${ext}`)
}
}
}
},
setupFiles: ['src/lib/vitest-browser-setup.ts'],
include: ['src/**/*.svelte.{test,spec}.{js,ts}'],
exclude: ['src/lib/server/**']
name: 'server',
environment: 'node',
include: ['src/**/*.spec.{js,ts}'],
exclude: ['src/**/*.svelte.spec.{js,ts}']
}
},

// Integration tests: *.test.ts (require a Feldera instance, run with `bun run test-integration`)
{
extends: './vite.config.ts',
test: {
name: 'server',
name: 'integration',
environment: 'node',
include: ['src/**/*.{test,spec}.{js,ts}'],
exclude: ['src/**/*.svelte.{test,spec}.{js,ts}']
include: ['src/**/*.test.{js,ts}'],
exclude: ['src/**/*.svelte.test.{js,ts}']
}
}
},

browserTestProject({ name: 'integration-client', include: ['src/**/*.svelte.test.{js,ts}'] })
]
}
} satisfies ViteUserConfigExport
Expand Down
Loading