Skip to content

pipeline-manager: pipeline monitoring events #5446

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
snkas wants to merge 2 commits into
base: main
Choose a base branch
from
Open

pipeline-manager: pipeline monitoring events #5446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8959912
pipeline-manager: pipeline monitoring events
snkas Feb 3, 2026
431c197
Implement Pipeline Events UI
Karakatiza666 Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion crates/fda/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ use std::fmt::Display;
use std::path::PathBuf;

use crate::make_client;
use feldera_rest_api::types::{ClusterMonitorEventFieldSelector, CompilationProfile};
use feldera_rest_api::types::{
ClusterMonitorEventFieldSelector, CompilationProfile, PipelineMonitorEventFieldSelector,
};

/// Autocompletion for pipeline names by trying to fetch them from the server.
fn pipeline_names(current: &std::ffi::OsStr) -> Vec<CompletionCandidate> {
Expand Down Expand Up @@ -697,6 +699,23 @@ pub enum PipelineAction {
#[command(flatten)]
args: BenchmarkArgs,
},
/// Retrieves all pipeline events (status only) and prints them.
Events {
/// The name of the pipeline.
#[arg(value_hint = ValueHint::Other, add = ArgValueCompleter::new(pipeline_names))]
name: String,
},
/// Retrieve specific pipeline event.
Event {
/// The name of the pipeline.
#[arg(value_hint = ValueHint::Other, add = ArgValueCompleter::new(pipeline_names))]
name: String,
/// Identifier (UUID) of the event or `latest`.
id: String,
/// Either `all` or `status` (default).
#[arg(default_value = "status")]
selector: PipelineMonitorEventFieldSelector,
},
}

#[derive(Args, Debug)]
Expand Down
155 changes: 155 additions & 0 deletions crates/fda/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1864,6 +1864,161 @@ async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client)
println!("Initiated rebalancing for pipeline {name}.");
}
PipelineAction::Bench { args } => bench::bench(client, format, args).await,
PipelineAction::Events { name } => {
let response = client
.list_pipeline_events()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().clone(),
"Unable to retrieve pipeline events",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![];
rows.push([
"id".to_string(),
"recorded_at".to_string(),
"resources_status".to_string(),
"resources_desired_status".to_string(),
"runtime_status".to_string(),
"runtime_desired_status".to_string(),
"program_status".to_string(),
"storage_status".to_string(),
]);
for event in response.iter() {
rows.push([
event.id.to_string(),
format!(
"{} ({:.0}s ago)",
event.recorded_at,
(Utc::now() - event.recorded_at).as_seconds_f64()
),
event.resources_status.to_string(),
event.resources_desired_status.to_string(),
event
.runtime_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event
.runtime_desired_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event.program_status.to_string(),
event.storage_status.to_string(),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline events")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Event { name, id, selector } => {
let response = client
.get_pipeline_event()
.pipeline_name(name.clone())
.event_id(id)
.selector(selector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().clone(),
"Unable to retrieve pipeline event",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![];
rows.push(["Field".to_string(), "Value".to_string()]);
rows.push(["id".to_string(), response.id.to_string()]);
rows.push([
"recorded_at".to_string(),
format!(
"{} ({:.0}s ago)",
response.recorded_at,
(Utc::now() - response.recorded_at).as_seconds_f64()
),
]);
rows.push([
"resources_status".to_string(),
response.resources_status.to_string(),
]);
if selector == PipelineMonitorEventFieldSelector::All
&& let Some(value) = &response.resources_status_details
{
rows.push(["resources_status_details".to_string(), value.to_string()]);
}
rows.push([
"resources_desired_status".to_string(),
response.resources_desired_status.to_string(),
]);
rows.push([
"runtime_status".to_string(),
response
.runtime_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
if selector == PipelineMonitorEventFieldSelector::All {
rows.push([
"runtime_status_details".to_string(),
response
.runtime_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
}
rows.push([
"runtime_desired_status".to_string(),
response
.runtime_desired_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
rows.push([
"program_status".to_string(),
response.program_status.to_string(),
]);
rows.push([
"storage_status".to_string(),
response.storage_status.to_string(),
]);
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline events")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
}
}

Expand Down
2 changes: 2 additions & 0 deletions crates/fda/test.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ fda program set-config p1 --profile dev
fda program set-config p1 --profile optimized
fda program config p1
fda program status p1
fda events p1
fda event p1 latest

fda create pudf program.sql --udf-toml udf.toml --udf-rs udf.rs
compare_output "fda program get pudf --udf-toml" "cat udf.toml"
Expand Down
16 changes: 16 additions & 0 deletions crates/pipeline-manager/migrations/V32__pipeline_monitor_event.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- Regularly the status of the pipeline is observed and stored in this table.
-- The rows in this table are regularly cleaned up to prevent it growing unbound.
Copy link
Copy Markdown
Contributor

@mihaibudiu mihaibudiu Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unbounded

CREATE TABLE IF NOT EXISTS pipeline_monitor_event (
id UUID PRIMARY KEY NOT NULL, -- Unique event identifier.
pipeline_id UUID NOT NULL, -- Identifier of the pipeline the event corresponds to.
recorded_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL, -- Timestamp when status recording started.
resources_status VARCHAR NOT NULL, -- Resources status.
resources_status_details VARCHAR NOT NULL, -- Resources status details.
resources_desired_status VARCHAR NOT NULL, -- Resources desired status.
runtime_status VARCHAR NULL, -- Runtime status (only set during deployment).
runtime_status_details VARCHAR NULL, -- Runtime status details (only set during deployment).
runtime_desired_status VARCHAR NULL, -- Runtime desired status (only set during deployment).
program_status VARCHAR NOT NULL, -- Program status.
storage_status VARCHAR NOT NULL, -- Storage status.
FOREIGN KEY (pipeline_id) REFERENCES pipeline(id) ON DELETE CASCADE
);
Copy link
Copy Markdown

@mythical-fred mythical-fred Feb 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No index on pipeline_id. All queries in pipeline_monitor.rs filter by pipeline_id but PostgreSQL does not auto-create an index for foreign-key referencing columns. Please add:

CREATE INDEX ON pipeline_monitor_event (pipeline_id, recorded_at DESC);

This covers the list (ORDER BY recorded_at DESC), latest-event, and cleanup queries.

2 changes: 2 additions & 0 deletions crates/pipeline-manager/proptest-regressions/db/test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,5 @@ cc c0ddd7741bf1667b3d70ea81ca4c298875235277256ee70f1fe2b6a9a50303ba # shrinks to
cc 126dfeb51d3f8f4707aaf4d12a3d6f90a0ac9fe7f12cffcaa014e8604dfb5f3c # shrinks to [NewPipeline(TenantId(01000000-0000-0000-0000-000000000000), 03000000-0000-0000-0000-000000000000, "v0", PipelineDescr { name: "pipeline-4", description: "", runtime_config: Object {"workers": Number(0), "storage": Null, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(false), "tracing": Bool(false), "tracing_endpoint_jaeger": String(""), "min_batch_size_records": Number(0), "max_buffering_delay_usecs": Number(0), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Null, "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Null, "storage_class": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(false), "dev_tweaks": Object {}}, program_code: "", udf_rust: "", udf_toml: "", program_config: Object {"profile": String("optimized"), "cache": Bool(false)} }), TransitProgramStatusToCompilingSql(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), TransitProgramStatusToSqlCompiled(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), SqlCompilationInfo { exit_code: 0, messages: [] }, Object {"schema": Object {"inputs": Array [], "outputs": Array []}, "main_rust": String("main-rust-0"), "udf_stubs": String("udf-stubs-0"), "dataflow": Null, "input_connectors": Object {}, "output_connectors": Object {}}), TransitProgramStatusToCompilingRust(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), TransitProgramStatusToSuccess(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), RustCompilationInfo { exit_code: 0, stdout: "stdout-0", stderr: "stderr-0" }, "source_checksum_0", "integrity_checksum_0", ""), NewOrUpdatePipeline(TenantId(01000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "pipeline-4", "v0", PipelineDescr { name: "pipeline-3", description: "\u{11cb6}^ $gn𑍐dÊ'kڦ=~\u{1cf42}>𖭯ℶ🛟q", runtime_config: Object {"workers": Number(9614), "storage": Object {"backend": Object {"name": String("default")}, "min_storage_bytes": Null, "min_step_storage_bytes": Null, "compression": String("default"), "cache_mib": Null}, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(true), "tracing": Bool(false), "tracing_endpoint_jaeger": String("\\&🕴"), "min_batch_size_records": Number(9821004243620302662), "max_buffering_delay_usecs": Number(409483829293609342), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Number(2119326179280987330), "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Number(7151796549280257912), "storage_class": Null}, "clock_resolution_usecs": Number(9929288629303979490), "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(true), "dev_tweaks": Object {}}, program_code: "ெ:ඍ𑎄2𝒪]₄&&𑶨<𑃠/🏿ꬵஓP", udf_rust: "=ঊോ🈕স?T𐀇ὙὛ*lૐ>X\u{1e00b}(ȺਠE🕴=Ѩ﷏;1", udf_toml: ".5oy�%ᏺj𖮎𐨓,", program_config: Object {"profile": Null, "cache": Bool(true)} }), TransitProgramStatusToSystemError(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(2), "יּ?!\u{113c7}<%\u{1e023},&ꠑÐ𐑠"), GetPipelineById(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000))]
cc 849625f53652fb9e6c8f1438bcc512d6d2f5a8abe49286048fce5fd6a1a71c87
cc 4058ca3fe2e46f2fb8bf26a0f86d8e2036b07b1ddd469b6af343004ae67373c0 # shrinks to [NewOrUpdatePipeline(TenantId(01000000-0000-0000-0000-000000000000), 03000000-0000-0000-0000-000000000000, "pipeline-3", "v0", false, PipelineDescr { name: "pipeline-3", description: "", runtime_config: Object {"workers": Number(0), "storage": Null, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(false), "tracing": Bool(false), "tracing_endpoint_jaeger": String(""), "min_batch_size_records": Number(0), "max_buffering_delay_usecs": Number(0), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Null, "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Null, "storage_class": Null, "service_account_name": Null, "namespace": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(false), "http_workers": Null, "io_workers": Null, "dev_tweaks": Object {}, "logging": Null}, program_code: "", udf_rust: "", udf_toml: "", program_config: Object {"profile": String("optimized"), "cache": Bool(false), "runtime_version": Null} }), TransitProgramStatusToCompilingSql(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), TransitProgramStatusToSqlCompiled(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), SqlCompilationInfo { exit_code: 1, messages: [] }, Object {"schema": Object {"inputs": Array [], "outputs": Array []}, "main_rust": String("main-rust-0"), "udf_stubs": String("udf-stubs-0"), "dataflow": Null, "input_connectors": Object {}, "output_connectors": Object {}}), TransitProgramStatusToCompilingRust(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), ClearOngoingRustCompilation("v0"), GetNextRustCompilation("v0")]
cc 4e812fc686724fb39aff5c4f0dbde67e36f477522457ca2b9fa5e5374c9ce61f # shrinks to [NewOrUpdatePipeline(TenantId(03000000-0000-0000-0000-000000000000), 03000000-0000-0000-0000-000000000000, "pipeline-4", "v2", false, PipelineDescr { name: "pipeline-4", description: "", runtime_config: Object {"workers": Number(0), "storage": Null, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(false), "tracing": Bool(false), "tracing_endpoint_jaeger": String(""), "min_batch_size_records": Number(0), "max_buffering_delay_usecs": Number(0), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Null, "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Null, "storage_class": Null, "service_account_name": Null, "namespace": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(false), "http_workers": Null, "io_workers": Null, "dev_tweaks": Object {}, "logging": Null, "pipeline_template_configmap": Null}, program_code: "", udf_rust: "", udf_toml: "", program_config: Object {"profile": String("optimized"), "cache": Bool(false), "runtime_version": Null} }), ClearOngoingSqlCompilation("v0"), SetDeploymentResourcesDesiredStatusProvisioned(TenantId(03000000-0000-0000-0000-000000000000), "pipeline-4", Paused), TransitDeploymentResourcesStatusToStopping(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(2), None, None), ListPipelineMonitorEvents(TenantId(03000000-0000-0000-0000-000000000000), "pipeline-4")]
cc db90e416d47dfca52aa161e45eb457083c8b2009dcce37f072d3173889981432 # shrinks to [NewPipeline(TenantId(03000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "v0", PipelineDescr { name: "pipeline-1", description: "", runtime_config: Object {"workers": Number(0), "hosts": Number(0), "storage": Null, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(false), "tracing": Bool(false), "tracing_endpoint_jaeger": String(""), "min_batch_size_records": Number(0), "max_buffering_delay_usecs": Number(0), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Null, "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Null, "storage_class": Null, "service_account_name": Null, "namespace": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(false), "http_workers": Null, "io_workers": Null, "dev_tweaks": Object {}, "logging": Null, "pipeline_template_configmap": Null}, program_code: "", udf_rust: "", udf_toml: "", program_config: Object {"profile": String("optimized"), "cache": Bool(false), "runtime_version": Null} }), UpdatePipeline(TenantId(03000000-0000-0000-0000-000000000000), "pipeline-1", None, None, "v0", false, None, None, None, None, None), ListPipelineMonitorEvents(TenantId(03000000-0000-0000-0000-000000000000), "pipeline-1")]
2 changes: 2 additions & 0 deletions crates/pipeline-manager/src/api/endpoints/pipeline_management.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
pub mod pipeline_events;

use crate::api::error::ApiError;
use crate::api::examples;
use crate::api::main::ServerState;
Expand Down
Loading