Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
CREATE TABLE IF NOT EXISTS pipeline_lifecycle_events (
event_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
pipeline_id uuid NOT NULL,
tenant_id uuid NOT NULL,
deployment_resources_status varchar NOT NULL,
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also: deployment_resources_desired_status

deployment_runtime_status varchar,
deployment_runtime_desired_status varchar,
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

storage_status
program_status

info text,
recorded_at TIMESTAMP NOT NULL DEFAULT now(),
FOREIGN KEY (pipeline_id) REFERENCES pipeline(id) ON DELETE CASCADE,
FOREIGN KEY (tenant_id) REFERENCES tenant(id) ON DELETE CASCADE
);
1 change: 1 addition & 0 deletions crates/pipeline-manager/proptest-regressions/db/test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,4 @@ cc 2e05366358e733acf779daba9549e22bbb573636b4ff01ae589e8040284cdb4a # shrinks to
cc 5902eed1de087fac3012980d700718483060cab6ad99c79204c11cd40753dc6c # shrinks to [NewOrUpdatePipeline(TenantId(03000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "pipeline-0", PipelineDescr { name: "pipeline-0", description: "", runtime_config: RuntimeConfig { workers: 0, storage: false, cpu_profiler: false, tracing: false, tracing_endpoint_jaeger: "", min_batch_size_records: 0, max_buffering_delay_usecs: 0, resources: ResourceConfig { cpu_cores_min: None, cpu_cores_max: None, memory_mb_min: None, memory_mb_max: None, storage_mb_max: None, storage_class: None }, min_storage_bytes: None }, program_code: "", program_config: ProgramConfig { profile: Some(Optimized) } }), NewOrUpdatePipeline(TenantId(03000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "pipeline-0", PipelineDescr { name: "pipeline-0", description: "", runtime_config: RuntimeConfig { workers: 0, storage: false, cpu_profiler: false, tracing: false, tracing_endpoint_jaeger: "", min_batch_size_records: 0, max_buffering_delay_usecs: 0, resources: ResourceConfig { cpu_cores_min: None, cpu_cores_max: None, memory_mb_min: None, memory_mb_max: None, storage_mb_max: None, storage_class: None }, min_storage_bytes: None }, program_code: "a", program_config: ProgramConfig { profile: Some(Optimized) } }), TransitProgramStatusToCompilingSql(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(01000000-0000-0000-0000-000000000000), Version(2)), TransitProgramStatusToCompilingRust(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(01000000-0000-0000-0000-000000000000), Version(2), ProgramInfo { schema: ProgramSchema { inputs: [], outputs: [] }, input_connectors: {}, output_connectors: {} }), TransitProgramStatusToPending(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(01000000-0000-0000-0000-000000000000), Version(2)), ListPipelines(TenantId(03000000-0000-0000-0000-000000000000))]
cc c0ddd7741bf1667b3d70ea81ca4c298875235277256ee70f1fe2b6a9a50303ba # shrinks to [NewPipeline(TenantId(03000000-0000-0000-0000-000000000000), 03000000-0000-0000-0000-000000000000, PipelineDescr { name: "pipeline-0", description: "", runtime_config: RuntimeConfig { workers: 0, storage: false, cpu_profiler: false, tracing: false, tracing_endpoint_jaeger: "", min_batch_size_records: 0, max_buffering_delay_usecs: 0, resources: ResourceConfig { cpu_cores_min: None, cpu_cores_max: None, memory_mb_min: None, memory_mb_max: None, storage_mb_max: None, storage_class: None }, min_storage_bytes: None }, program_code: "", program_config: ProgramConfig { profile: Some(Optimized) } }), TransitProgramStatusToCompilingSql(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), TransitProgramStatusToCompilingRust(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), ProgramInfo { schema: ProgramSchema { inputs: [], outputs: [] }, input_connectors: {}, output_connectors: {} }), TransitProgramStatusToSuccess(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), ""), TransitDeploymentStatusToFailed(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), ErrorResponse { message: "This is an example error response", error_code: "SomeExampleError", details: Object {"extra-info": Number(0)} }), TransitProgramStatusToCompilingSql(TenantId(03000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1))]
cc 126dfeb51d3f8f4707aaf4d12a3d6f90a0ac9fe7f12cffcaa014e8604dfb5f3c # shrinks to [NewPipeline(TenantId(01000000-0000-0000-0000-000000000000), 03000000-0000-0000-0000-000000000000, "v0", PipelineDescr { name: "pipeline-4", description: "", runtime_config: Object {"workers": Number(0), "storage": Null, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(false), "tracing": Bool(false), "tracing_endpoint_jaeger": String(""), "min_batch_size_records": Number(0), "max_buffering_delay_usecs": Number(0), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Null, "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Null, "storage_class": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(false), "dev_tweaks": Object {}}, program_code: "", udf_rust: "", udf_toml: "", program_config: Object {"profile": String("optimized"), "cache": Bool(false)} }), TransitProgramStatusToCompilingSql(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), TransitProgramStatusToSqlCompiled(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), SqlCompilationInfo { exit_code: 0, messages: [] }, Object {"schema": Object {"inputs": Array [], "outputs": Array []}, "main_rust": String("main-rust-0"), "udf_stubs": String("udf-stubs-0"), "dataflow": Null, "input_connectors": Object {}, "output_connectors": Object {}}), TransitProgramStatusToCompilingRust(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1)), TransitProgramStatusToSuccess(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(1), RustCompilationInfo { exit_code: 0, stdout: "stdout-0", stderr: "stderr-0" }, "source_checksum_0", "integrity_checksum_0", ""), NewOrUpdatePipeline(TenantId(01000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "pipeline-4", "v0", PipelineDescr { name: "pipeline-3", description: "\u{11cb6}^ $gn𑍐dÊ'kڦ=~\u{1cf42}>𖭯ℶ🛟q", runtime_config: Object {"workers": Number(9614), "storage": Object {"backend": Object {"name": String("default")}, "min_storage_bytes": Null, "min_step_storage_bytes": Null, "compression": String("default"), "cache_mib": Null}, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(true), "tracing": Bool(false), "tracing_endpoint_jaeger": String("\\&🕴"), "min_batch_size_records": Number(9821004243620302662), "max_buffering_delay_usecs": Number(409483829293609342), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Number(2119326179280987330), "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Number(7151796549280257912), "storage_class": Null}, "clock_resolution_usecs": Number(9929288629303979490), "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(true), "dev_tweaks": Object {}}, program_code: "ெ:ඍ𑎄2𝒪]₄&&𑶨<𑃠/🏿ꬵஓP", udf_rust: "=ঊോ🈕স?T𐀇ὙὛ*lૐ>X\u{1e00b}(ȺਠE🕴=Ѩ﷏;1", udf_toml: ".5oy�%ᏺj𖮎𐨓,", program_config: Object {"profile": Null, "cache": Bool(true)} }), TransitProgramStatusToSystemError(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000), Version(2), "יּ?!\u{113c7}<%\u{1e023},&ꠑÐ𐑠"), GetPipelineById(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(03000000-0000-0000-0000-000000000000))]
cc 02de73819c5ea51c45bc2ce97f22c7aab777e3c55cdcc3c0deeaa87e396d5334 # shrinks to [NewPipeline(TenantId(01000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "v0", PipelineDescr { name: "pipeline-3", description: "", runtime_config: Object {"workers": Number(0), "storage": Null, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(false), "tracing": Bool(false), "tracing_endpoint_jaeger": String(""), "min_batch_size_records": Number(0), "max_buffering_delay_usecs": Number(0), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Null, "memory_mb_min": Null, "memory_mb_max": Null, "storage_mb_max": Null, "storage_class": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(false), "http_workers": Null, "io_workers": Null, "dev_tweaks": Object {}, "logging": Null}, program_code: "", udf_rust: "", udf_toml: "", program_config: Object {"profile": String("optimized"), "cache": Bool(false), "runtime_version": Null} }), TransitDeploymentStatusToStopping(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(01000000-0000-0000-0000-000000000000), Version(1), None, None), TransitDeploymentStatusToStopped(TenantId(01000000-0000-0000-0000-000000000000), PipelineId(01000000-0000-0000-0000-000000000000), Version(1)), UpdatePipeline(TenantId(01000000-0000-0000-0000-000000000000), "", None, None, "v0", Some(Object {"workers": String("abc")}), None, Some("/¥\u{16ff0}🂿ࡠF𐕶//\\ꬎ?:E¥)6🈐﹛Ⱥ𑤉="), None, None), NewPipeline(TenantId(01000000-0000-0000-0000-000000000000), 01000000-0000-0000-0000-000000000000, "v0", PipelineDescr { name: "pipeline-3", description: "{i", runtime_config: Object {"workers": Number(30332), "storage": Object {"backend": Object {"name": String("default")}, "min_storage_bytes": Null, "min_step_storage_bytes": Null, "compression": String("default"), "cache_mib": Null}, "fault_tolerance": Object {"model": String("none"), "checkpoint_interval_secs": Number(60)}, "cpu_profiler": Bool(true), "tracing": Bool(false), "tracing_endpoint_jaeger": String("𑚗ἰѨ&࠵dym,𐑳Y𛲀01ﷇ!୫ᤵⶩ\\𛱶𑍌/ﶟ𐡍"), "min_batch_size_records": Number(15839775634250258066), "max_buffering_delay_usecs": Number(3347010042285470709), "resources": Object {"cpu_cores_min": Null, "cpu_cores_max": Number(9.39201457952098e162), "memory_mb_min": Number(1246125802016676048), "memory_mb_max": Number(14803455586453162047), "storage_mb_max": Null, "storage_class": Null}, "clock_resolution_usecs": Null, "pin_cpus": Array [], "provisioning_timeout_secs": Null, "max_parallel_connector_init": Null, "init_containers": Null, "checkpoint_during_suspend": Bool(true), "http_workers": Null, "io_workers": Number(4522380348662173641), "dev_tweaks": Object {}, "logging": Null}, program_code: "ᏠѨ`z8ਘ\"=𐕲/?ⷙ£៖Ⱥ¸VMU", udf_rust: "P]#>\u{1732}\u{11340}m&꧗\"$𐎲ঌ|𑼴%𑜿?`g=🇸୦꩑ணmp", udf_toml: "𐅙>🕴$𖭮𝒗?c𐠼.e$*𞟠\\Ⱥ𞸧'/mz¥ȺѨᦦ.എ\u{1e08f}?𑓙", program_config: Object {"profile": String("unoptimized"), "cache": Bool(false), "runtime_version": Null} }), GetPipelineLifecycleEvents(TenantId(01000000-0000-0000-0000-000000000000), "pipeline-3", 529798559)]
1 change: 1 addition & 0 deletions crates/pipeline-manager/src/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ pub mod demo;
pub mod endpoints;
pub mod error;
mod examples;
pub mod lifecycle_events;
pub mod main;
pub mod support_data_collector;
pub mod util;
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use feldera_types::query_params::MetricsParameters;
use log::{debug, info};
use std::time::Duration;

pub mod lifecycle_events;
pub mod support_bundle;

/// Push data to a SQL table.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
use actix_web::{
get,
web::{self, Data as WebData, ReqData},
HttpResponse,
};

use crate::{
api::{examples, main::ServerState},
db::{storage::Storage, types::tenant::TenantId},
error::ManagerError,
};

/// Query parameters for lifecycle events endpoint.
#[derive(serde::Deserialize)]
pub struct EventsParameters {
pub max_events: u32,
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Default? I think last 20 makes sense.

}

/// Get lifecycle events for a pipeline.
///
/// Returns a list of lifecycle events for the specified pipeline.
/// The number of events returned is controlled by the `max_events` query parameter.
///
/// # Parameters
/// - `pipeline_name`: Unique pipeline name (path parameter)
/// - `max_events`: Maximum number of events to return (query parameter, required)
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mention default.

///
/// # Returns
/// - 200 OK: List of lifecycle events for the pipeline
/// - 404 NOT_FOUND: Pipeline with that name does not exist
/// - 503 SERVICE_UNAVAILABLE: Disconnected or timeout during response
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this specific 503 can happen, nothing is being HTTP interacted with?

/// - 500 INTERNAL_SERVER_ERROR: Internal error
#[utoipa::path(
context_path = "/v0",
security(("JSON web token (JWT) or API key" = [])),
params(
("pipeline_name" = String, Path, description = "Unique pipeline name"),
("max_events" = u32, Query, description = "Maximum number of events to return")
),
responses(
(status = OK
, description = "List of lifecycle events for the pipeline"
, content_type = "application/json"
, body = Vec<PipelineLifecycleEvent>),
(status = NOT_FOUND
, description = "Pipeline with that name does not exist"
, body = ErrorResponse
, example = json!(examples::error_unknown_pipeline_name())),
(status = SERVICE_UNAVAILABLE
, body = ErrorResponse
, examples(
("Disconnected during response" = (value = json!(examples::error_pipeline_interaction_disconnected()))),
("Response timeout" = (value = json!(examples::error_pipeline_interaction_timeout())))
)
),
(status = INTERNAL_SERVER_ERROR, body = ErrorResponse),
),
tag = "Pipeline interaction"
)]
#[get("/pipelines/{pipeline_name}/lifecycle_events")]
pub(crate) async fn get_pipeline_lifecycle_events(
state: WebData<ServerState>,
tenant_id: ReqData<TenantId>,
path: web::Path<String>,
query: web::Query<EventsParameters>,
) -> Result<HttpResponse, ManagerError> {
let pipeline_name = path.into_inner();
let max_events = query.max_events;

let events = state
.db
.lock()
.await
.get_pipeline_lifecycle_events(*tenant_id, &pipeline_name, max_events)
.await?;

Ok(HttpResponse::Ok()
.content_type("application/json")
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think .json() already sets the content type by itself.

.json(events))
}
15 changes: 15 additions & 0 deletions crates/pipeline-manager/src/api/lifecycle_events.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use uuid::Uuid;

// Pipeline Lifecycle Events
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Description of what it represent?

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, ToSchema)]
pub struct PipelineLifecycleEvent {
pub event_id: Uuid,
pub deployment_resources_status: String,
pub deployment_runtime_status: Option<String>,
pub deployment_runtime_desired_status: Option<String>,
pub info: Option<String>,
pub recorded_at: NaiveDateTime,
}
4 changes: 4 additions & 0 deletions crates/pipeline-manager/src/api/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ It contains the following fields:
endpoints::pipeline_interaction::commit_transaction,
endpoints::pipeline_interaction::get_pipeline_time_series,
endpoints::pipeline_interaction::get_pipeline_time_series_stream,
endpoints::pipeline_interaction::lifecycle_events::get_pipeline_lifecycle_events,

// API keys
endpoints::api_key::list_api_keys,
Expand Down Expand Up @@ -257,6 +258,8 @@ It contains the following fields:
crate::api::endpoints::pipeline_management::PostPutPipeline,
crate::api::endpoints::pipeline_management::PatchPipeline,
crate::api::endpoints::pipeline_management::PostStopPipelineParameters,
// Lifecycle Events
crate::api::lifecycle_events::PipelineLifecycleEvent,

// Storage
crate::db::types::storage::StorageStatus,
Expand Down Expand Up @@ -487,6 +490,7 @@ fn api_scope() -> Scope {
.service(endpoints::pipeline_interaction::completion_status)
.service(endpoints::pipeline_interaction::start_transaction)
.service(endpoints::pipeline_interaction::commit_transaction)
.service(endpoints::pipeline_interaction::lifecycle_events::get_pipeline_lifecycle_events)
// API keys endpoints
.service(endpoints::api_key::list_api_keys)
.service(endpoints::api_key::get_api_key)
Expand Down
2 changes: 2 additions & 0 deletions crates/pipeline-manager/src/auth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,8 @@ mod test {
enable_https: false,
https_tls_cert_path: None,
https_tls_key_path: None,
lifecycle_events_retention_days: 0,
lifecycle_events_cleanup_frequency_secs: 1,
};

let manager_config = ApiServerConfig {
Expand Down
2 changes: 2 additions & 0 deletions crates/pipeline-manager/src/compiler/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ impl CompilerTest {
enable_https: false,
https_tls_cert_path: None,
https_tls_key_path: None,
lifecycle_events_retention_days: 0,
lifecycle_events_cleanup_frequency_secs: 1,
};
let compiler_config = CompilerConfig {
sql_compiler_path:
Expand Down
10 changes: 10 additions & 0 deletions crates/pipeline-manager/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,14 @@ pub struct CommonConfig {
/// `/path/to/tls.key`).
#[arg(long)]
pub https_tls_key_path: Option<String>,

/// Retention period for pipeline lifecycle events (in days).
#[arg(long, default_value_t = 7)]
pub lifecycle_events_retention_days: u16,

/// Frequency at which pipeline lifecycle events are cleaned up (in seconds).
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Frequency (in seconds) at which pipeline lifecycle events that have surpassed the retention period are cleaned up.

#[arg(long, default_value_t = 60 * 60)]
pub lifecycle_events_cleanup_frequency_secs: u64,
}

impl CommonConfig {
Expand Down Expand Up @@ -398,6 +406,8 @@ impl CommonConfig {
enable_https: false,
https_tls_cert_path: None,
https_tls_key_path: None,
lifecycle_events_retention_days: 7,
lifecycle_events_cleanup_frequency_secs: 60 * 60,
}
}
}
Expand Down
Loading