Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
360 changes: 216 additions & 144 deletions crates/adapters/src/controller.rs

Large diffs are not rendered by default.

78 changes: 40 additions & 38 deletions crates/adapters/src/controller/stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use super::{EndpointId, InputEndpointConfig, OutputEndpointConfig};
use crate::{
PipelineState,
controller::{
TransactionInitiators,
TransactionInfo, TransactionState,
checkpoint::{CheckpointInputEndpointMetrics, CheckpointOutputEndpointMetrics},
journal::{InputChecksums, InputLog},
},
Expand All @@ -57,13 +57,13 @@ use feldera_types::{
adapter_stats::{
CompletedWatermark, ConnectorError, ConnectorHealth, ExternalInputEndpointMetrics,
ExternalInputEndpointStatus, ExternalOutputEndpointMetrics, ExternalOutputEndpointStatus,
ShortEndpointConfig, TransactionStatus,
ShortEndpointConfig,
},
config::{FtModel, PipelineConfig},
coordination::Completion,
suspend::SuspendError,
time_series::SampleStatistics,
transaction::{CommitProgressSummary, TransactionId},
transaction::CommitProgressSummary,
};
use memory_stats::memory_stats;
use parking_lot::{RwLock, RwLockReadGuard};
Expand Down Expand Up @@ -159,15 +159,6 @@ pub struct GlobalControllerMetrics {
/// new and modified views.
bootstrap_in_progress: AtomicBool,

/// Status of the current transaction.
pub transaction_status: Atomic<TransactionStatus>,

/// ID of the current transaction or 0 if no transaction is in progress.
pub transaction_id: Atomic<TransactionId>,

/// Entities that initiated the current transaction.
pub transaction_initiators: Mutex<TransactionInitiators>,

/// Transaction commit progress, if a transaction is committing.
pub commit_progress: Mutex<Option<CommitProgressSummary>>,

Expand Down Expand Up @@ -291,9 +282,6 @@ impl GlobalControllerMetrics {
Self {
state: Atomic::new(PipelineState::Paused),
bootstrap_in_progress: AtomicBool::new(false),
transaction_id: Atomic::new(0),
transaction_status: Atomic::new(TransactionStatus::NoTransaction),
transaction_initiators: Mutex::new(TransactionInitiators::default()),
commit_progress: Mutex::new(None),
start_time,
incarnation_uuid,
Expand Down Expand Up @@ -709,13 +697,6 @@ impl ControllerStatus {
.set_bootstrap_in_progress(bootstrap_in_progress);
}

pub fn transaction_in_progress(&self) -> bool {
self.global_metrics
.transaction_status
.load(Ordering::Acquire)
!= TransactionStatus::NoTransaction
}

pub fn request_step(&self, circuit_thread_unparker: &Unparker) {
let old = self.global_metrics.set_step_requested();
if !old {
Expand Down Expand Up @@ -1189,10 +1170,17 @@ impl ControllerStatus {
&self,
suspend_error: Result<(), SuspendError>,
pipeline_complete: bool,
transaction_info: TransactionInfo,
) -> feldera_types::adapter_stats::ExternalControllerStatus {
use feldera_types::adapter_stats;

// Convert global metrics
let total_processed_records = self
.global_metrics
.total_processed_records
.load(Ordering::Acquire);

#[allow(clippy::manual_unwrap_or_default)]
#[allow(clippy::manual_unwrap_or)]
let global_metrics = adapter_stats::ExternalGlobalControllerMetrics {
state: match self.global_metrics.state.load(Ordering::Acquire) {
PipelineState::Paused => adapter_stats::PipelineState::Paused,
Expand All @@ -1203,18 +1191,35 @@ impl ControllerStatus {
.global_metrics
.bootstrap_in_progress
.load(Ordering::Acquire),
transaction_status: self
.global_metrics
.transaction_status
.load(Ordering::Acquire),
transaction_id: self.global_metrics.transaction_id.load(Ordering::Acquire),
transaction_status: match transaction_info.transaction_state {
TransactionState::None => adapter_stats::TransactionStatus::NoTransaction,
TransactionState::Started { .. } => {
adapter_stats::TransactionStatus::TransactionInProgress
}
TransactionState::Committing { .. } => {
adapter_stats::TransactionStatus::CommitInProgress
}
},
transaction_msecs: transaction_info
.transaction_state
.start_time()
.and_then(|start| start.elapsed().as_millis().try_into().ok()),
transaction_records: transaction_info
.transaction_state
.processed_records()
.map(|n| total_processed_records - n),
transaction_id: if let Some(tid) = transaction_info.transaction_state.tid() {
// The current transaction ID.
tid
} else if let Some(tid) = transaction_info.initiators.transaction_id {
// The transaction that will start when we execute a step.
tid
} else {
// No transaction
0
},
commit_progress: self.global_metrics.commit_progress.lock().unwrap().clone(),
transaction_initiators: self
.global_metrics
.transaction_initiators
.lock()
.unwrap()
.to_api_type(),
transaction_initiators: transaction_info.initiators.to_api_type(),
start_time: self.global_metrics.start_time,
incarnation_uuid: self.global_metrics.incarnation_uuid,
initial_start_time: self.global_metrics.initial_start_time,
Expand Down Expand Up @@ -1257,10 +1262,7 @@ impl ControllerStatus {
.global_metrics
.total_input_bytes
.load(Ordering::Acquire),
total_processed_records: self
.global_metrics
.total_processed_records
.load(Ordering::Acquire),
total_processed_records,
total_processed_bytes: self
.global_metrics
.total_processed_bytes
Expand Down
2 changes: 2 additions & 0 deletions crates/adapters/src/controller/test.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::{
Controller, PipelineConfig,
controller::TransactionInfo,
preprocess::{DecryptionPreprocessorFactory, PassthroughPreprocessorFactory},
test::{
DEFAULT_TIMEOUT_MS, TestStruct, generate_test_batch, init_test_logger, test_circuit, wait,
Expand Down Expand Up @@ -2303,6 +2304,7 @@ fn test_external_controller_status_serialization() {
PermanentSuspendError::UnsupportedInputEndpoint("kafka_input".to_string()),
])),
false,
TransactionInfo::default(),
);
external_status.global_metrics.rss_bytes = 1024 * 1024 * 512; // 512 MB
external_status.global_metrics.cpu_msecs = 45_000;
Expand Down
9 changes: 7 additions & 2 deletions crates/adapters/src/transport/nats/input/test.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::controller::TransactionInfo;
use crate::test::{
DEFAULT_TIMEOUT_MS, TestStruct, init_test_logger, mock_input_pipeline, test_circuit, wait,
};
Expand Down Expand Up @@ -340,8 +341,12 @@ outputs:
if let Err(()) = result {
println!(
"Controller status:\n{}",
serde_json::to_string_pretty(&controller.status().to_api_type(Ok(()), false))
.unwrap()
serde_json::to_string_pretty(&controller.status().to_api_type(
Ok(()),
false,
TransactionInfo::default(),
))
.unwrap()
);
panic!("Failed to receive expected records within timeout");
}
Expand Down
16 changes: 16 additions & 0 deletions crates/feldera-types/src/adapter_stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,22 @@ pub struct ExternalGlobalControllerMetrics {
/// ID of the current transaction or 0 if no transaction is in progress.
#[schema(value_type = i64)]
pub transaction_id: TransactionId,
/// Elapsed time in milliseconds, according to `transaction_status`:
///
/// - [TransactionStatus::TransactionInProgress]: Time that this transaction
/// has been in progress.
///
/// - [TransactionStatus::CommitInProgress]: Time that this transaction has
/// been committing.
pub transaction_msecs: Option<u64>,
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These new fields will need to be added to the Python SDK.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

/// Number of records in this transaction, according to
/// `transaction_status`:
///
/// - [TransactionStatus::TransactionInProgress]: Number of records added so
/// far. More records might be added.
///
/// - [TransactionStatus::CommitInProgress]: Final number of records.
pub transaction_records: Option<u64>,
/// Progress of the current transaction commit, if one is in progress.
pub commit_progress: Option<CommitProgressSummary>,
/// Entities that initiated the current transaction.
Expand Down
7 changes: 6 additions & 1 deletion crates/storage/src/histogram.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,12 @@ impl ExponentialHistogram {
/// Records the time elapsed since `start` in the histogram, as a count of
/// microseconds.
pub fn record_elapsed(&self, start: Instant) {
self.record(start.elapsed().as_micros());
self.record_duration(start.elapsed());
}

/// Records `duration` as a count of microseconds.
pub fn record_duration(&self, duration: Duration) {
self.record(duration.as_micros());
}

/// Returns a snapshot of the histogram.
Expand Down
2 changes: 2 additions & 0 deletions docs.feldera.com/docs/operations/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,9 @@ These metrics report the status of [transactions].

| Name | Type | Description |
| :--- | :--- | :---------- |
| <a name='transaction_commit_seconds'>`transaction_commit_seconds`</a> |histogram | Transaction commit time, that is, from starting commit to finishing commit. |
| <a name='transaction_completed_operators'>`transaction_completed_operators`</a> |gauge | Number of operators that have been fully flushed while the current transaction is committing. This is 0 if no transaction is active, or if a transaction is running but has not yet started committing. |
| <a name='transaction_in_progress_operators'>`transaction_in_progress_operators`</a> |gauge | Number of operators that are currently being flushed while the current transaction is committing. This is 0 if no transaction is active, or if a transaction is running but has not yet started committing. |
| <a name='transaction_ingest_seconds'>`transaction_ingest_seconds`</a> |histogram | Transaction ingestion time, that is, from transaction start to start of commit. |
| <a name='transaction_remaining_operators'>`transaction_remaining_operators`</a> |gauge | Number of operators that have not started flushing while the current transaction is committing. This is 0 if no transaction is active, or if a transaction is running but has not yet started committing. |
| <a name='transaction_state'>`transaction_state`</a> |gauge | 0 when no transaction is active, 1 when a transaction has started, 2 while a transaction is committing. |
14 changes: 14 additions & 0 deletions openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -8485,6 +8485,20 @@
"transaction_initiators": {
"$ref": "#/components/schemas/TransactionInitiators"
},
"transaction_msecs": {
"type": "integer",
"format": "int64",
"description": "Elapsed time in milliseconds, according to `transaction_status`:\n\n- [TransactionStatus::TransactionInProgress]: Time that this transaction\nhas been in progress.\n\n- [TransactionStatus::CommitInProgress]: Time that this transaction has\nbeen committing.",
"nullable": true,
"minimum": 0
},
"transaction_records": {
"type": "integer",
"format": "int64",
"description": "Number of records in this transaction, according to\n`transaction_status`:\n\n- [TransactionStatus::TransactionInProgress]: Number of records added so\nfar. More records might be added.\n\n- [TransactionStatus::CommitInProgress]: Final number of records.",
"nullable": true,
"minimum": 0
},
"transaction_status": {
"$ref": "#/components/schemas/TransactionStatus"
},
Expand Down
2 changes: 2 additions & 0 deletions python/feldera/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ def __init__(self):
self.bootstrap_in_progress: Optional[bool] = None
self.transaction_status: Optional[TransactionStatus] = None
self.transaction_id: Optional[int] = None
self.transaction_msecs: Optional[int] = None
self.transaction_records: Optional[int] = None
self.commit_progress: Optional[CommitProgressSummary] = None
self.transaction_initiators: Optional[TransactionInitiators] = None
self.rss_bytes: Optional[int] = None
Expand Down
Loading