use crate::catalog::OutputCollectionHandles;

use crate::controller::checkpoint::{

CheckpointInputEndpointMetrics, CheckpointOffsets, CheckpointOutputEndpointMetrics,

use crate::controller::journal::Journal;

use crate::controller::stats::{InputEndpointMetrics, OutputEndpointMetrics, ProcessedRecords};

use crate::controller::sync::{

CHECKPOINT_SYNC_PULL_DURATION_SECONDS, CHECKPOINT_SYNC_PULL_FAILURES,

CHECKPOINT_SYNC_PULL_SUCCESS, CHECKPOINT_SYNC_PULL_TRANSFER_SPEED,

CHECKPOINT_SYNC_PULL_TRANSFERRED_BYTES, CHECKPOINT_SYNC_PUSH_DURATION_SECONDS,

CHECKPOINT_SYNC_PUSH_FAILURES, CHECKPOINT_SYNC_PUSH_SUCCESS,

CHECKPOINT_SYNC_PUSH_TRANSFER_SPEED, CHECKPOINT_SYNC_PUSH_TRANSFERRED_BYTES, SYNCHRONIZER,

use crate::samply::SamplySpan;

use crate::server::metrics::{HistogramDiv, LabelStack, MetricsFormatter, MetricsWriter, Value};

use crate::server::{InitializationState, ServerState};

use crate::transport::Step;

use crate::transport::clock::now_endpoint_config;

use crate::transport::{input_transport_config_to_endpoint, output_transport_config_to_endpoint};

use crate::util::{LongOperationWarning, run_on_thread_pool};

use crate::{

CircuitCatalog, Encoder, InputConsumer, OutputConsumer, OutputEndpoint, ParseError,

PipelineState, TransportInputEndpoint,

use crate::{PipelinePhase, create_integrated_output_endpoint};

use anyhow::{Context, Error as AnyError, anyhow};

use arrow::datatypes::Schema;

use arrow::util::pretty::pretty_format_batches;

use atomic::Atomic;

use checkpoint::Checkpoint;

use chrono::{DateTime, Utc};

use crossbeam::{

queue::SegQueue,

sync::{Parker, ShardedLock, Unparker},

use datafusion::prelude::*;

use dbsp::circuit::circuit_builder::BootstrapInfo;

use dbsp::circuit::metrics::{

COMPACTION_STALL_TIME_NANOSECONDS, DBSP_OPERATOR_COMMIT_LATENCY_MICROSECONDS, DBSP_STEP,

DBSP_STEP_LATENCY_MICROSECONDS, FILES_CREATED, FILES_DELETED, TOTAL_LATE_RECORDS,

use dbsp::circuit::tokio::TOKIO;

use dbsp::circuit::{CheckpointCommitter, CircuitStorageConfig, DevTweaks, Mode};

use dbsp::storage::backend::{StorageBackend, StoragePath};

use dbsp::{

DBSPHandle,

circuit::{CircuitConfig, Layout},

profile::{DbspProfile, GraphProfile},

use dbsp::{Runtime, WeakRuntime};

use enum_map::EnumMap;

use feldera_adapterlib::format::BufferSize;

use feldera_adapterlib::metrics::{ConnectorMetrics, ValueType};

use feldera_adapterlib::transport::{Resume, Watermark};

use feldera_ir::LirCircuit;

use feldera_storage::histogram::{ExponentialHistogram, ExponentialHistogramSnapshot};

use feldera_storage::metrics::{

READ_BLOCKS_BYTES, READ_LATENCY_MICROSECONDS, SYNC_LATENCY_MICROSECONDS, WRITE_BLOCKS_BYTES,

WRITE_LATENCY_MICROSECONDS,

use feldera_types::adapter_stats::{

ConnectorHealth, ExternalControllerStatus, ExternalInputEndpointStatus,

ExternalOutputEndpointStatus,

use feldera_types::checkpoint::CheckpointMetadata;

use feldera_types::coordination::{

self, AdHocCatalog, AdHocTableType, CheckpointCoordination, Completion, StepAction, StepInputs,

StepRequest, StepStatus, TransactionCoordination,

use feldera_types::format::json::JsonLines;

use feldera_types::pipeline_diff::PipelineDiff;

use feldera_types::runtime_status::BootstrapPolicy;

use feldera_types::secret_resolver::resolve_secret_references_in_connector_config;

use feldera_types::suspend::{PermanentSuspendError, SuspendError, TemporarySuspendError};

use feldera_types::time_series::SampleStatistics;

use feldera_types::transaction::{StartTransactionResponse, TransactionId};

use governor::DefaultDirectRateLimiter;

use governor::Quota;

use governor::RateLimiter;

use itertools::Itertools;

use journal::StepMetadata;

use memory_stats::memory_stats;

use nonzero_ext::nonzero;

use rmpv::Value as RmpValue;

use serde_json::Value as JsonValue;

use stats::StepResults;

use std::borrow::Cow;

use std::collections::HashMap;

use std::collections::HashSet;

use std::collections::btree_map::Entry;

use std::io::ErrorKind;

use std::mem::replace;

use std::ops::Range;

use std::panic::{AssertUnwindSafe, catch_unwind};

use std::path::PathBuf;

use std::sync::mpsc::{Receiver, SendError, Sender, SyncSender, channel, sync_channel};

use std::sync::{LazyLock, Mutex, Weak};

use std::thread::{self, sleep};

use std::{

collections::{BTreeMap, BTreeSet},

mem,

sync::{

Arc,

atomic::{AtomicBool, AtomicU64, Ordering},

},

thread::JoinHandle,

time::{Duration, Instant},

use tokio::sync::Notify;

use tokio::{

fs::File,

io::{AsyncReadExt, BufReader},

sync::{

Mutex as TokioMutex,

oneshot::{self, error::TryRecvError},

},

task::spawn_blocking,

use tracing::{debug, debug_span, error, info, trace, warn};

use uuid::Uuid;

use validate::validate_config;

mod checkpoint;

mod error;

mod journal;

mod pipeline_diff;

mod stats;

mod sync;

mod validate;

use crate::adhoc::table::AdHocTable;

use crate::adhoc::{create_session_context, execute_sql};

use crate::catalog::{SerBatchReader, SerTrace};

use crate::format::parquet::relation_to_arrow_fields;

use crate::format::{MessageOrientedPreprocessedParser, StreamingPreprocessedParser};

use crate::format::{get_input_format, get_output_format};

use crate::integrated::create_integrated_input_endpoint;

pub use error::{ConfigError, ControllerError};

pub use feldera_types::config::{

ConnectorConfig, FormatConfig, InputEndpointConfig, OutputEndpointConfig, PipelineConfig,

RuntimeConfig, TransportConfig,

use feldera_types::config::{

DEFAULT_MAX_WORKER_BATCH_SIZE, FileBackendConfig, FtConfig, FtModel, OutputBufferConfig,

StorageBackendConfig, SyncConfig,

use feldera_types::constants::{STATE_FILE, STEPS_FILE};

use feldera_types::format::json::{JsonFlavor, JsonParserConfig, JsonUpdateFormat};

use feldera_types::program_schema::{SqlIdentifier, canonical_identifier};

pub use pipeline_diff::compute_pipeline_diff;

pub use stats::{CompletionToken, ControllerStatus, InputEndpointStatus};

pub(crate) const MAX_API_CONNECTIONS: u64 = 100;

pub type EndpointId = u64;

static CHECKPOINT_RUNTIME: ExponentialHistogram = ExponentialHistogram::new();

static CHECKPOINT_DELAY: ExponentialHistogram = ExponentialHistogram::new();

static CHECKPOINT_WRITTEN_MEGABYTES: ExponentialHistogram = ExponentialHistogram::new();

static TRANSACTION_INGEST_TIME: ExponentialHistogram = ExponentialHistogram::new();

static TRANSACTION_COMMIT_TIME: ExponentialHistogram = ExponentialHistogram::new();

static CHECKPOINT_PROCESSED_RECORDS: AtomicU64 = AtomicU64::new(0);

static COMMIT_UPDATE_INTERVAL: Duration = Duration::from_secs(1);

static COMMIT_DISPLAY_INTERVAL: Duration = Duration::from_secs(10);

pub struct ControllerBuilder {

impl ControllerBuilder {

pub struct Controller {

pub type GraphProfileCallbackFn = Box<dyn FnOnce(Result<GraphProfile, ControllerError>) + Send>;

pub type JsonProfileCallbackFn = Box<dyn FnOnce(Result<DbspProfile, ControllerError>) + Send>;

pub type CheckpointCallbackFn = Box<dyn FnOnce(Result<Checkpoint, Arc<ControllerError>>) + Send>;

pub type SuspendCallbackFn = Box<dyn FnOnce(Result<(), Arc<ControllerError>>) + Send>;

pub type SyncCheckpointCallbackFn = Box<dyn FnOnce(Result<(), Arc<ControllerError>>) + Send>;

pub type RebalanceCallbackFn = Box<dyn FnOnce(Result<(), ControllerError>) + Send>;

enum Command {

impl Command {

impl Controller {