use crate::{

Controller, PipelineConfig,

controller::TransactionInfo,

preprocess::{DecryptionPreprocessorFactory, PassthroughPreprocessorFactory},

test::{

DEFAULT_TIMEOUT_MS, TestStruct, generate_test_batch, init_test_logger, test_circuit, wait,

},

transport::set_barrier,

use anyhow::anyhow;

use csv::{ReaderBuilder as CsvReaderBuilder, WriterBuilder as CsvWriterBuilder};

use feldera_types::{

config::{InputEndpointConfig, OutputEndpointConfig},

constants::STATE_FILE,

memory_pressure::MemoryPressure,

use serde_json::json;

use std::{

borrow::Cow,

cmp::min,

collections::BTreeMap,

fs::{File, create_dir, remove_file},

io::Write,

iter::repeat_n,

ops::Range,

path::{Path, PathBuf},

sync::{atomic::Ordering, mpsc},

thread::sleep,

time::Duration,

use tempfile::{NamedTempFile, TempDir};

use tokio::sync::oneshot;

use tracing::info;

use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};

use proptest::prelude::*;

fn test_start_after_cyclic() {

fn test_start_after() {

proptest! {

#![proptest_config(ProptestConfig::with_cases(30))]

#[test]

fn proptest_csv_file(

data in generate_test_batch(5000),

min_batch_size_records in 1..100usize,

max_buffering_delay_usecs in 1..2000usize,

input_buffer_size_bytes in 1..1000usize,

output_buffer_size_records in 1..100usize)

{

let temp_input_file = NamedTempFile::new().unwrap();

let temp_output_path = NamedTempFile::new().unwrap().into_temp_path();

let output_path = temp_output_path.to_str().unwrap().to_string();

temp_output_path.close().unwrap();

let secrets_dir = TempDir::new().unwrap();

create_dir(secrets_dir.path().join("kubernetes")).unwrap();

create_dir(secrets_dir.path().join("kubernetes/paths")).unwrap();

std::fs::write(secrets_dir.path().join("kubernetes/paths/input"), temp_input_file.path().as_os_str().as_encoded_bytes()).unwrap();

std::fs::write(secrets_dir.path().join("kubernetes/paths/output"), &output_path).unwrap();

let config: PipelineConfig = serde_json::from_value(json!({

"secrets_dir": secrets_dir.path(),

"min_batch_size_records": min_batch_size_records,

"max_buffering_delay_usecs": max_buffering_delay_usecs,

"name": "test",

"workers": 4,

"inputs": {

"test_input1": {

"stream": "test_input1",

"transport": {

"name": "file_input",

"config": {

"path": "${secret:kubernetes:paths/input}",

"buffer_size_bytes": input_buffer_size_bytes,

"follow": false

}

},

"format": {

"name": "csv"

}

}

},

"outputs": {

"test_output1": {

"stream": "test_output1",

"transport": {

"name": "file_output",

"config": {

"path": "${secret:kubernetes:paths/output}"

}

},

"format": {

"name": "csv",

"config": {

"buffer_size_records": output_buffer_size_records,

}

}

}

}

})).unwrap();

info!("input file: {}", temp_input_file.path().display());

info!("output file: {output_path}");

let controller = Controller::with_test_config(

|circuit_config| Ok(test_circuit::<TestStruct>(circuit_config, &[], &[None])),

&config,

Box::new(|e, _| panic!("error: {e}")),

)

.unwrap();

let mut writer = CsvWriterBuilder::new()

.has_headers(false)

.from_writer(temp_input_file.as_file());

for val in data.iter().cloned() {

writer.serialize(val).unwrap();

}

writer.flush().unwrap();

println!("wait for {} records", data.len());

controller.start();

// Wait for the pipeline to output all records.

wait(|| controller.pipeline_complete(), DEFAULT_TIMEOUT_MS).unwrap();

assert_eq!(controller.status().output_status().get(&0).unwrap().transmitted_records(), data.len() as u64);

controller.stop().unwrap();

let mut expected = data;

expected.sort();

let mut actual: Vec<_> = CsvReaderBuilder::new()

.has_headers(false)

.from_path(&output_path)

.unwrap()

.deserialize::<(TestStruct, i32)>()

.map(|res| {

let (val, weight) = res.unwrap();

assert_eq!(weight, 1);

val

})

.collect();

actual.sort();

// Don't leave garbage in the FS.

remove_file(&output_path).unwrap();

assert_eq!(actual, expected);

}

struct FtTestRound {

impl FtTestRound {

fn check_file_contents(path: &Path, expect: Range<usize>) {

fn count_endpoint_records(controller: &Controller, output_index: usize) -> u64 {

fn collect_endpoint_records(controller: &Controller, n: usize) -> Vec<usize> {

fn wait_for_records(controller: &Controller, expect_n: &[usize]) {

fn test_ft(rounds: &[FtTestRound]) {

fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {

fn multiple_input_files(

fn _test_concurrent_init(max_parallel_connector_init: u64) {

fn test_concurrent_init() {

fn test_connector_init_error() {

fn ft_with_checkpoints() {

fn ft_immediate_checkpoints() {

fn ft_without_checkpoints() {

fn ft_with_checkpoints_with_pauses() {

fn ft_without_checkpoints_with_pauses() {

fn ft_alternating() {