/// Unique name of the data format.

fn name(&self) -> Cow<'static, str>;

/// Extract parser configuration from an HTTP request.

///

/// Returns the extracted configuration cast to the `ErasedSerialize` trait

/// object (to keep this trait object-safe).

///

/// # Discussion

///

/// We could rely on the `serde_urlencoded` crate to deserialize the config

/// from the HTTP request, which is what most implementations will do

/// internally; however allowing the implementation to override this

/// method enables additional flexibility. For example, an

/// implementation may use `Content-Type` and other request headers, set

/// HTTP-specific defaults for config fields, etc.

fn config_from_http_request(

&self,

endpoint_name: &str,

request: &HttpRequest,

) -> Result<Box<dyn ErasedSerialize>, ControllerError>;

/// Create a new parser for the format.

///

/// # Arguments

///

/// * `input_stream` - Input stream of the circuit to push parsed data to.

///

/// * `config` - Format-specific configuration.

fn new_parser(

&self,

endpoint_name: &str,

input_stream: &InputCollectionHandle,

config: &serde_json::Value,

) -> Result<Box<dyn Parser>, ControllerError>;

/// Pushes all of the records into the circuit input handle, and discards

/// those records.

fn flush(&mut self);

/// Returns the number of buffered records and bytes.

fn len(&self) -> BufferSize;

/// Hashes the records in the input buffer into `hasher`, in order. This is

/// used to ensure that input data remains the same for replay, so, for

/// equal data, it should remain the same from one program run to the next.

/// Data might be divided into `InputBuffer`s differently from one run to

/// the next, so the data fed into `hasher` should be the same if, for

/// example, records 0..10 then 10..20 are fed in one run and 0..5, 5..15,

/// 15..20 in another.

fn hash(&self, hasher: &mut dyn Hasher);

fn is_empty(&self) -> bool {

self.len().is_empty()

}

/// Removes the first `n` records from this input buffer and returns a new

/// [InputBuffer] that holds them. If fewer than `n` records are available,

/// returns all of them. May return `None` if this input buffer is empty (or

/// if `n` is 0).

///

/// Some implementations can't implement `n` with full granularity. These

/// implementations might return more than `n` records.

///

/// This is useful for extracting the records from one of several parser

/// threads to send to a single common thread to be pushed later.

///

/// # Byte accounting

///

/// This function must not increase or decrease the total number of bytes.

/// That is, if the returned buffer is named `head`, `self.len().bytes`

/// before the call must equal `self.len().bytes + head.len().bytes`

/// following the call. Violating this invariant will cause the number of

/// buffered bytes reported by a pipeline never to fall to zero (or to wrap

/// around to `u64::MAX`).

fn take_some(&mut self, n: usize) -> Option<Box<dyn InputBuffer>>;

fn take_all(&mut self) -> Option<Box<dyn InputBuffer>> {

self.take_some(usize::MAX)

}

/// The exact number of records in the buffer.

pub records: usize,

/// The number of bytes attributable to the buffer.

///

/// This need not be exact. (When a buffer is split with

/// [InputBuffer::take_some], it will usually not be exact.)

pub bytes: usize,

/// The size of an empty buffer.

pub fn empty() -> Self {

Self::default()

}

/// Returns true if this buffer is empty.

pub fn is_empty(&self) -> bool {

self.records == 0 && self.bytes == 0

}

type Output = Self;

fn add(self, rhs: Self) -> Self::Output {

BufferSize {

records: self.records + rhs.records,

bytes: self.bytes + rhs.bytes,

}

}

fn add_assign(&mut self, rhs: Self) {

self.records += rhs.records;

self.bytes += rhs.bytes;

}

fn len(&self) -> BufferSize {

self.as_ref()

.map_or(BufferSize::empty(), |buffer| buffer.len())

}

fn hash(&self, hasher: &mut dyn Hasher) {

if let Some(buffer) = self {

buffer.hash(hasher)

}

}

fn flush(&mut self) {

if let Some(buffer) = self.as_mut() {

buffer.flush()

}

}

fn take_some(&mut self, n: usize) -> Option<Box<dyn InputBuffer>> {

self.as_mut().and_then(|buffer| buffer.take_some(n))

}

fn len(&self) -> BufferSize {

self.as_ref().len()

}

fn hash(&self, hasher: &mut dyn Hasher) {

self.as_ref().hash(hasher)

}

fn flush(&mut self) {

self.as_mut().flush()

}

fn take_some(&mut self, n: usize) -> Option<Box<dyn InputBuffer>> {

self.as_mut().take_some(n)

}

fn flush(&mut self) {

for v in self.iter_mut() {

v.flush();

}

}

fn len(&self) -> BufferSize {

let mut size = BufferSize::empty();

for v in self.iter() {

size += v.len();

}

size

}

fn hash(&self, hasher: &mut dyn Hasher) {

for v in self.iter() {

v.hash(hasher);

}

}

fn take_some(&mut self, n: usize) -> Option<Box<dyn InputBuffer>> {

let mut result = Vec::new();

let mut remaining = n;

// Index of first buffer that should be preserved

let mut index = 0;

for v in self.iter_mut() {

if remaining == 0 {

break;

}

let buf = v.take_some(remaining);

if let Some(ib) = buf {

let len = ib.len().records;

if remaining >= len {

// This buffer will be completely used

index += 1;

}

remaining = remaining.saturating_sub(len);

result.push(ib);

}

}

self.drain(0..index);

if result.is_empty() {

None

} else {

Some(Box::new(result))

}

}

T: Any,

fn inner<T>(input: Vec<Box<dyn InputBuffer>>, output: &mut Vec<Box<T>>)

where

T: Any,

{

for buffer in input {

let any = buffer as Box<dyn Any>;

match any.downcast::<Vec<Box<dyn InputBuffer>>>() {

Ok(vec) => inner(*vec, output),

Err(any) => output.push(any.downcast().unwrap()),

}

}

}

let mut output = Vec::new();

inner(buffers, &mut output);

output

pub fn new(buffer: Box<dyn StagedBuffers>, size: BufferSize) -> Self {

Self { buffer, size }

}

fn flush(&mut self) {

self.buffer.flush()

}

fn len(&self) -> BufferSize {

self.size

}

fn hash(&self, _hasher: &mut dyn Hasher) {

unimplemented!()

}

fn take_some(&mut self, _n: usize) -> Option<Box<dyn InputBuffer>> {

unimplemented!()

}

/// Parses `data` into records and returns the records and any parse errors

/// that occurred.

///

/// XXX it would be even better if this were `&self` and avoided keeping

/// state entirely.

fn parse(

&mut self,

data: &[u8],

metadata: Option<ConnectorMetadata>,

) -> (Option<Box<dyn InputBuffer>>, Vec<ParseError>);

/// Stages all of the `buffers`, which must have been obtained from this

/// [Parser] or one forked from this one, into a [StagedBuffers] that may

/// later be used to push the collected data into the circuit. See

/// [StagedBuffers] for more information.

fn stage(&self, buffers: Vec<Box<dyn InputBuffer>>) -> Box<dyn StagedBuffers>;

/// Returns an object that can be used to break a stream of incoming data

/// into complete records to pass to [Parser::parse].

fn splitter(&self) -> Box<dyn Splitter>;

/// Create a new parser with the same configuration as `self`.

///

/// Used by multithreaded transport endpoints to create multiple parallel

/// input pipelines.

fn fork(&self) -> Box<dyn Parser>;

preprocessor: Box<dyn Preprocessor>,

stream_splitter: StreamSplitter,

parser: Box<dyn Parser>,

pub fn new(preprocessor: Box<dyn Preprocessor>, parser: Box<dyn Parser>) -> Self {

Self {

preprocessor,

stream_splitter: StreamSplitter::new(parser.splitter()),

parser,

}

}

fn parse(

&mut self,

data: &[u8],

metadata: Option<ConnectorMetadata>,

) -> (Option<Box<dyn InputBuffer>>, Vec<ParseError>) {

let (pre_data, mut pre_errors) = self.preprocessor.process(data);

self.stream_splitter.append(&pre_data);

let mut parsed: Vec<Box<dyn InputBuffer>> = Vec::new();

while let Some(chunk) = self.stream_splitter.next(true) {

let (parsed_data, mut parse_errors) = self.parser.parse(chunk, metadata.clone());

pre_errors.append(&mut parse_errors);

if let Some(data) = parsed_data {

parsed.push(data);

}

}

if parsed.is_empty() {

(None, pre_errors)

} else {

(Some(Box::new(parsed)), pre_errors)

}

}

fn stage(&self, buffers: Vec<Box<dyn InputBuffer>>) -> Box<dyn StagedBuffers> {

self.parser.stage(buffers)

}

fn splitter(&self) -> Box<dyn Splitter> {

let pre_splitter = self.preprocessor.splitter();

if let Some(splitter) = pre_splitter {

return splitter;

}

self.parser.splitter()

}

fn fork(&self) -> Box<dyn Parser> {

Box::new(StreamingPreprocessedParser::new(

self.preprocessor.fork(),

self.parser.fork(),

))

}

pub fn new(preprocessor: Box<dyn Preprocessor>, parser: Box<dyn Parser>) -> Self {

Self {

preprocessor,

parser,

}

}

fn parse(

&mut self,

data: &[u8],

metadata: Option<ConnectorMetadata>,

) -> (Option<Box<dyn InputBuffer>>, Vec<ParseError>) {

let (pre_data, mut pre_errors) = self.preprocessor.process(data);

let mut parser_splitter = self.parser.splitter();

let mut parsed: Vec<Box<dyn InputBuffer>> = Vec::new();

let mut remaining = pre_data.as_slice();

// Use the parser to divide the message received from the preprocessor into chunks and parse each of them

while !remaining.is_empty() {

let chunk;

let split_offset = parser_splitter.input(remaining).unwrap_or(remaining.len());

(chunk, remaining) = remaining.split_at(split_offset);

let (parsed_data, mut parse_errors) = self.parser.parse(chunk, metadata.clone());

pre_errors.append(&mut parse_errors);

if let Some(data) = parsed_data {

parsed.push(data);

}

}

if parsed.is_empty() {

(None, pre_errors)

} else {

(Some(Box::new(parsed)), pre_errors)

}

}

fn stage(&self, buffers: Vec<Box<dyn InputBuffer>>) -> Box<dyn StagedBuffers> {

self.parser.stage(buffers)

}

fn splitter(&self) -> Box<dyn Splitter> {

let pre_splitter = self.preprocessor.splitter();

if let Some(splitter) = pre_splitter {

return splitter;

}

self.parser.splitter()

}

fn fork(&self) -> Box<dyn Parser> {

Box::new(MessageOrientedPreprocessedParser::new(

self.preprocessor.fork(),

self.parser.fork(),

))

}

/// Looks for a record boundary in `data`. Returns:

///

/// - `None`, if `data` does not necessarily complete a record.

///

/// - `Some(n)`, if the first `n` bytes of data, plus any data previously

/// presented for which `None` was returned, form one or more complete

/// records. If `n < data.len()`, then the caller should re-present

/// `data[n..]` for further splitting.

fn input(&mut self, data: &[u8]) -> Option<usize>;

/// Clears any state in this splitter and prepares it to start splitting new

/// data.

fn clear(&mut self);

buffer: Vec<u8>,

start: u64,

fragment: Range<usize>,

fed: usize,

splitter: Box<dyn Splitter>,

/// Returns a new stream splitter that finds breakpoints with `splitter`.

pub fn new(splitter: Box<dyn Splitter>) -> Self {

Self {

buffer: Vec::new(),

start: 0,

fragment: 0..0,

fed: 0,

splitter,

}

}

/// Returns the next full chunk of input, if any. `eoi` specifies whether

/// the input stream is complete. If `eoi` is true and this function returns

/// `None`, then there are no more chunks.

pub fn next(&mut self, eoi: bool) -> Option<&[u8]> {

match self

.splitter

.input(&self.buffer[self.fed..self.fragment.end])

{

Some(n) => {

let chunk = &self.buffer[self.fragment.start..self.fed + n];

self.fed += n;

self.fragment.start = self.fed;

Some(chunk)

}

None => {

self.fed = self.fragment.end;

if eoi && !self.fragment.is_empty() {

let chunk = &self.buffer[self.fragment.clone()];

self.fragment.start = self.fragment.end;

Some(chunk)

} else {

None

}

}

}

}

/// Appends `data` to the data to be broken into chunks.

pub fn append(&mut self, data: &[u8]) {

let final_len = self.fragment.len() + data.len();

if final_len > self.buffer.len() {

self.buffer.reserve(final_len - self.buffer.len());

}

self.buffer.copy_within(self.fragment.clone(), 0);

self.buffer.resize(self.fragment.len(), 0);

self.buffer.extend(data);

self.fed -= self.fragment.start;

self.start += self.fragment.start as u64;

self.fragment = 0..self.buffer.len();

}

// Reads no more than `limit` bytes of data from `file` into the splitter,

// with an initial minimum buffer size of `buffer_size`. Returns the number

// of bytes read or an I/O error.

pub fn read(

&mut self,

file: &mut File,

buffer_size: usize,

limit: usize,

) -> Result<usize, IoError> {

// Move data to beginning of buffer.

if self.fragment.start != 0 {

self.buffer.copy_within(self.fragment.clone(), 0);

self.fed -= self.fragment.start;

self.start += self.fragment.start as u64;

self.fragment = 0..self.fragment.len();

}

// Make sure there's some space to read data.

if self.fragment.len() == self.buffer.len() {

self.buffer

.resize(max(buffer_size, self.buffer.capacity() * 2), 0);

}

// Read data.

let mut space = &mut self.buffer[self.fragment.len()..];

if space.len() > limit {

space = &mut space[..limit];

}

let result = file.read(space);

if let Ok(n) = result {

self.fragment.end += n;

}

result

}

/// Returns the logical stream position of the next byte to be returned by

/// the splitter.

pub fn position(&self) -> u64 {

self.start + self.fragment.start as u64

}

/// Sets the logical stream position of the next byte to be returned by

/// the splitter to `offset`, and discards other state.

pub fn seek(&mut self, offset: u64) {

self.start = offset;

self.fragment = 0..0;

self.fed = 0;

self.splitter.clear();

}

/// Resets the splitter's state as if it were newly created.

pub fn reset(&mut self) {

self.seek(0);

}

/// Unique name of the data format.

fn name(&self) -> Cow<'static, str>;

/// Extract encoder configuration from an HTTP request.

///

/// Returns the extracted configuration cast to the `ErasedSerialize` trait

/// object (to keep this trait object-safe).

fn config_from_http_request(

&self,

endpoint_name: &str,

request: &HttpRequest,

) -> Result<Box<dyn ErasedSerialize>, ControllerError>;

/// Create a new encoder for the format.

///

/// # Arguments

///

/// * `config` - Format-specific configuration.

///

/// * `consumer` - Consumer to send encoded data batches to.

fn new_encoder(

&self,

endpoint_name: &str,

config: &ConnectorConfig,

key_schema: &Option<Relation>,

value_schema: &Relation,

consumer: Box<dyn OutputConsumer>,

) -> Result<Box<dyn Encoder>, ControllerError>;

/// Returns a reference to the consumer that the encoder is connected to.

fn consumer(&mut self) -> &mut dyn OutputConsumer;

/// Encode a batch of updates, push encoded buffers to the consumer

/// using [`OutputConsumer::push_buffer`].

fn encode(&mut self, batch: Arc<dyn SerBatchReader>) -> AnyResult<()>;

/// Maximum buffer size that this transport can transmit.

/// The encoder should not generate buffers exceeding this size.

fn max_buffer_size_bytes(&self) -> usize;

fn batch_start(&mut self, step: Step);

/// See OutputEndpoint::push_buffer.

fn push_buffer(&mut self, buffer: &[u8], num_records: usize);

/// See OutputEndpoint::push_key.

fn push_key(

&mut self,

key: Option<&[u8]>,

val: Option<&[u8]>,

headers: &[(&str, Option<&[u8]>)],

num_records: usize,

);

fn batch_end(&mut self);

/// Returns the approximate amount of memory used by the connector's

/// underlying implementation. For the Kafka connectors, for example, this

/// is the amount of memory used by librdkafka. Not all connectors use a

/// substantial amount of memory, so the default implementation returns 0.

fn memory(&self) -> usize {

0

}

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> {

self.0.fmt(f)

}

pub fn new(

description: String,

event_number: Option<u64>,

field: Option<String>,

invalid_text: Option<&str>,

invalid_bytes: Option<&[u8]>,

suggestion: Option<Cow<'static, str>>,

error_tag: Option<String>,

) -> Self {

Self(Box::new(ParseErrorInner::new(

description,

event_number,

field,

invalid_text,

invalid_bytes,

suggestion,

error_tag,

)))

}

pub fn text_event_error<E>(

msg: &str,

error: E,

event_number: u64,

invalid_text: Option<&str>,

suggestion: Option<Cow<'static, str>>,

) -> Self

where

E: ToString,

{

Self(Box::new(ParseErrorInner::text_event_error(

msg,

error,

event_number,

invalid_text,

suggestion,

)))

}

pub fn text_envelope_error(

description: String,

invalid_text: &str,

suggestion: Option<Cow<'static, str>>,

) -> Self {

Self(Box::new(ParseErrorInner::text_envelope_error(

description,

invalid_text,

suggestion,

)))

}

pub fn bin_event_error(

description: String,

event_number: u64,

invalid_bytes: &[u8],

suggestion: Option<Cow<'static, str>>,

) -> Self {

Self(Box::new(ParseErrorInner::bin_event_error(

description,

event_number,

invalid_bytes,

suggestion,

)))

}

pub fn bin_envelope_error(

description: String,

invalid_bytes: &[u8],

suggestion: Option<Cow<'static, str>>,

) -> Self {

Self(Box::new(ParseErrorInner::bin_envelope_error(

description,

invalid_bytes,

suggestion,

)))

}

/// Returns a new `ParseError` with the description modified by `f`.

///

/// Can be used, e.g., to prepend context to the description.

pub fn map_description<F>(self, f: F) -> Self

where

F: FnOnce(&str) -> String,

{

let mut inner = self.0;

let description = f(&inner.description);

inner.description = description;

Self(inner)

}

pub fn get_error_tag(&self) -> Option<String> {

self.0.get_error_tag()

}

/// Error description.

description: String,

/// Event number relative to the start of the stream.

///

/// An input stream is a series data change events (row insertions,

/// deletions, and updates). This field specifies the index (starting

/// from 1) of the event that caused the error, relative to the start of

/// the stream. In some cases this index cannot be identified, e.g., if

/// the error makes an entire block of events unparseable.

event_number: Option<u64>,

/// Field that failed to parse.

///

/// Only set when the parsing error can be attributed to a

/// specific field.

field: Option<String>,

/// Invalid fragment of input data.

///

/// Used for binary data formats and for text-based formats when the input

/// is not valid UTF-8 string.

invalid_bytes: Option<Vec<u8>>,

/// Invalid fragment of the input text.

///

/// Only used for text-based formats and in cases when input is valid UTF-8.

invalid_text: Option<String>,

/// Any additional information that may help fix the problem, e.g., example

/// of a valid input.

suggestion: Option<Cow<'static, str>>,

/// Used for rate limiting

tag: Option<String>,

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> {

let event = if let Some(event_number) = self.event_number {

format!(" (event #{})", event_number)

} else {

String::new()

};

let invalid_fragment = if let Some(invalid_bytes) = &self.invalid_bytes {

format!("\nInvalid bytes: {invalid_bytes:?}")

} else if let Some(invalid_text) = &self.invalid_text {

format!("\nInvalid fragment: '{invalid_text}'")

} else {

String::new()

};

let suggestion = if let Some(suggestion) = &self.suggestion {

format!("\n{suggestion}")

} else {

String::new()

};

write!(

f,

"Parse error{event}: {}{invalid_fragment}{suggestion}",

self.description

)

}

pub fn new(

description: String,

event_number: Option<u64>,

field: Option<String>,

invalid_text: Option<&str>,

invalid_bytes: Option<&[u8]>,