feldera · ryzhyk · Jun 29, 2024 · Jun 27, 2024 · Jun 27, 2024 · Jun 27, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- [SQL] Added `MATERIALIZED` views
+  ([#1959](https://github.com/feldera/feldera/pull/1959))
+
 ## [0.19.0] - 2024-06-25
 
 - [SQL] Preliminary support for MAP-typed values

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/adapters/src/format/json/output.rs b/crates/adapters/src/format/json/output.rs
@@ -494,7 +494,7 @@ mod test {
         let mut encoder = JsonEncoder::new(
             Box::new(consumer),
             config,
-            &Relation::new("TestStruct", false, TestStruct::schema()),
+            &Relation::new("TestStruct", false, TestStruct::schema(), false),
         );
         let zsets = batches
             .iter()
@@ -671,7 +671,7 @@ mod test {
         let mut encoder = JsonEncoder::new(
             Box::new(consumer),
             config,
-            &Relation::new("TestStruct", false, TestStruct::schema()),
+            &Relation::new("TestStruct", false, TestStruct::schema(), false),
         );
         let zset = OrdZSet::from_keys((), test_data()[0].clone());
 

diff --git a/crates/adapters/src/format/parquet/test.rs b/crates/adapters/src/format/parquet/test.rs
@@ -115,7 +115,7 @@ fn parquet_output() {
     let mut encoder = ParquetEncoder::new(
         Box::new(consumer),
         config,
-        Relation::new("TestStruct2", false, TestStruct2::schema()),
+        Relation::new("TestStruct2", false, TestStruct2::schema(), false),
     )
     .expect("Can't create encoder");
     let zset = OrdZSet::from_keys(

diff --git a/crates/adapters/src/static_compile/catalog.rs b/crates/adapters/src/static_compile/catalog.rs
@@ -44,6 +44,7 @@ impl Catalog {
             ))
         })
     }
+
     /// Add an input stream of Z-sets to the catalog.
     ///
     /// Adds a `DeCollectionHandle` to the catalog, which will deserialize
@@ -78,6 +79,37 @@ impl Catalog {
         self.register_output_zset(stream, schema);
     }
 
+    /// Like `register_input_zset`, but additionally materializes the integral
+    /// of the stream and makes it queryable.
+    pub fn register_materialized_input_zset<Z, D>(
+        &mut self,
+        stream: Stream<RootCircuit, Z>,
+        handle: ZSetHandle<Z::Key>,
+        schema: &str,
+    ) where
+        D: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<Z::Key>
+            + Clone
+            + Debug
+            + Send
+            + 'static,
+        Z: ZSet + Debug + Send + Sync,
+        Z::InnerBatch: Send,
+        Z::Key: Sync + From<D>,
+    {
+        let relation_schema: Relation = Self::parse_relation_schema(schema).unwrap();
+
+        self.register_input_collection_handle(InputCollectionHandle::new(
+            relation_schema,
+            DeZSetHandle::new(handle),
+        ))
+        .unwrap();
+
+        // Inputs are also outputs.
+        self.register_materialized_output_zset(stream, schema);
+    }
+
     /// Add an input stream created using `add_input_set` to catalog.
     ///
     /// Adds a `DeCollectionHandle` to the catalog, which will deserialize
@@ -112,6 +144,37 @@ impl Catalog {
         self.register_output_zset(stream, schema);
     }
 
+    /// Like `register_input_set`, but additionally materializes the integral
+    /// of the stream and makes it queryable.
+    pub fn register_materialized_input_set<Z, D>(
+        &mut self,
+        stream: Stream<RootCircuit, Z>,
+        handle: SetHandle<Z::Key>,
+        schema: &str,
+    ) where
+        D: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<Z::Key>
+            + Clone
+            + Debug
+            + Send
+            + 'static,
+        Z: ZSet + Debug + Send + Sync,
+        Z::InnerBatch: Send,
+        Z::Key: Sync + From<D>,
+    {
+        let relation_schema: Relation = Self::parse_relation_schema(schema).unwrap();
+
+        self.register_input_collection_handle(InputCollectionHandle::new(
+            relation_schema,
+            DeSetHandle::new(handle),
+        ))
+        .unwrap();
+
+        // Inputs are also outputs.
+        self.register_materialized_output_zset(stream, schema);
+    }
+
     /// Register an input handle created using `add_input_map`.
     ///
     /// Elements are inserted by value and deleted by key.  On insert, the
@@ -173,6 +236,52 @@ impl Catalog {
         self.register_output_map(stream, value_key_func, schema);
     }
 
+    /// Like `register_input_map`, but additionally materializes the integral
+    /// of the stream and makes it queryable.
+    pub fn register_materialized_input_map<K, KD, V, VD, U, UD, VF, UF>(
+        &mut self,
+        stream: Stream<RootCircuit, OrdIndexedZSet<K, V>>,
+        handle: MapHandle<K, V, U>,
+        value_key_func: VF,
+        update_key_func: UF,
+        schema: &str,
+    ) where
+        VF: Fn(&V) -> K + Clone + Send + Sync + 'static,
+        UF: Fn(&U) -> K + Clone + Send + Sync + 'static,
+        KD: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<K>
+            + Send
+            + 'static,
+        VD: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<V>
+            + Clone
+            + Debug
+            + Default
+            + Send
+            + 'static,
+        UD: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<U>
+            + Send
+            + 'static,
+        K: DBData + Sync + From<KD>,
+        V: DBData + Sync + From<VD>,
+        U: DBData + Sync + From<UD>,
+    {
+        let relation_schema: Relation = Self::parse_relation_schema(schema).unwrap();
+
+        self.register_input_collection_handle(InputCollectionHandle::new(
+            relation_schema,
+            DeMapHandle::new(handle, value_key_func.clone(), update_key_func.clone()),
+        ))
+        .unwrap();
+
+        // Inputs are also outputs.
+        self.register_materialized_output_map(stream, value_key_func, schema);
+    }
+
     /// Add an output stream of Z-sets to the catalog.
     pub fn register_output_zset<Z, D>(&mut self, stream: Stream<RootCircuit, Z>, schema: &str)
     where
@@ -189,6 +298,44 @@ impl Catalog {
     {
         let schema: Relation = Self::parse_relation_schema(schema).unwrap();
 
+        // Create handle for the stream itself.
+        let delta_handle = stream.output();
+
+        let handles = OutputCollectionHandles {
+            schema,
+            delta_handle: Box::new(<SerCollectionHandleImpl<_, D, ()>>::new(delta_handle))
+                as Box<dyn SerCollectionHandle>,
+
+            neighborhood_descr_handle: None,
+            neighborhood_handle: None,
+            neighborhood_snapshot_handle: None,
+            num_quantiles_handle: None,
+            quantiles_handle: None,
+        };
+
+        self.register_output_batch_handles(handles).unwrap();
+    }
+
+    /// Like `register_output_zset`, but additionally materializes the integral
+    /// of the stream and makes it queryable.
+    pub fn register_materialized_output_zset<Z, D>(
+        &mut self,
+        stream: Stream<RootCircuit, Z>,
+        schema: &str,
+    ) where
+        D: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<Z::Key>
+            + Clone
+            + Debug
+            + Send
+            + 'static,
+        Z: ZSet + Debug + Send + Sync,
+        Z::InnerBatch: Send,
+        Z::Key: Sync + From<D>,
+    {
+        let schema: Relation = Self::parse_relation_schema(schema).unwrap();
+
         let circuit = stream.circuit();
 
         // Create handle for the stream itself.
@@ -287,6 +434,49 @@ impl Catalog {
     /// streams contain values only.  Clients, e.g., the web console, can
     /// work with maps and z-sets in the same way.
     pub fn register_output_map<K, KD, V, VD, F>(
+        &mut self,
+        stream: Stream<RootCircuit, OrdIndexedZSet<K, V>>,
+        _key_func: F,
+        schema: &str,
+    ) where
+        F: Fn(&V) -> K + Clone + Send + Sync + 'static,
+        KD: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<K>,
+        VD: for<'de> DeserializeWithContext<'de, SqlSerdeConfig>
+            + SerializeWithContext<SqlSerdeConfig>
+            + From<V>
+            + Default
+            + Debug
+            + Clone
+            + Send
+            + 'static,
+        K: DBData + Send + Sync + From<KD> + Default,
+        V: DBData + Send + Sync + From<VD> + Default,
+    {
+        let schema: Relation = Self::parse_relation_schema(schema).unwrap();
+
+        // Create handle for the stream itself.
+        let delta_handle = stream.map(|(_k, v)| v.clone()).output();
+
+        let handles = OutputCollectionHandles {
+            schema,
+            delta_handle: Box::new(<SerCollectionHandleImpl<_, VD, ()>>::new(delta_handle))
+                as Box<dyn SerCollectionHandle>,
+
+            neighborhood_descr_handle: None,
+            neighborhood_handle: None,
+            neighborhood_snapshot_handle: None,
+            num_quantiles_handle: None,
+            quantiles_handle: None,
+        };
+
+        self.register_output_batch_handles(handles).unwrap();
+    }
+
+    /// Like `register_output_map`, but additionally materializes the integral
+    /// of the stream and makes it queryable.
+    pub fn register_materialized_output_map<K, KD, V, VD, F>(
         &mut self,
         stream: Stream<RootCircuit, OrdIndexedZSet<K, V>>,
         key_func: F,
@@ -493,7 +683,7 @@ mod test {
 
             let (input, hinput) = circuit.add_input_map::<u32, TestStruct, TestStruct, _>(|v, u| *v = u.clone());
 
-            catalog.register_input_map::<u32, u32, TestStruct, TestStruct, TestStruct, TestStruct, _, _>(
+            catalog.register_materialized_input_map::<u32, u32, TestStruct, TestStruct, TestStruct, TestStruct, _, _>(
                 input.clone(),
                 hinput,
                 |test_struct| test_struct.id,

diff --git a/crates/adapters/src/test/kafka.rs b/crates/adapters/src/test/kafka.rs
@@ -245,7 +245,7 @@ impl BufferConsumer {
         let buffer = MockDeZSet::new();
 
         // Input parsers don't care about schema yet.
-        let schema = Relation::new("mock_schema", false, vec![]);
+        let schema = Relation::new("mock_schema", false, vec![], false);
 
         let mut parser = format
             .new_parser(

diff --git a/crates/adapters/src/test/mod.rs b/crates/adapters/src/test/mod.rs
@@ -97,7 +97,7 @@ where
 {
     let input_handle = <MockDeZSet<T, U>>::new();
     // Input parsers don't care about schema yet.
-    let schema = Relation::new("mock_schema", false, vec![]);
+    let schema = Relation::new("mock_schema", false, vec![], false);
     let consumer = MockInputConsumer::from_handle(
         &InputCollectionHandle::new(schema, input_handle.clone()),
         config,
@@ -155,13 +155,14 @@ where
         let (input, hinput) = circuit.add_input_zset::<T>();
 
         let input_schema =
-            serde_json::to_string(&Relation::new("test_input1", false, schema.clone())).unwrap();
+            serde_json::to_string(&Relation::new("test_input1", false, schema.clone(), false))
+                .unwrap();
 
         let output_schema =
-            serde_json::to_string(&Relation::new("test_output1", false, schema)).unwrap();
+            serde_json::to_string(&Relation::new("test_output1", false, schema, false)).unwrap();
 
-        catalog.register_input_zset(input.clone(), hinput, &input_schema);
-        catalog.register_output_zset(input, &output_schema);
+        catalog.register_materialized_input_zset(input.clone(), hinput, &input_schema);
+        catalog.register_materialized_output_zset(input, &output_schema);
 
         Ok(catalog)
     })
@@ -196,7 +197,7 @@ where
     let buffer = MockDeZSet::<T, T>::new();
 
     // Input parsers don't care about schema yet.
-    let schema = Relation::new("mock_schema", false, vec![]);
+    let schema = Relation::new("mock_schema", false, vec![], false);
 
     let mut parser = format
         .new_parser(

diff --git a/crates/pipeline-types/src/program_schema.rs b/crates/pipeline-types/src/program_schema.rs
@@ -50,14 +50,17 @@ pub struct Relation {
     pub case_sensitive: bool,
     #[cfg_attr(feature = "testing", proptest(value = "Vec::new()"))]
     pub fields: Vec<Field>,
+    #[serde(default)]
+    pub materialized: bool,
 }
 
 impl Relation {
-    pub fn new(name: &str, case_sensitive: bool, fields: Vec<Field>) -> Self {
+    pub fn new(name: &str, case_sensitive: bool, fields: Vec<Field>, materialized: bool) -> Self {
         Self {
             name: name.to_string(),
             case_sensitive,
             fields,
+            materialized,
         }
     }
 

diff --git a/crates/pipeline-types/src/transport/delta_table.rs b/crates/pipeline-types/src/transport/delta_table.rs
@@ -170,7 +170,10 @@ fn test_delta_reader_config_serde() {
 
     let expected = r#"{"uri":"protocol:/path/to/somewhere","timestamp_column":"ts","mode":"follow","snapshot_filter":"ts BETWEEN '2005-01-01 00:00:00' AND '2010-12-31 23:59:59'","version":null,"datetime":"2010-12-31 00:00:00Z","customoption1":"val1","customoption2":"val2"}"#;
 
-    assert_eq!(serialized_config, expected);
+    assert_eq!(
+        serde_json::from_str::<serde_json::Value>(&serialized_config).unwrap(),
+        serde_json::from_str::<serde_json::Value>(&expected).unwrap()
+    );
 }
 
 impl DeltaTableReaderConfig {

diff --git a/crates/pipeline_manager/src/db/test.rs b/crates/pipeline_manager/src/db/test.rs
@@ -862,8 +862,8 @@ async fn versioning() {
             tenant_id,
             program_id,
             ProgramSchema {
-                inputs: vec![Relation::new("t1", false, vec![])],
-                outputs: vec![Relation::new("v1", false, vec![])],
+                inputs: vec![Relation::new("t1", false, vec![], false)],
+                outputs: vec![Relation::new("v1", false, vec![], false)],
             },
         )
         .await
@@ -941,10 +941,10 @@ async fn versioning() {
             program_id,
             ProgramSchema {
                 inputs: vec![
-                    Relation::new("t1", false, vec![]),
-                    Relation::new("t2", false, vec![]),
+                    Relation::new("t1", false, vec![], false),
+                    Relation::new("t2", false, vec![], false),
                 ],
-                outputs: vec![Relation::new("v1", false, vec![])],
+                outputs: vec![Relation::new("v1", false, vec![], false)],
             },
         )
         .await
@@ -963,8 +963,8 @@ async fn versioning() {
             tenant_id,
             program_id,
             ProgramSchema {
-                inputs: vec![Relation::new("tnew1", false, vec![])],
-                outputs: vec![Relation::new("vnew1", false, vec![])],
+                inputs: vec![Relation::new("tnew1", false, vec![], false)],
+                outputs: vec![Relation::new("vnew1", false, vec![], false)],
             },
         )
         .await