apache · andygrove · Aug 6, 2026 · Aug 6, 2026
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -39,13 +39,13 @@ repository = "https://github.com/apache/datafusion-java"
 [workspace.dependencies]
 arrow = { version = "58", features = ["ffi"] }
 async-trait = "0.1"
-datafusion = { version = "53.1.0" }
-datafusion-proto = "53.1.0"
-datafusion-spark = "53.1.0"
-datafusion-substrait = "53.1.0"
+datafusion = { version = "54.1.0" }
+datafusion-proto = "54.1.0"
+datafusion-spark = "54.1.0"
+datafusion-substrait = "54.1.0"
 futures = "0.3"
 jni = "0.21"
-# Pinned to the major DataFusion 53.1 pulls in transitively (0.13.x) so we
+# Pinned to the major DataFusion 54.1 pulls in transitively (0.13.x) so we
 # share the same `dyn ObjectStore` vtable and don't double-link.
 object_store = { version = "0.13", default-features = false }
 prost = "0.14"

diff --git a/core/pom.xml b/core/pom.xml
@@ -129,7 +129,7 @@ under the License.
                             <url>https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-common/proto/datafusion_common.proto</url>
                             <outputDirectory>${project.build.directory}/proto/datafusion/proto-common/proto</outputDirectory>
                             <outputFileName>datafusion_common.proto</outputFileName>
-                            <sha512>d6f3368372ea277cc23e26f196994b81616d38599357bb374cbd7eb1760e649a789e4c133d86a395ac701049a500348da2ec039d3f978ac5d8112c2876dded1f</sha512>
+                            <sha512>e6a20115badec14f641b9237b0faa3f553162992444208322558ea3fcfaa24ca15334e4945827259dcb7c1de7f9aed659d496974eb8eeb1cfcf54ad4d7f6d20f</sha512>
                         </configuration>
                     </execution>
                     <execution>
@@ -140,7 +140,7 @@ under the License.
                             <url>https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto/proto/datafusion.proto</url>
                             <outputDirectory>${project.build.directory}/proto/datafusion/proto/proto</outputDirectory>
                             <outputFileName>datafusion.proto</outputFileName>
-                            <sha512>c3d162b8e2a418e03f74caceaccfd934af89bb95a12ede13d4cc1701d24c734d74b1e96372142b173db05938dab7f965ad60d476363308c441677a63ea5fbcf7</sha512>
+                            <sha512>bdae5361d32b4d4f0418677ce650ee8cc212fddfe9390b2c033ee4ab3bb3c969a6148b7b318b9bb268ee9384f18d4254ebe6e0701f15639059a8f2800d6e7847</sha512>
                         </configuration>
                     </execution>
                 </executions>

diff --git a/core/src/main/java/org/apache/datafusion/IoException.java b/core/src/main/java/org/apache/datafusion/IoException.java
@@ -21,9 +21,8 @@
 
 /**
  * IO-shaped failure: a local filesystem read failed, an object store request failed, or a parquet /
- * arrow / avro decoder reported a malformed file. Surfaces upstream {@code
- * DataFusionError::IoError}, {@code ObjectStore}, {@code ArrowError}, {@code ParquetError}, and
- * {@code AvroError}.
+ * arrow decoder reported a malformed file. Surfaces upstream {@code DataFusionError::IoError},
+ * {@code ObjectStore}, {@code ParquetError}, and the IO-shaped {@code ArrowError} variants.
  *
  * <p>Note: this is {@code org.apache.datafusion.IoException} (lowercase {@code o}), distinct from
  * {@code java.io.IOException}. The {@code IoException} spelling matches the orthography of the

diff --git a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
@@ -195,7 +195,7 @@ void describeReturnsSummaryStats() throws Exception {
           }
         }
       }
-      // DataFusion 53.1 reports these seven labels.
+      // DataFusion 54.1 reports these seven labels.
       assertTrue(seen.contains("count"), () -> "labels=" + seen);
       assertTrue(seen.contains("null_count"), () -> "labels=" + seen);
       assertTrue(seen.contains("mean"), () -> "labels=" + seen);

diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
@@ -73,7 +73,14 @@ static void checkFeatureEnabled() {
 
   /**
    * Build a minimal Substrait {@code Plan} that scans a registered named table {@code tableName}
-   * with two columns {@code (id int32, v int32)} and projects them through unchanged.
+   * with two columns {@code (id int64, v int64)} and projects them through unchanged.
+   *
+   * <p>The columns are declared {@code NULLABILITY_NULLABLE} to match the schema DataFusion infers
+   * for the CSV these tests register — CSV inference always yields nullable fields. DataFusion's
+   * Substrait consumer verifies that a field the plan declares non-nullable really is non-nullable
+   * in the table, and rejects the plan otherwise: a plan built around a "never null" assumption
+   * must not silently run against data that can contain nulls. The enclosing struct stays {@code
+   * REQUIRED} — the row itself is always present.
    */
   private static Plan namedTableScanPlan(String tableName) {
     NamedStruct schema =
@@ -87,13 +94,13 @@ private static Plan namedTableScanPlan(String tableName) {
                             .setI64(
                                 Type.I64
                                     .newBuilder()
-                                    .setNullability(Type.Nullability.NULLABILITY_REQUIRED)))
+                                    .setNullability(Type.Nullability.NULLABILITY_NULLABLE)))
                     .addTypes(
                         Type.newBuilder()
                             .setI64(
                                 Type.I64
                                     .newBuilder()
-                                    .setNullability(Type.Nullability.NULLABILITY_REQUIRED)))
+                                    .setNullability(Type.Nullability.NULLABILITY_NULLABLE)))
                     .setNullability(Type.Nullability.NULLABILITY_REQUIRED))
             .build();
     ReadRel read =

diff --git a/native-common/Cargo.toml b/native-common/Cargo.toml
@@ -27,13 +27,6 @@ publish = false
 readme = "README.md"
 description = "Shared JNI plumbing for DataFusion Java native crates: error-to-exception mapping, the per-cdylib Tokio runtime singleton, and the async-stream-to-FFI_ArrowArrayStream bridge."
 
-[features]
-# `datafusion-jni` builds DataFusion with `avro`, which adds the
-# `DataFusionError::AvroError` variant our classifier maps to IoException.
-# Feature-forwarded so consumers that don't read Avro (the Spark helper)
-# don't pull the apache-avro stack into their cdylib.
-avro = ["datafusion/avro"]
-
 [dependencies]
 datafusion = { workspace = true }
 futures = { workspace = true }

diff --git a/native-common/src/errors.rs b/native-common/src/errors.rs
@@ -97,10 +97,6 @@ fn classify(err: &DataFusionError) -> &'static str {
         DataFusionError::IoError(_)
         | DataFusionError::ObjectStore(_)
         | DataFusionError::ParquetError(_) => "org/apache/datafusion/IoException",
-        // The AvroError variant only exists when DataFusion is built with its
-        // `avro` feature, forwarded by this crate's own `avro` feature.
-        #[cfg(feature = "avro")]
-        DataFusionError::AvroError(_) => "org/apache/datafusion/IoException",
         // ArrowError is a 21-variant grab bag -- only some of those variants
         // are actually IO-shaped. DivideByZero / ArithmeticOverflow / Compute
         // / Cast / InvalidArgument / Memory etc. are execution-time failures
@@ -125,9 +121,15 @@ fn classify(err: &DataFusionError) -> &'static str {
 /// Map an [`ArrowError`] variant onto the Java exception class to throw.
 /// Only the genuinely IO-shaped variants (`IoError`, `IpcError`) land on
 /// `IoException`; everything else is execution-time and routes through
-/// `ExecutionException`. Schema/parse-shaped variants route through
-/// `PlanException` so a malformed IPC schema or a parse error surfaces as a
-/// query problem rather than an execution failure.
+/// `ExecutionException` -- including `AvroError`, which is where Avro decode
+/// failures land now that DataFusion 54 reads Avro through `arrow-avro`
+/// instead of `apache-avro` (the old `DataFusionError::AvroError` variant is
+/// gone). That puts Avro alongside the `CsvError` / `JsonError` decoder
+/// variants, which this arm already classified as execution failures.
+///
+/// Schema/parse-shaped variants route through `PlanException` so a malformed
+/// IPC schema or a parse error surfaces as a query problem rather than an
+/// execution failure.
 ///
 /// Variants without a clean caller-facing category (`CDataInterface`, the
 /// various overflow/index-overflow markers) fall through to the parent.

diff --git a/native/Cargo.toml b/native/Cargo.toml
@@ -82,9 +82,8 @@ arrow = { workspace = true }
 async-trait = { workspace = true }
 datafusion = { workspace = true, features = ["avro"] }
 # Shared JNI plumbing (error->exception mapping, runtime singleton,
-# StreamingReader). `avro` keeps the classifier's AvroError->IoException arm
-# in sync with the `avro` feature on `datafusion` above.
-datafusion-jni-common = { path = "../native-common", features = ["avro"] }
+# StreamingReader).
+datafusion-jni-common = { path = "../native-common" }
 datafusion-proto = { workspace = true }
 # Apache Spark-compatible functions + expression planners. Optional and
 # gated behind the `spark` feature (in the default set). The `core` feature

diff --git a/native/src/cache_manager.rs b/native/src/cache_manager.rs
@@ -29,10 +29,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use datafusion::execution::cache::cache_manager::CacheManagerConfig;
-use datafusion::execution::cache::cache_unit::{
-    DefaultFileStatisticsCache, DefaultFilesMetadataCache,
-};
-use datafusion::execution::cache::DefaultListFilesCache;
+use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache;
+use datafusion::execution::cache::{DefaultFilesMetadataCache, DefaultListFilesCache};
 
 use crate::proto_gen::CacheManagerOptionsProto;
 use datafusion_jni_common::errors::JniResult;
@@ -76,8 +74,20 @@ pub(crate) fn build_config(
         config.list_files_cache_ttl = ttl;
     }
 
-    if opts.file_statistics_cache_enabled.unwrap_or(false) {
-        config.table_files_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
+    // DataFusion 54 renamed `table_files_statistics_cache` to
+    // `file_statistics_cache` and made the *limit* the on/off switch:
+    // `CacheManager::try_new` now builds a `DefaultFileStatisticsCache`
+    // whenever `file_statistics_cache_limit > 0`, even when the cache slot is
+    // `None`. The default limit is non-zero, so an explicit `false` from the
+    // Java surface has to zero the limit -- otherwise upstream would install a
+    // stats cache the caller just asked us not to.
+    if let Some(enabled) = opts.file_statistics_cache_enabled {
+        if enabled {
+            config.file_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
+        } else {
+            config.file_statistics_cache = None;
+            config.file_statistics_cache_limit = 0;
+        }
     }
 
     Ok(Some(config))

diff --git a/native/src/memory.rs b/native/src/memory.rs
@@ -120,7 +120,27 @@ impl TrackingMemoryPool {
     }
 }
 
+/// DataFusion 54 added `Display` as a supertrait of [`MemoryPool`] so pools
+/// can render themselves in resource-exhausted messages. Mirror upstream's
+/// wrapper convention: name the wrapper, then defer to the inner pool for the
+/// limit/usage detail, and add the counters this wrapper exists to expose.
+impl std::fmt::Display for TrackingMemoryPool {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let (current, peak) = self.snapshot();
+        write!(
+            f,
+            "{}(current: {current}, peak: {peak}, inner: {})",
+            self.name(),
+            self.inner
+        )
+    }
+}
+
 impl MemoryPool for TrackingMemoryPool {
+    fn name(&self) -> &str {
+        "tracking"
+    }
+
     fn register(&self, consumer: &datafusion::execution::memory_pool::MemoryConsumer) {
         self.inner.register(consumer);
     }

diff --git a/native/src/table_provider.rs b/native/src/table_provider.rs
@@ -22,7 +22,6 @@
 //! `TableProvider` trait; it currently only supports a single-partition, no-pushdown scan,
 //! with future pushdown and partitioning support tracked as follow-ups.
 
-use std::any::Any;
 use std::fmt;
 use std::sync::Arc;
 
@@ -74,10 +73,6 @@ impl fmt::Debug for JavaTableProvider {
 
 #[async_trait]
 impl TableProvider for JavaTableProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         Arc::clone(&self.schema)
     }
@@ -152,10 +147,6 @@ impl ExecutionPlan for JavaScanExec {
         "JavaScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.plan_properties
     }

diff --git a/native/src/udf.rs b/native/src/udf.rs
@@ -17,7 +17,6 @@
 
 //! Java-backed scalar UDF support.
 
-use std::any::Any;
 use std::fmt;
 
 use datafusion::arrow::array::{make_array, Array, ArrayRef, StructArray};
@@ -80,10 +79,6 @@ impl std::hash::Hash for JavaScalarUdf {
 }
 
 impl ScalarUDFImpl for JavaScalarUdf {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn name(&self) -> &str {
         &self.name
     }

diff --git a/pom.xml b/pom.xml
@@ -40,7 +40,7 @@ under the License.
         <maven.compiler.target>17</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <junit.version>5.11.3</junit.version>
-        <datafusion.version>53.1.0</datafusion.version>
+        <datafusion.version>54.1.0</datafusion.version>
         <protobuf.version>3.25.5</protobuf.version>
         <arrow.version>19.0.0</arrow.version>
         <avro.version>1.12.0</avro.version>