feast-dev · feast-ci-bot · May 10, 2021 · May 5, 2021 · May 7, 2021
diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/bigtable/BigTableSinkRelation.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/bigtable/BigTableSinkRelation.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.functions.{col, length, struct, udf}
 import org.apache.spark.sql.{DataFrame, Row, SQLContext}
 import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation}
 import org.apache.spark.sql.types.{StringType, StructType}
+import feast.ingestion.utils.StringUtils
 import feast.ingestion.stores.serialization.Serializer
 import org.apache.hadoop.security.UserGroupInformation
 import org.apache.spark.SparkEnv
@@ -154,7 +155,7 @@ class BigTableSinkRelation(
 
   private def tableName: String = {
     val entities = config.entityColumns.mkString("__")
-    s"${config.projectName}__${entities}"
+    StringUtils.trimAndHash(s"${config.projectName}__${entities}", maxTableNameLength)
   }
 
   private def joinEntityKey: UserDefinedFunction = udf { r: Row =>
@@ -164,6 +165,7 @@ class BigTableSinkRelation(
   private val metadataColumnFamily = "metadata"
   private val schemaKeyPrefix      = "schema#"
   private val emptyQualifier       = ""
+  private val maxTableNameLength   = 50
 
   private def isSystemColumn(name: String) =
     (config.entityColumns ++ Seq(config.timestampColumn)).contains(name)

diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/cassandra/CassandraSinkRelation.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/cassandra/CassandraSinkRelation.scala
@@ -16,6 +16,7 @@
  */
 package feast.ingestion.stores.cassandra
 
+import feast.ingestion.utils.StringUtils
 import feast.ingestion.stores.serialization.Serializer
 import org.apache.spark.sql.expressions.UserDefinedFunction
 import org.apache.spark.sql.functions.{col, lit, struct, udf}
@@ -63,8 +64,11 @@ class CassandraSinkRelation(
     writer.append()
   }
 
+  val maxTableNameLength = 48
+
   def sanitizedForCassandra(expr: String): String = {
-    expr.replace('-', '_')
+    val replacedString = expr.replace('-', '_')
+    StringUtils.trimAndHash(replacedString, maxTableNameLength)
   }
 
   val tableName = {

diff --git a/spark/ingestion/src/main/scala/feast/ingestion/utils/StringUtils.scala b/spark/ingestion/src/main/scala/feast/ingestion/utils/StringUtils.scala
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2021 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.ingestion.utils
+
+import com.google.common.hash.Hashing
+
+object StringUtils {
+  private def suffixHash(expr: String): String = {
+    Hashing.murmur3_32().hashBytes(expr.getBytes).toString
+  }
+
+  def trimAndHash(expr: String, maxLength: Int): String = {
+    // Length 8 suffix as derived from murmurhash_32 implementation
+    val maxPrefixLength = maxLength - 8
+    if (expr.length > maxLength)
+      expr
+        .take(maxPrefixLength)
+        .concat(suffixHash(expr.substring(maxPrefixLength)))
+    else
+      expr
+  }
+}