Skip to content

feat: Add snowflake support for date & number with scale #3148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 1 commit into from
Aug 29, 2022
+13 −8
Merged

feat: Add snowflake support for date & number with scale #3148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion sdk/python/feast/infra/materialization/snowflake_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,11 @@ def generate_snowflake_materialization_query(
)

if feature_value_type_name == "UNIX_TIMESTAMP":
feature_sql = f'{feature_sql}(DATE_PART(EPOCH_NANOSECOND, "{feature.name}")) AS "{feature.name}"'
feature_sql = f'{feature_sql}(DATE_PART(EPOCH_NANOSECOND, "{feature.name}"::TIMESTAMP_LTZ)) AS "{feature.name}"'
elif feature_value_type_name == "DOUBLE":
feature_sql = (
f'{feature_sql}("{feature.name}"::DOUBLE) AS "{feature.name}"'
)
else:
feature_sql = f'{feature_sql}("{feature.name}") AS "{feature.name}"'

Expand Down
13 changes: 6 additions & 7 deletions sdk/python/feast/infra/offline_stores/snowflake_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,18 +264,17 @@ def get_table_column_names_and_types(
]
else:
raise NotImplementedError(
"Numbers larger than INT64 are not supported"
"NaNs or Numbers larger than INT64 are not supported"
)
else:
raise NotImplementedError(
"The following Snowflake Data Type is not supported: DECIMAL -- Convert to DOUBLE"
)
elif row["type_code"] in [3, 5, 9, 10, 12]:
row["snowflake_type"] = "NUMBERwSCALE"

elif row["type_code"] in [5, 9, 10, 12]:
error = snowflake_unsupported_map[row["type_code"]]
raise NotImplementedError(
f"The following Snowflake Data Type is not supported: {error}"
)
elif row["type_code"] in [1, 2, 4, 6, 7, 8, 11, 13]:
elif row["type_code"] in [1, 2, 3, 4, 6, 7, 8, 11, 13]:
row["snowflake_type"] = snowflake_type_code_map[row["type_code"]]
else:
raise NotImplementedError(
Expand All @@ -291,6 +290,7 @@ def get_table_column_names_and_types(
0: "NUMBER",
1: "DOUBLE",
2: "VARCHAR",
3: "DATE",
4: "TIMESTAMP",
6: "TIMESTAMP_LTZ",
7: "TIMESTAMP_TZ",
Expand All @@ -300,7 +300,6 @@ def get_table_column_names_and_types(
}

snowflake_unsupported_map = {
3: "DATE -- Convert to TIMESTAMP",
5: "VARIANT -- Try converting to VARCHAR",
9: "OBJECT -- Try converting to VARCHAR",
10: "ARRAY -- Try converting to VARCHAR",
Expand Down
2 changes: 2 additions & 0 deletions sdk/python/feast/type_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,8 +625,10 @@ def snowflake_type_to_feast_value_type(snowflake_type: str) -> ValueType:
"VARCHAR": ValueType.STRING,
"NUMBER32": ValueType.INT32,
"NUMBER64": ValueType.INT64,
"NUMBERwSCALE": ValueType.DOUBLE,
"DOUBLE": ValueType.DOUBLE,
"BOOLEAN": ValueType.BOOL,
"DATE": ValueType.UNIX_TIMESTAMP,
"TIMESTAMP": ValueType.UNIX_TIMESTAMP,
"TIMESTAMP_TZ": ValueType.UNIX_TIMESTAMP,
"TIMESTAMP_LTZ": ValueType.UNIX_TIMESTAMP,
Expand Down