Skip to content

Commit 490e0a7

Browse files
author
Chen Zhiling
committed
Add Redis storage implementation (#547)
* Add Redis storage * Remove staleness check; can be checked at the service level * Remove staleness related tests * Add dependencies to top level pom * Clean up code
1 parent e1bb6a6 commit 490e0a7

File tree

12 files changed

+1772
-0
lines changed

12 files changed

+1772
-0
lines changed

‎storage/api/src/main/java/feast/storage/common/retry/BackOffExecutor.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.storage.common.retry;
18+
19+
import java.io.Serializable;
20+
import org.apache.beam.sdk.util.BackOff;
21+
import org.apache.beam.sdk.util.BackOffUtils;
22+
import org.apache.beam.sdk.util.FluentBackoff;
23+
import org.apache.beam.sdk.util.Sleeper;
24+
import org.joda.time.Duration;
25+
26+
public class BackOffExecutor implements Serializable {
27+
28+
private final Integer maxRetries;
29+
private final Duration initialBackOff;
30+
31+
public BackOffExecutor(Integer maxRetries, Duration initialBackOff) {
32+
this.maxRetries = maxRetries;
33+
this.initialBackOff = initialBackOff;
34+
}
35+
36+
public void execute(Retriable retriable) throws Exception {
37+
FluentBackoff backoff =
38+
FluentBackoff.DEFAULT.withMaxRetries(maxRetries).withInitialBackoff(initialBackOff);
39+
execute(retriable, backoff);
40+
}
41+
42+
private void execute(Retriable retriable, FluentBackoff backoff) throws Exception {
43+
Sleeper sleeper = Sleeper.DEFAULT;
44+
BackOff backOff = backoff.backoff();
45+
while (true) {
46+
try {
47+
retriable.execute();
48+
break;
49+
} catch (Exception e) {
50+
if (retriable.isExceptionRetriable(e) && BackOffUtils.next(sleeper, backOff)) {
51+
retriable.cleanUpAfterFailure();
52+
} else {
53+
throw e;
54+
}
55+
}
56+
}
57+
}
58+
}

‎storage/api/src/main/java/feast/storage/common/retry/Retriable.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.storage.common.retry;
18+
19+
public interface Retriable {
20+
void execute() throws Exception;
21+
22+
Boolean isExceptionRetriable(Exception e);
23+
24+
void cleanUpAfterFailure();
25+
}

‎storage/connectors/redis/pom.xml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,64 @@
1313
<name>Feast Storage Connector for Redis</name>
1414

1515
<dependencies>
16+
<dependency>
17+
<groupId>io.lettuce</groupId>
18+
<artifactId>lettuce-core</artifactId>
19+
</dependency>
20+
21+
<dependency>
22+
<groupId>org.apache.commons</groupId>
23+
<artifactId>commons-lang3</artifactId>
24+
<version>3.9</version>
25+
</dependency>
26+
27+
<dependency>
28+
<groupId>com.google.auto.value</groupId>
29+
<artifactId>auto-value-annotations</artifactId>
30+
<version>1.6.6</version>
31+
</dependency>
32+
33+
<dependency>
34+
<groupId>com.google.auto.value</groupId>
35+
<artifactId>auto-value</artifactId>
36+
<version>1.6.6</version>
37+
<scope>provided</scope>
38+
</dependency>
39+
40+
<dependency>
41+
<groupId>org.mockito</groupId>
42+
<artifactId>mockito-core</artifactId>
43+
<version>2.23.0</version>
44+
<scope>test</scope>
45+
</dependency>
46+
47+
<!-- To run actual Redis for ingestion integration test -->
48+
<dependency>
49+
<groupId>com.github.kstyrc</groupId>
50+
<artifactId>embedded-redis</artifactId>
51+
<scope>test</scope>
52+
</dependency>
53+
54+
<dependency>
55+
<groupId>org.apache.beam</groupId>
56+
<artifactId>beam-runners-direct-java</artifactId>
57+
<version>${org.apache.beam.version}</version>
58+
<scope>test</scope>
59+
</dependency>
60+
61+
<dependency>
62+
<groupId>org.hamcrest</groupId>
63+
<artifactId>hamcrest-core</artifactId>
64+
<scope>test</scope>
65+
</dependency>
66+
67+
<dependency>
68+
<groupId>org.hamcrest</groupId>
69+
<artifactId>hamcrest-library</artifactId>
70+
<scope>test</scope>
71+
</dependency>
72+
73+
1674
<dependency>
1775
<groupId>junit</groupId>
1876
<artifactId>junit</artifactId>

‎storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retrieval/FeatureRowDecoder.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.storage.connectors.redis.retrieval;
18+
19+
import feast.core.FeatureSetProto.FeatureSetSpec;
20+
import feast.core.FeatureSetProto.FeatureSpec;
21+
import feast.types.FeatureRowProto.FeatureRow;
22+
import feast.types.FieldProto.Field;
23+
import java.util.Comparator;
24+
import java.util.List;
25+
import java.util.stream.Collectors;
26+
import java.util.stream.IntStream;
27+
28+
public class FeatureRowDecoder {
29+
30+
private final String featureSetRef;
31+
private final FeatureSetSpec spec;
32+
33+
public FeatureRowDecoder(String featureSetRef, FeatureSetSpec spec) {
34+
this.featureSetRef = featureSetRef;
35+
this.spec = spec;
36+
}
37+
38+
/**
39+
* A feature row is considered encoded if the feature set and field names are not set. This method
40+
* is required for backward compatibility purposes, to allow Feast serving to continue serving non
41+
* encoded Feature Row ingested by an older version of Feast.
42+
*
43+
* @param featureRow Feature row
44+
* @return boolean
45+
*/
46+
public Boolean isEncoded(FeatureRow featureRow) {
47+
return featureRow.getFeatureSet().isEmpty()
48+
&& featureRow.getFieldsList().stream().allMatch(field -> field.getName().isEmpty());
49+
}
50+
51+
/**
52+
* Validates if an encoded feature row can be decoded without exception.
53+
*
54+
* @param featureRow Feature row
55+
* @return boolean
56+
*/
57+
public Boolean isEncodingValid(FeatureRow featureRow) {
58+
return featureRow.getFieldsList().size() == spec.getFeaturesList().size();
59+
}
60+
61+
/**
62+
* Decoding feature row by repopulating the field names based on the corresponding feature set
63+
* spec.
64+
*
65+
* @param encodedFeatureRow Feature row
66+
* @return boolean
67+
*/
68+
public FeatureRow decode(FeatureRow encodedFeatureRow) {
69+
final List<Field> fieldsWithoutName = encodedFeatureRow.getFieldsList();
70+
71+
List<String> featureNames =
72+
spec.getFeaturesList().stream()
73+
.sorted(Comparator.comparing(FeatureSpec::getName))
74+
.map(FeatureSpec::getName)
75+
.collect(Collectors.toList());
76+
List<Field> fields =
77+
IntStream.range(0, featureNames.size())
78+
.mapToObj(
79+
featureNameIndex -> {
80+
String featureName = featureNames.get(featureNameIndex);
81+
return fieldsWithoutName
82+
.get(featureNameIndex)
83+
.toBuilder()
84+
.setName(featureName)
85+
.build();
86+
})
87+
.collect(Collectors.toList());
88+
return encodedFeatureRow
89+
.toBuilder()
90+
.clearFields()
91+
.setFeatureSet(featureSetRef)
92+
.addAllFields(fields)
93+
.build();
94+
}
95+
}

0 commit comments

Comments
 (0)