Skip to content

[WIP] Clean up Feast configuration #604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
woop wants to merge 4 commits into from
Closed

[WIP] Clean up Feast configuration #604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e4f8fe9
Add general storage API and refactor existing store implementations (…
Apr 7, 2020
e7482af
Update Python SDK so FeatureSet can import Schema from Tensorflow met…
davidheryanto Apr 10, 2020
381dd59
Add validation to Core configuration and fix version loading
woop Apr 11, 2020
03e5673
Set default build version in Feast Core "version" field in Feast Prop…
woop Apr 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,14 @@
<configuration>
<skip>false</skip>
</configuration>
<executions>
<execution>
<id>build-info</id>
<goals>
<goal>build-info</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
Expand Down Expand Up @@ -207,5 +215,21 @@
<artifactId>jaxb-api</artifactId>
</dependency>

<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.0.Final</version>
</dependency>
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator</artifactId>
<version>6.1.2.Final</version>
</dependency>
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator-annotation-processor</artifactId>
<version>6.1.2.Final</version>
</dependency>

</dependencies>
</project>
232 changes: 217 additions & 15 deletions core/src/main/java/feast/core/config/FeastProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,53 +16,255 @@
*/
package feast.core.config;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import feast.core.config.FeastProperties.JobProperties.RunnerOptions;
import feast.core.config.FeastProperties.StreamProperties.FeatureStreamOptions;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import javax.annotation.PostConstruct;
import javax.validation.ConstraintViolation;
import javax.validation.ConstraintViolationException;
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
import javax.validation.constraints.AssertTrue;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Positive;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.validator.constraints.URL;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.info.BuildProperties;

@Getter
@Setter
@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true)
public class FeastProperties {

private String version;
private JobProperties jobs;
@Autowired
public FeastProperties(BuildProperties buildProperties) {
setVersion(buildProperties.getVersion());
}

public FeastProperties() {}

/* Feast Core Build Version */
@NotBlank private String version = "unknown";

/* Population job properties */
@NotNull private JobProperties jobs;

@NotNull
/* Feast Kafka stream properties */
private StreamProperties stream;

@Getter
@Setter
public static class JobProperties {

@NotBlank
/* Apache Beam runner type. Possible options: DirectRunner, DataflowRunner */
private String runner;
private Map<String, String> options;

/* Apache Beam runner options for population jobs */
private RunnerOptions runnerOptions;

/* (Optional) Additional arguments to pass to Beam population jobs */
private Map<String, String> extraRunnerOptions;

@NotNull
/* Population job metric properties */
private MetricsProperties metrics;
private JobUpdatesProperties updates;
}

@Getter
@Setter
public static class JobUpdatesProperties {
/* Timeout in seconds for each attempt to update or submit a new job to the runner */
@Positive private long jobUpdateTimeout;

/* Job update polling interval in millisecond. How frequently Feast will update running jobs. */
@Positive private long pollingIntervalMillis;

/** Apache Beam runner options for population jobs */
@Getter
@Setter
public static class RunnerOptions {

/* (Dataflow Runner Only) Project id to use when launching jobs. */
@NotBlank private String project;

/* (Dataflow Runner Only) The Google Compute Engine region for creating Dataflow jobs. */
@NotBlank private String region;

/* (Dataflow Runner Only) GCP availability zone for operations. */
@NotBlank private String zone;

/* (Dataflow Runner Only) Run the job as a specific service account, instead of the default GCE robot. */
@NotBlank private String serviceAccount;

/* (Dataflow Runner Only) GCE network for launching workers. */
@NotBlank private String network;

/* (Dataflow Runner Only) GCE subnetwork for launching workers. */
@NotBlank private String subnetwork;

/* (Dataflow Runner Only) Machine type to create Dataflow worker VMs as. */
private String workerMachineType;

/* (Dataflow Runner Only) The autoscaling algorithm to use for the workerpool. */
private String autoscalingAlgorithm;

/* (Dataflow Runner Only) Specifies whether worker pools should be started with public IP addresses. */
private Boolean usePublicIps;

/**
* (Dataflow Runner Only) A pipeline level default location for storing temporary files.
* Support Google Cloud Storage locations, e.g. gs://bucket/object
*/
@NotBlank private String tempLocation;

/* (Dataflow Runner Only) The maximum number of workers to use for the workerpool. */
private Integer maxNumWorkers;

/**
* (Direct Runner Only) Controls the amount of target parallelism the DirectRunner will use.
* Defaults to the greater of the number of available processors and 3. Must be a value
* greater than zero.
*/
private Integer targetParallelism;

private long timeoutSeconds;
private long pollingIntervalMillis;
/* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */
private String deadLetterTableSpec;
}

public Map<String, String> getRunnerOptionsMap() {
// First collect the existing "extra options"
Map<String, String> combinedOptions = new HashMap<String, String>(getExtraRunnerOptions());

// Convert all fields in RunnerOptions to <String, String> and merge
ObjectMapper oMapper = new ObjectMapper();
combinedOptions.putAll(
oMapper.convertValue(
getRunnerOptions(), new TypeReference<HashMap<String, String>>() {}));

return combinedOptions;
}
}

@AssertTrue
public boolean isValidJobRunnerSelected() {
String[] validRunners = new String[] {"DataflowRunner", "DirectRunner"};
return Arrays.asList(validRunners).contains(getJobs().getRunner());
}

/** Properties used to configure Feast's managed Kafka feature stream. */
@Getter
@Setter
public static class StreamProperties {

private String type;
private Map<String, String> options;
/* Feature stream type. Only "kafka" is supported. */
@NotBlank private String type;

/* Feature stream options */
@NotNull private FeatureStreamOptions options;

/** Feature stream options */
@Getter
@Setter
public static class FeatureStreamOptions {

/* Kafka topic to use for feature sets without source topics. */
@NotBlank private String topic = "feast-features";

/**
* Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined
* source.
*/
@NotBlank private String bootstrapServers = "localhost:9092";

/* Defines the number of copies of managed feature stream Kafka. */
@Positive private short replicationFactor = 1;

/* Number of Kafka partitions to to use for managed feature stream. */
@Positive private int partitions = 1;
}
}

@AssertTrue
public boolean isValidStreamTypeSelected() {
return Objects.equals(getStream().getType(), "kafka");
}

/** Feast population job metrics */
@Getter
@Setter
public static class MetricsProperties {

/* Population job metrics enabled */
private boolean enabled;
private String type;
private String host;
private int port;

/* Metric type. Possible options: statsd */
@NotBlank private String type;

/* Host of metric sink */
@URL private String host;

/* Port of metric sink */
@Positive private int port;
}

/**
* Validates all FeastProperties. This method runs after properties have been initialized and
* individually and conditionally validates each class.
*/
@PostConstruct
public void validate() {
ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
Validator validator = factory.getValidator();

// Validate root fields in FeastProperties
Set<ConstraintViolation<FeastProperties>> violations = validator.validate(this);
if (!violations.isEmpty()) {
throw new ConstraintViolationException(violations);
}

// Validate Stream properties
Set<ConstraintViolation<StreamProperties>> streamPropertyViolations =
validator.validate(getStream());
if (!streamPropertyViolations.isEmpty()) {
throw new ConstraintViolationException(streamPropertyViolations);
}

// Validate Stream Options
Set<ConstraintViolation<FeatureStreamOptions>> featureStreamOptionsViolations =
validator.validate(getStream().getOptions());
if (!featureStreamOptionsViolations.isEmpty()) {
throw new ConstraintViolationException(featureStreamOptionsViolations);
}

// Validate JobProperties
Set<ConstraintViolation<JobProperties>> jobPropertiesViolations = validator.validate(getJobs());
if (!jobPropertiesViolations.isEmpty()) {
throw new ConstraintViolationException(jobPropertiesViolations);
}

// Validate RunnerOptions
Set<ConstraintViolation<RunnerOptions>> runnerOptionsViolations =
validator.validate(getJobs().getRunnerOptions());
if (!runnerOptionsViolations.isEmpty()) {
throw new ConstraintViolationException(runnerOptionsViolations);
}

// Validate MetricsProperties
if (getJobs().getMetrics().isEnabled()) {
Set<ConstraintViolation<MetricsProperties>> jobMetricViolations =
validator.validate(getJobs().getMetrics());
if (!jobMetricViolations.isEmpty()) {
throw new ConstraintViolationException(jobMetricViolations);
}
}
}
}
9 changes: 4 additions & 5 deletions core/src/main/java/feast/core/config/FeatureStreamConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ public Source getDefaultSource(FeastProperties feastProperties) {
SourceType featureStreamType = SourceType.valueOf(streamProperties.getType().toUpperCase());
switch (featureStreamType) {
case KAFKA:
String bootstrapServers = streamProperties.getOptions().get("bootstrapServers");
String topicName = streamProperties.getOptions().get("topic");
String bootstrapServers = streamProperties.getOptions().getBootstrapServers();
String topicName = streamProperties.getOptions().getTopic();
Map<String, Object> map = new HashMap<>();
map.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
map.put(
Expand All @@ -59,9 +59,8 @@ public Source getDefaultSource(FeastProperties feastProperties) {
NewTopic newTopic =
new NewTopic(
topicName,
Integer.valueOf(streamProperties.getOptions().getOrDefault("numPartitions", "1")),
Short.valueOf(
streamProperties.getOptions().getOrDefault("replicationFactor", "1")));
streamProperties.getOptions().getPartitions(),
streamProperties.getOptions().getReplicationFactor());
CreateTopicsResult createTopicsResult =
client.createTopics(Collections.singleton(newTopic));
try {
Expand Down
17 changes: 3 additions & 14 deletions core/src/main/java/feast/core/config/JobConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,13 @@
import com.google.api.services.dataflow.DataflowScopes;
import com.google.common.base.Strings;
import feast.core.config.FeastProperties.JobProperties;
import feast.core.config.FeastProperties.JobUpdatesProperties;
import feast.core.job.JobManager;
import feast.core.job.Runner;
import feast.core.job.dataflow.DataflowJobManager;
import feast.core.job.direct.DirectJobRegistry;
import feast.core.job.direct.DirectRunnerJobManager;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.HashMap;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -55,10 +53,7 @@ public JobManager getJobManager(

JobProperties jobProperties = feastProperties.getJobs();
Runner runner = Runner.fromString(jobProperties.getRunner());
if (jobProperties.getOptions() == null) {
jobProperties.setOptions(new HashMap<>());
}
Map<String, String> jobOptions = jobProperties.getOptions();
Map<String, String> jobOptions = jobProperties.getRunnerOptionsMap();
switch (runner) {
case DATAFLOW:
if (Strings.isNullOrEmpty(jobOptions.getOrDefault("region", null))
Expand All @@ -77,7 +72,7 @@ public JobManager getJobManager(
credential);

return new DataflowJobManager(
dataflow, jobProperties.getOptions(), jobProperties.getMetrics());
dataflow, jobProperties.getRunnerOptionsMap(), jobProperties.getMetrics());
} catch (IOException e) {
throw new IllegalStateException(
"Unable to find credential required for Dataflow monitoring API", e);
Expand All @@ -88,7 +83,7 @@ public JobManager getJobManager(
}
case DIRECT:
return new DirectRunnerJobManager(
jobProperties.getOptions(), directJobRegistry, jobProperties.getMetrics());
jobProperties.getRunnerOptionsMap(), directJobRegistry, jobProperties.getMetrics());
default:
throw new IllegalArgumentException("Unsupported runner: " + jobProperties.getRunner());
}
Expand All @@ -99,10 +94,4 @@ public JobManager getJobManager(
public DirectJobRegistry directJobRegistry() {
return new DirectJobRegistry();
}

/** Extracts job update options from feast core options. */
@Bean
public JobUpdatesProperties jobUpdatesProperties(FeastProperties feastProperties) {
return feastProperties.getJobs().getUpdates();
}
}
Loading