feast-dev · feast-ci-bot · Jan 8, 2020 · Jan 3, 2020 · Jan 7, 2020 · Dec 24, 2019
@@ -173,3 +173,9 @@ dmypy.json
 # Pyre type checker
 .pyre/
 .vscode
+
+# .flattened-pom.xml is generated by flatten-maven-plugin.
+# This pom should not be committed because it is only used during release / deployment.
+.flattened-pom.xml
+
+sdk/python/docs/html
@@ -169,6 +169,34 @@ postsubmits:
     # https://github.com/semver/semver/issues/232
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
 
+  - name: publish-java-sdk
+    decorate: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command:
+        - bash
+        - -c
+        - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
+        volumeMounts:
+        - name: gpg-keys
+          mountPath: /etc/gpg
+          readOnly: true
+        - name: maven-settings
+          mountPath: /root/.m2/settings.xml
+          subPath: settings.xml
+          readOnly: true
+      volumes:
+      - name: gpg-keys
+        secret:
+          secretName: gpg-keys
+      - name: maven-settings
+        secret:
+          secretName: maven-settings
+    branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
+
   - name: publish-docker-images
     decorate: true
     spec:
@@ -278,4 +306,5 @@ postsubmits:
         secret:
           secretName: feast-service-account
     branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -e 
+set -o pipefail
+
+GPG_KEY_IMPORT_DIR=/etc/gpg
+
+usage()
+{
+    echo "usage: publish-java-sdk.sh
+
+  --revision            Value for the revision e.g. '0.2.3'
+  --gpg-key-import-dir  Directory containing existing GPG keys to import.
+                        The directory should contain these 2 files:
+                        - public-key
+                        - private-key
+                        The default value is '/etc/gpg'
+
+  This script assumes the GPG private key is protected by a passphrase.
+  The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml
+  file, credentials to upload releases to Sonatype must also be provided.
+
+  # Example settings: ~/.m2/settings.xml
+  <settings>
+    <servers>
+      <server>
+        <id>ossrh</id>
+        <username>SONATYPE_USER</username>
+        <password>SONATYPE_PASSWORD</password>
+      </server>
+    </servers>
+    <profiles>
+      <profile>
+        <id>ossrh</id>
+        <properties>
+          <gpg.passphrase>GPG_PASSPHRASE</gpg.passphrase>
+        </properties>
+      </profile>
+    </profiles>
+  </settings>
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --revision )             REVISION="$2";            shift;;
+      --gpg-key-import-dir )   GPG_KEY_IMPORT_DIR="$2";  shift;;
+      -h | --help )            usage;                    exit;; 
+      * )                      usage;                    exit 1
+  esac
+  shift
+done
+
+if [ -z $REVISION ]; then usage; exit 1; fi
+
+echo "============================================================"
+echo "Checking Maven and GPG versions"
+echo "============================================================"
+mvn --version 
+echo ""
+gpg --version
+
+echo "============================================================"
+echo "Importing GPG keys"
+echo "============================================================"
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
+
+echo "============================================================"
+echo "Deploying Java SDK with revision: $REVISION"
+echo "============================================================"
+mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy
@@ -138,7 +138,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 30
@@ -191,7 +191,7 @@ spring:
     web-environment: false
 EOF
 
-nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/serving.warehouse.application.yml \
   &> /var/log/feast-serving-warehouse.log &
 sleep 15

@@ -75,6 +75,9 @@ Building jars for Feast
 # Build jars for Feast
 mvn --quiet --batch-mode --define skipTests=true clean package
 
+ls -lh core/target/*jar
+ls -lh serving/target/*jar
+
 echo "
 ============================================================
 Starting Feast Core
@@ -121,7 +124,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 30
@@ -172,7 +175,7 @@ spring:
     web-environment: false
 EOF
 
-nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/serving.online.application.yml \
   &> /var/log/feast-serving-online.log &
 sleep 15

@@ -32,7 +32,7 @@ build-cli:
 	$(MAKE) -C cli build-all
 
 build-java:
-	mvn clean verify -Drevision=$(VERSION)
+	mvn clean verify
 
 build-docker:
 	docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .

@@ -21,7 +21,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <groupId>feast</groupId>
+        <groupId>dev.feast</groupId>
         <artifactId>feast-parent</artifactId>
         <version>${revision}</version>
     </parent>
@@ -39,16 +39,12 @@
                     <skip>false</skip>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>org.xolstice.maven.plugins</groupId>
-                <artifactId>protobuf-maven-plugin</artifactId>
-            </plugin>
         </plugins>
     </build>
 
     <dependencies>
         <dependency>
-            <groupId>feast</groupId>
+            <groupId>dev.feast</groupId>
             <artifactId>feast-ingestion</artifactId>
             <version>${project.version}</version>
         </dependency>

@@ -66,7 +66,7 @@ public FeastResourceCollector feastResourceCollector(
   /**
    * Register custom Prometheus collector that exports metrics about JVM resource usage.
    *
-   * @return @{link {@link JVMResourceCollector}}
+   * @return {@link JVMResourceCollector}
    */
   @Bean
   public JVMResourceCollector jvmResourceCollector() {

@@ -82,7 +82,7 @@ public SpecService(
    * required. If the version is provided then it will be used for the lookup. If the version is
    * omitted then the latest version will be returned.
    *
-   * @param GetFeatureSetRequest containing the name and version of the feature set
+   * @param request containing the name and version of the feature set
    * @return GetFeatureSetResponse containing a single feature set
    */
   public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
@@ -141,7 +141,7 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
    *
    * <p>The version filter is optional; If not provided, this method will return all featureSet
    * versions of the featureSet name provided. Valid version filters should optionally contain a
-   * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1
+   * comparator (&lt;, &lt;=, &gt;, etc) and a version number, e.g. 10, &lt;10, &gt;=1
    *
    * @param filter filter containing the desired featureSet name and version filter
    * @return ListFeatureSetsResponse with list of featureSets found matching the filter

@@ -49,8 +49,9 @@ public class PackageUtil {
    * handled by default in Apache Beam.
    *
    * <pre>
-   * @code
+   * <code>
    * URL url = new URL("jar:file:/tmp/springexample/target/spring-example-1.0-SNAPSHOT.jar!/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar!/");
+   * </code>
    * String resolvedPath = resolveSpringBootPackageClasspath(url);
    * // resolvedPath should point to "/tmp/springexample/target/spring-example-1.0-SNAPSHOT/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar"
    * // Note that spring-example-1.0-SNAPSHOT.jar is extracted in the process.

@@ -0,0 +1,43 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+<dependency>
+  <groupId>dev.feast</groupId>
+  <artifactId>datatypes-java</artifactId>
+  <version>0.3.6-SNAPSHOT</version>
+</dependency>
+```
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Publishing
+----------
+
+TODO: this module should be published to Maven Central upon Feast releases—this
+needs to be set up in POM configuration and release automation.
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2019 The Feast Authors
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <name>Feast Data Types for Java</name>
+    <description>
+        Data types and service contracts used throughout Feast components and
+        their interchanges. These are generated from Protocol Buffers and gRPC
+        definitions included in the package.
+    </description>
+    <artifactId>datatypes-java</artifactId>
+
+    <parent>
+      <groupId>dev.feast</groupId>
+      <artifactId>feast-parent</artifactId>
+      <version>${revision}</version>
+      <relativePath>../..</relativePath>
+    </parent>
+
+    <build>
+      <plugins>
+        <plugin>
+          <groupId>org.xolstice.maven.plugins</groupId>
+          <artifactId>protobuf-maven-plugin</artifactId>
+          <configuration>
+            <checkStaleness>true</checkStaleness>
+            <protocArtifact>
+                com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
+            </protocArtifact>
+            <pluginId>grpc-java</pluginId>
+            <pluginArtifact>
+                io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
+            </pluginArtifact>
+          </configuration>
+          <executions>
+            <execution>
+              <goals>
+                <goal>compile</goal>
+                <goal>compile-custom</goal>
+                <goal>test-compile</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </build>
+
+    <dependencies>
+      <dependency>
+        <groupId>io.grpc</groupId>
+        <artifactId>grpc-services</artifactId>
+      </dependency>
+    </dependencies>
+</project>
@@ -0,0 +1 @@
+../../../../../protos/third_party
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \
+RUN mvn --also-make --projects core,ingestion \
   -DskipTests=true --batch-mode package
 #
 # Unpack the jar and copy the files into production Docker image

@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects serving -Drevision=$REVISION \
+RUN mvn --also-make --projects serving \
   -DskipTests=true --batch-mode package
 
 # ============================================================