apache · sciabarracom · Jul 6, 2021 · Jul 6, 2021 · Jul 6, 2021 · Jul 7, 2021
diff --git a/.travis.yml b/.travis.yml
@@ -39,13 +39,13 @@ script:
   - "./tools/travis/build.sh && ./tools/travis/test.sh"
 deploy:
   - provider: script
-    script: "./tools/travis/publish.sh openwhisk ${TRAVIS_TAG%@*} ${TRAVIS_TAG##*@}"
+    script: "./tools/travis/publish.sh openwhisk ${TRAVIS_TAG%@*} ${TRAVIS_TAG##*@}" && && ./tools/travis/publish.sh openwhisk 39 nightly"
     on:
       tags: true
       all_branches: true
       repo: apache/openwhisk-runtime-python
   - provider: script
-    script: "./tools/travis/publish.sh openwhisk 3 nightly && ./tools/travis/publish.sh openwhisk 3-ai nightly"
+    script: "./tools/travis/publish.sh openwhisk 3 nightly && ./tools/travis/publish.sh openwhisk 3-ai nightly" && ./tools/travis/publish.sh openwhisk 39 ${TRAVIS_TAG}"
     on:
       branch: master
       repo: apache/openwhisk-runtime-python

diff --git a/README.md b/README.md
@@ -47,10 +47,9 @@ Build using Python 3.7 (recommended)
 ```
 docker build -t actionloop-python-v3.7:1.0-SNAPSHOT $(pwd)/core/python3ActionLoop
 ```
-This tutorial assumes you're building with python 3.7. But if you want to use python 2.7 you can use:
-```
-docker build -t actionloop-python-v2.7:1.0-SNAPSHOT $(pwd)/core/python2ActionLoop
-```
+
+For runtime 3.9 or 3.6-ai you need also to copy `bin` and `lib` folders from 3.7 in the Docker folder.
+
 
 2.1. Check docker `IMAGE ID` (3rd column) for repository `actionloop-python-v3.7`
 ```
@@ -344,10 +343,10 @@ To build all those images, run the following command.
 
 You can optionally build a specific image by modifying the Gradle command. For example:
 ```
-./gradlew core:python3ActionLoop:distDocker
+./gradlew core:python3Action:distDocker
 ```
 
-The build will produce Docker images such as `actionloop-python-v3.7`
+The build will produce Docker images such as `action-python-v3.7`
 and will also tag the same image with the `whisk/` prefix. The latter
 is a convenience, which if you're testing with a local OpenWhisk
 stack, allows you to skip pushing the image to Docker Hub.
@@ -374,11 +373,11 @@ in first with the `docker` CLI.
 ### Using Your Image as an OpenWhisk Action
 
 You can now use this image as an OpenWhisk action. For example, to use
-the image `actionloop-python-v3.7` as an action runtime, you would run
+the image `action-python-v3.7` as an action runtime, you would run
 the following command.
 
 ```
-wsk action update myAction myAction.py --docker $DOCKER_USER/actionloop-python-v3.7
+wsk action update myAction myAction.py --docker $DOCKER_USER/action-python-v3.7
 ```
 
 ## Test Runtimes
@@ -400,14 +399,53 @@ Gradle allows you to selectively run tests. For example, the following
 command runs tests which match the given pattern and excludes all
 others.
 ```
-./gradlew :tests:test --tests *ActionLoopContainerTests*
+./gradlew :tests:test --tests Python*Tests
 ```
 
 ## Python 3 AI Runtime
-This action runtime enables developers to create AI Services with OpenWhisk. It comes with preinstalled libraries useful for running machine learning and deep learning inferences. [Read more about this runtime here](./core/python3AiActionLoop).
+This action runtime enables developers to create AI Services with OpenWhisk. It comes with preinstalled libraries useful for running machine learning and deep learning inferences. [Read more about this runtime here](./core/python3AiAction).
 
 ## Import Project into IntelliJ
 
 Follow these steps to import the project into your IntelliJ IDE.
 - Import project as gradle project.
 - Make sure the working directory is root of the project/repo.
+
+# Using extra libraries
+
+If you need more libraries for your Python action,  you can include a virtualenv in the zip file of the action.
+
+The requirement is that the zip file must have a subfolder named `virtualenv` with a script `virtualenv\bin\activate_this.py` working in an Linux AMD64 environment. It will be executed at start time to use your extra libraries.
+
+## Using requirements.txt
+
+Virtual envs are usually built listing your dependencies in a `requirements.txt`.
+
+If you have an action that requires addition libraries, you can just include `requirements.txt`.
+
+You have to create a folder `myaction` with at least two files:
+
+```
+__main__.py
+requirements.txt
+```
+
+Then zip your action and deploy to OpenWhisk, the requirements will be installed for you at init time, creating a suitable virtualenv.
+
+Keep in mind that resolving requirements involves downloading and install software, so your action timeout limit may need to be adjusted accordingly. Instead, you should consider using precompilation to resolve the requirements at build time.
+
+
+## Precompilation of a virtualenv
+
+The action containers can actually generate a virtualenv for you, provided you have a requirements.txt.
+
+
+If you have an action in the format described before (with a `requirements.txt`) you can build the zip file with the included files with:
+
+```
+zip -j -r myaction | docker run -i action-python-v3.7 -compile main >myaction.zip
+```
+
+You may use `v3.9` or `v3.6-ai` as well according to your Python version needs.
+
+The resulting action includes a virtualenv already built for you and that is fast to deploy and start as all the dependencies are already resolved. Note that there is a limit on the size of the zip file and this approach will not work for installing large libraries like Pandas or Numpy, instead use the provide "v.3.6-ai"  runtime instead which provides these libraries already for you.
diff --git a/core/python3AiActionLoop/Dockerfile → core/python36AiAction/Dockerfile b/core/python3AiActionLoop/Dockerfile → core/python36AiAction/Dockerfile
@@ -37,7 +37,7 @@ RUN curl -sL \
 FROM tensorflow/tensorflow:1.15.2-py3-jupyter
 
 # select the builder to use
-ARG GO_PROXY_BUILD_FROM=release
+ARG GO_PROXY_BUILD_FROM=source
 
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
             curl \
@@ -53,15 +53,19 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \
             && rm -rf /var/lib/apt/lists/*
 
 # PyTorch
-RUN pip3 install torch torchvision
+# persistent as it fails often
+RUN while ! pip list | grep torch ;\
+    do pip install torch ; done ;\
+    while ! pip list | grep torchvision ;\
+    do pip install torchvision ; done
 
 # rclone
 RUN curl -L https://downloads.rclone.org/rclone-current-linux-amd64.deb -o rclone.deb \
     && dpkg -i rclone.deb \
     && rm rclone.deb
 
 COPY requirements.txt requirements.txt
-RUN pip3 install --upgrade pip six &&\
+RUN pip3 install --upgrade pip six wheel &&\
     pip3 install --no-cache-dir -r requirements.txt &&\
     ln -sf /usr/bin/python3 /usr/local/bin/python
 

diff --git a/core/python3AiActionLoop/README.md → core/python36AiAction/README.md b/core/python3AiActionLoop/README.md → core/python36AiAction/README.md
diff --git a/core/python36AiAction/build.gradle b/core/python36AiAction/build.gradle
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ext.dockerImageName = 'action-python-v3.6-ai'
+apply from: '../../gradle/docker.gradle'
+
+distDocker.dependsOn 'copyLib'
+distDocker.dependsOn 'copyBin'
+distDocker.finalizedBy('cleanup')
+
+task copyLib(type: Copy) {
+    from '../python3Action/lib'
+    into './lib'
+}
+
+task copyBin(type: Copy) {
+    from '../python3Action/bin'
+    into './bin'
+}
+
+task cleanup(type: Delete) {
+    delete 'bin'
+    delete 'lib'
+}
diff --git a/core/python3AiActionLoop/requirements.txt → core/python36AiAction/requirements.txt b/core/python3AiActionLoop/requirements.txt → core/python36AiAction/requirements.txt
diff --git a/...onLoop/samples/smart-body-crop/.gitignore → ...Action/samples/smart-body-crop/.gitignore b/...onLoop/samples/smart-body-crop/.gitignore → ...Action/samples/smart-body-crop/.gitignore
diff --git a/...ionLoop/samples/smart-body-crop/common.py → ...iAction/samples/smart-body-crop/common.py b/...ionLoop/samples/smart-body-crop/common.py → ...iAction/samples/smart-body-crop/common.py
@@ -84,7 +84,7 @@ class CocoPart(Enum):
 
 NMS_Threshold = 0.1
 InterMinAbove_Threshold = 6
-Inter_Threshold = 0.1
+Inter_Threashold = 0.1
 Min_Subset_Cnt = 4
 Min_Subset_Score = 0.8
 Max_Human = 96
@@ -162,7 +162,7 @@ def estimate_pose(heatMat, pafMat):
                 # if two humans share a part (same part idx and coordinates), merge those humans
                 if set(c1['uPartIdx']) & set(c2['uPartIdx']) != empty_set:
                     is_merged = True
-                    # extend human1 connections with human2 connections
+                    # extend human1 connectios with human2 connections
                     conns_by_human[h1].extend(conns_by_human[h2])
                     conns_by_human.pop(h2)  # delete human2
                     break
@@ -243,7 +243,7 @@ def get_score(x1, y1, x2, y2, pafMatX, pafMatY):
         pafYs[idx] = pafMatY[my][mx]
 
     local_scores = pafXs * vx + pafYs * vy
-    thidxs = local_scores > Inter_Threshold
+    thidxs = local_scores > Inter_Threashold
 
     return sum(local_scores * thidxs), sum(thidxs)
 

diff --git a/...onLoop/samples/smart-body-crop/crop.ipynb → ...Action/samples/smart-body-crop/crop.ipynb b/...onLoop/samples/smart-body-crop/crop.ipynb → ...Action/samples/smart-body-crop/crop.ipynb
diff --git a/...samples/smart-body-crop/fashion-men-1.jpg → ...samples/smart-body-crop/fashion-men-1.jpg b/...samples/smart-body-crop/fashion-men-1.jpg → ...samples/smart-body-crop/fashion-men-1.jpg
diff --git a/...Loop/samples/smart-body-crop/inference.py → ...tion/samples/smart-body-crop/inference.py b/...Loop/samples/smart-body-crop/inference.py → ...tion/samples/smart-body-crop/inference.py
diff --git a/core/python39Action/Dockerfile b/core/python39Action/Dockerfile
@@ -0,0 +1,62 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# build go proxy from source
+FROM golang:1.15 AS builder_source
+ARG GO_PROXY_GITHUB_USER=apache
+ARG GO_PROXY_GITHUB_BRANCH=master
+RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \
+   https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\
+   cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \
+   mv proxy /bin/proxy
+
+# or build it from a release
+FROM golang:1.15 AS builder_release
+ARG GO_PROXY_RELEASE_VERSION=1.15@1.17.0
+RUN curl -sL \
+  https://github.com/apache/openwhisk-runtime-go/archive/{$GO_PROXY_RELEASE_VERSION}.tar.gz\
+  | tar xzf -\
+  && cd openwhisk-runtime-go-*/main\
+  && GO111MODULE=on go build -o /bin/proxy
+
+FROM python:3.9-buster
+
+# select the builder to use
+ARG GO_PROXY_BUILD_FROM=source
+
+# Install common modules for python
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN mkdir -p /action
+WORKDIR /
+COPY --from=builder_source /bin/proxy /bin/proxy_source
+COPY --from=builder_release /bin/proxy /bin/proxy_release
+RUN mv /bin/proxy_${GO_PROXY_BUILD_FROM} /bin/proxy
+ADD bin/compile /bin/compile
+ADD lib/launcher.py /lib/launcher.py
+
+# log initialization errors
+ENV OW_LOG_INIT_ERROR=1
+# the launcher must wait for an ack
+ENV OW_WAIT_FOR_ACK=1
+# using the runtime name to identify the execution environment
+ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.9
+# compiler script
+ENV OW_COMPILER=/bin/compile
+
+ENTRYPOINT ["/bin/proxy"]
diff --git a/core/python3AiActionLoop/build.gradle → core/python39Action/build.gradle b/core/python3AiActionLoop/build.gradle → core/python39Action/build.gradle
@@ -15,5 +15,24 @@
  * limitations under the License.
  */
 
-ext.dockerImageName = 'actionloop-python-v3.6-ai'
+ext.dockerImageName = 'action-python-v3.9'
 apply from: '../../gradle/docker.gradle'
+
+distDocker.dependsOn 'copyLib'
+distDocker.dependsOn 'copyBin'
+distDocker.finalizedBy('cleanup')
+
+task copyLib(type: Copy) {
+    from '../python3Action/lib'
+    into './lib'
+}
+
+task copyBin(type: Copy) {
+    from '../python3Action/bin'
+    into './bin'
+}
+
+task cleanup(type: Delete) {
+    delete 'bin'
+    delete 'lib'
+}
diff --git a/core/python39Action/requirements.txt b/core/python39Action/requirements.txt
@@ -0,0 +1,12 @@
+# default available packages for action-python-v39
+beautifulsoup4 == 4.9.3
+httplib2 == 0.19.1
+kafka_python == 1.4.7
+lxml == 4.6.3
+python-dateutil == 2.8.1
+requests == 2.25.1
+scrapy == 1.8.0
+simplejson == 3.17.2
+virtualenv == 20.4.7
+twisted == 21.2.0
+netifaces == 0.11.0
diff --git a/core/python3ActionLoop/CHANGELOG.md → core/python3Action/CHANGELOG.md b/core/python3ActionLoop/CHANGELOG.md → core/python3Action/CHANGELOG.md
@@ -19,6 +19,11 @@
 
 # Python 3 OpenWhisk Runtime Container
 
+# to include
+ - Use 1.17.1 of openwhisk-runtime-go to support symlinks in zips (required for virtualenvs)
+ - Support for python-3.9
+ - Support for generating actions with virtualenvs resolving requirements.txt
+
 ## 1.16.0
   - Introduce tutorial to deploy python runtimes locally (#101)
   - Use 1.17.0 release of openwhisk-runtime-go (#98)

diff --git a/core/python3ActionLoop/Dockerfile → core/python3Action/Dockerfile b/core/python3ActionLoop/Dockerfile → core/python3Action/Dockerfile
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 # build go proxy from source
 FROM golang:1.15 AS builder_source
-
 ARG GO_PROXY_GITHUB_USER=apache
 ARG GO_PROXY_GITHUB_BRANCH=master
 RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \
@@ -36,20 +36,11 @@ RUN curl -sL \
 FROM python:3.7-buster
 
 # select the builder to use
-ARG GO_PROXY_BUILD_FROM=release
+ARG GO_PROXY_BUILD_FROM=source
 
 # Install common modules for python
-RUN pip install \
-    beautifulsoup4==4.6.3 \
-    httplib2==0.11.3 \
-    kafka_python==1.4.3 \
-    lxml==4.2.5 \
-    python-dateutil==2.7.3 \
-    requests==2.19.1 \
-    scrapy==1.5.1 \
-    simplejson==3.16.0 \
-    virtualenv==16.0.0 \
-    twisted==18.7.0
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 RUN mkdir -p /action
 WORKDIR /
@@ -63,7 +54,7 @@ ADD lib/launcher.py /lib/launcher.py
 ENV OW_LOG_INIT_ERROR=1
 # the launcher must wait for an ack
 ENV OW_WAIT_FOR_ACK=1
-# using the runtime name to identify the execution environment
+# execution environment
 ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.7
 # compiler script
 ENV OW_COMPILER=/bin/compile

diff --git a/core/python3ActionLoop/Makefile → core/python3Action/Makefile b/core/python3ActionLoop/Makefile → core/python3Action/Makefile