Skip to content

Commit d932c57

Browse files
partheaparthea
authored
ci: add ci for packages directory (googleapis#10675)
* ci: add ci for packages directory * clarify comment for set -eo pipefail * add high level overview for ci/run_conditional_tests.sh * document required environment variables * add high level overview for ci/run_single_test.sh * add python 3.11 to ci * remove python 3.11 not yet supported in github actions
1 parent ef221d4 commit d932c57

File tree

5 files changed

+388
-0
lines changed

5 files changed

+388
-0
lines changed

‎.github/workflows/docs.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
on:
2+
pull_request:
3+
branches:
4+
- main
5+
name: docs
6+
7+
permissions:
8+
contents: read
9+
10+
jobs:
11+
docs:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v3
16+
- name: Setup Python
17+
uses: actions/setup-python@v4
18+
with:
19+
python-version: "3.9"
20+
- name: Install nox
21+
run: |
22+
python -m pip install --upgrade setuptools pip wheel
23+
python -m pip install nox
24+
- name: Run docs
25+
env:
26+
BUILD_TYPE: presubmit
27+
TEST_TYPE: docs
28+
PY_VERSION: "3.9"
29+
run: |
30+
ci/run_conditional_tests.sh
31+
docfx:
32+
runs-on: ubuntu-latest
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@v3
36+
- name: Setup Python
37+
uses: actions/setup-python@v4
38+
with:
39+
python-version: "3.9"
40+
- name: Install nox
41+
run: |
42+
python -m pip install --upgrade setuptools pip wheel
43+
python -m pip install nox
44+
- name: Run docfx
45+
env:
46+
BUILD_TYPE: presubmit
47+
TEST_TYPE: docfx
48+
PY_VERSION: "3.9"
49+
run: |
50+
ci/run_conditional_tests.sh

‎.github/workflows/lint.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
on:
2+
pull_request:
3+
branches:
4+
- main
5+
name: lint
6+
7+
permissions:
8+
contents: read
9+
10+
jobs:
11+
lint:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v3
16+
- name: Setup Python
17+
uses: actions/setup-python@v4
18+
with:
19+
python-version: "3.9"
20+
- name: Install nox
21+
run: |
22+
python -m pip install --upgrade setuptools pip wheel
23+
python -m pip install nox
24+
- name: Run lint
25+
env:
26+
BUILD_TYPE: presubmit
27+
TEST_TYPE: lint
28+
PY_VERSION: "3.9"
29+
run: |
30+
ci/run_conditional_tests.sh
31+
- name: Run lint_setup_py
32+
env:
33+
BUILD_TYPE: presubmit
34+
TEST_TYPE: lint_setup_py
35+
PY_VERSION: "3.9"
36+
run: |
37+
ci/run_conditional_tests.sh

‎.github/workflows/unittest.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
on:
2+
pull_request:
3+
branches:
4+
- main
5+
name: unittest
6+
7+
permissions:
8+
contents: read
9+
10+
jobs:
11+
unit:
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
python: ['3.7', '3.8', '3.9', '3.10']
16+
steps:
17+
- name: Checkout
18+
uses: actions/checkout@v3
19+
- name: Setup Python
20+
uses: actions/setup-python@v4
21+
with:
22+
python-version: ${{ matrix.python }}
23+
- name: Install nox
24+
run: |
25+
python -m pip install --upgrade setuptools pip wheel
26+
python -m pip install nox
27+
- name: Run unit tests
28+
env:
29+
COVERAGE_FILE: .coverage-${{ matrix.python }}
30+
BUILD_TYPE: presubmit
31+
TEST_TYPE: unit
32+
PY_VERSION: ${{ matrix.python }}
33+
run: |
34+
ci/run_conditional_tests.sh
35+
- name: Upload coverage results
36+
uses: actions/upload-artifact@v3
37+
with:
38+
name: coverage-artifacts
39+
path: .coverage-${{ matrix.python }}
40+
prerelease:
41+
runs-on: ubuntu-latest
42+
strategy:
43+
matrix:
44+
python: ['3.10']
45+
steps:
46+
- name: Checkout
47+
uses: actions/checkout@v3
48+
- name: Setup Python
49+
uses: actions/setup-python@v4
50+
with:
51+
python-version: ${{ matrix.python }}
52+
- name: Install nox
53+
run: |
54+
python -m pip install --upgrade setuptools pip wheel
55+
python -m pip install nox
56+
- name: Run prerelease tests
57+
env:
58+
BUILD_TYPE: presubmit
59+
TEST_TYPE: prerelease
60+
PY_VERSION: ${{ matrix.python }}
61+
run: |
62+
ci/run_conditional_tests.sh
63+
64+
cover:
65+
runs-on: ubuntu-latest
66+
needs:
67+
- unit
68+
steps:
69+
- name: Checkout
70+
uses: actions/checkout@v3
71+
- name: Setup Python
72+
uses: actions/setup-python@v4
73+
with:
74+
python-version: "3.10"
75+
- name: Set number of files changes in packages directory
76+
id: packages
77+
run: echo "::set-output name=num_files_changed::$(git diff HEAD~1 -- packages/*.yml | wc -l)"
78+
- name: Install coverage
79+
if: ${{ steps.date.packages.num_files_changed > 0 }}
80+
run: |
81+
python -m pip install --upgrade setuptools pip wheel
82+
python -m pip install coverage
83+
- name: Download coverage results
84+
if: ${{ steps.date.packages.num_files_changed > 0 }}
85+
uses: actions/download-artifact@v3
86+
with:
87+
name: coverage-artifacts
88+
path: .coverage-results/
89+
- name: Report coverage results
90+
if: ${{ steps.date.packages.num_files_changed > 0 }}
91+
run: |
92+
coverage combine .coverage-results/.coverage*
93+
coverage report --show-missing --fail-under=100

‎ci/run_conditional_tests.sh‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/bash
2+
# Copyright 2022 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# This script requires the following environment variables to be set:
17+
# `BUILD_TYPE` should be one of ["presubmit", "continuous"]
18+
# `TEST_TYPE` should be one of ["lint", "lint_setup_py", "docs", "docfx", "prerelease"]
19+
# `PY_VERSION` should be one of ["3.7", "3.8", "3.9", "3.10", "3.11"]
20+
21+
# `TEST_TYPE` and `PY_VERSION` are required by the script `ci/run_single_test.sh`
22+
23+
# This script will determine which directories have changed
24+
# under the `packages` folder. For `BUILD_TYPE=="presubmit"`,
25+
# we'll compare against the `packages` folder in HEAD,
26+
# whereas for `BUILD_TYPE=="continuous"` we'll compare changes
27+
# with HEAD~1. For all directories that have changed files, we will
28+
# run the script located at `${PROJECT_ROOT}/ci/run_single_test.sh`.
29+
30+
# `-e` enables the script to automatically fail when a command fails
31+
# `-o pipefail` sets the exit code to non-zero if any command fails,
32+
# or zero if all commands in the pipeline exit successfully.
33+
set -eo pipefail
34+
35+
export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
36+
37+
# A script file for running the test in a sub project.
38+
test_script="${PROJECT_ROOT}/ci/run_single_test.sh"
39+
40+
if [ ${BUILD_TYPE} == "presubmit" ]; then
41+
# For presubmit build, we want to know the difference from the
42+
# common commit in origin/main.
43+
GIT_DIFF_ARG="origin/main..."
44+
45+
# Then fetch enough history for finding the common commit.
46+
git fetch origin main --deepen=200
47+
48+
elif [ ${BUILD_TYPE} == "continuous" ]; then
49+
# For continuous build, we want to know the difference in the last
50+
# commit. This assumes we use squash commit when merging PRs.
51+
GIT_DIFF_ARG="HEAD~.."
52+
53+
# Then fetch one last commit for getting the diff.
54+
git fetch origin main --deepen=1
55+
56+
else
57+
# Run everything.
58+
GIT_DIFF_ARG=""
59+
fi
60+
61+
# Then detect changes in the test scripts.
62+
63+
set +e
64+
git diff --quiet ${GIT_DIFF_ARG} ci
65+
changed=$?
66+
set -e
67+
if [[ "${changed}" -eq 0 ]]; then
68+
echo "no change detected in ci"
69+
else
70+
echo "change detected in ci, we should test everything"
71+
GIT_DIFF_ARG=""
72+
fi
73+
74+
# Now we have a fixed list, but we can change it to autodetect if
75+
# necessary.
76+
77+
subdirs=(
78+
packages
79+
)
80+
81+
RETVAL=0
82+
83+
for subdir in ${subdirs[@]}; do
84+
for d in `ls -d ${subdir}/*/`; do
85+
should_test=false
86+
if [ -n "${GIT_DIFF_ARG}" ]; then
87+
echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'"
88+
set +e
89+
git diff --quiet ${GIT_DIFF_ARG} ${d}
90+
changed=$?
91+
set -e
92+
if [[ "${changed}" -eq 0 ]]; then
93+
echo "no change detected in ${d}, skipping"
94+
else
95+
echo "change detected in ${d}"
96+
should_test=true
97+
fi
98+
else
99+
# If GIT_DIFF_ARG is empty, run all the tests.
100+
should_test=true
101+
fi
102+
if [ "${should_test}" = true ]; then
103+
echo "running test in ${d}"
104+
pushd ${d}
105+
# Temporarily allow failure.
106+
set +e
107+
${test_script}
108+
ret=$?
109+
set -e
110+
if [ ${ret} -ne 0 ]; then
111+
RETVAL=${ret}
112+
fi
113+
popd
114+
fi
115+
done
116+
done
117+
118+
exit ${RETVAL}

‎ci/run_single_test.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
#
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# This script requires the following environment variables to be set:
18+
# `TEST_TYPE` should be one of ["lint", "lint_setup_py", "docs", "docfx", "prerelease"]
19+
# `PY_VERSION` should be one of ["3.7", "3.8", "3.9", "3.10", "3.11"]
20+
21+
# This script is called by the `ci/run_conditional_tests.sh` script.
22+
# A specific `nox` session will be run, depending on the value of
23+
# `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is
24+
# `lint`, the `nox -s lint` session will be run.
25+
26+
27+
set -e
28+
29+
if [ -z "${TEST_TYPE}" ]; then
30+
echo "missing TEST_TYPE env var"
31+
exit 1
32+
fi
33+
34+
if [ -z "${PY_VERSION}" ]; then
35+
echo "missing PY_VERSION env var"
36+
exit 1
37+
fi
38+
39+
# Don't fail on errors so we can capture all of the output
40+
set +e
41+
42+
case ${TEST_TYPE} in
43+
lint)
44+
nox -s lint
45+
retval=$?
46+
;;
47+
lint_setup_py)
48+
nox -s lint_setup_py
49+
retval=$?
50+
;;
51+
docs)
52+
nox -s docs
53+
retval=$?
54+
;;
55+
docfx)
56+
nox -s docfx
57+
retval=$?
58+
;;
59+
prerelease)
60+
nox -s prerelease_deps-3.10
61+
retval=$?
62+
;;
63+
unit)
64+
case ${PY_VERSION} in
65+
"3.7")
66+
nox -s unit-3.7
67+
retval=$?
68+
;;
69+
"3.8")
70+
nox -s unit-3.8
71+
retval=$?
72+
;;
73+
"3.9")
74+
nox -s unit-3.9
75+
retval=$?
76+
;;
77+
"3.10")
78+
nox -s unit-3.10
79+
retval=$?
80+
;;
81+
"3.11")
82+
nox -s unit-3.11
83+
retval=$?
84+
;;
85+
*)
86+
;;
87+
esac
88+
esac
89+
90+
exit ${retval}

0 commit comments

Comments
 (0)