Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/actionlint.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
self-hosted-runner:
# Labels of self-hosted runner in array of string
labels:
- ubuntu-latest-m
- ubuntu-latest-m
- arm-ubuntu-22-04-runner-one
214 changes: 214 additions & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,220 @@ jobs:
name: stackql_linux_amd64
path: build/stackql

linuxarmbuild:
name: Linux arm64 Build
runs-on: arm-ubuntu-22-04-runner-one
steps:

- name: Check out code into the Go module directory
uses: actions/checkout@v4.1.1

- name: Set up Go 1.x
uses: actions/setup-go@v5.0.0
with:
go-version: ^1.21
check-latest: true
cache: true
id: go

- name: Setup Python
uses: actions/setup-python@v5.0.0
with:
cache: pip
python-version: '3.11'

- name: Git Ref Parse
id: git_ref_parse
run: |
{
echo "SOURCE_NAME=${GITHUB_REF#refs/*/}"
echo "SOURCE_TAG=${GITHUB_REF#refs/tags/}"
echo "SOURCE_TAG=${GITHUB_REF#refs/tags/}"
} >> "${GITHUB_STATE}"

- name: Install psql
run: |
sudo apt-get update
sudo apt-get install --yes build-essential
sudo apt-get install --yes --no-install-recommends postgresql-client
gccVersion="$(gcc --version 2>&1 | head -n 1)"
javaVersion="$(java -version 2>&1 | head -n 1)"
echo "gcc version: ${gccVersion}"
echo "java version: ${javaVersion}"

- name: Install Python dependencies
run: |
pip3 install -r cicd/requirements.txt

- name: Generate rewritten registry for simulations
run: |
python3 test/python/registry-rewrite.py

- name: Get dependencies
run: |
git config --global "url.https://${GH_ACCESS_TOKEN}@github.com/.insteadOf" https://github.com/
go get -v -t -d ./...
go install golang.org/x/perf/cmd/benchstat@latest
env:
GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}}
GOPRIVATE: ${{env.GOPRIVATE}}

- name: Generate Build Flags and Build
env:
BUILDCOMMITSHA: ${{github.sha}}
BUILDBRANCH: ${{github.ref}}
BUILDPLATFORM: ${{runner.os}}
BUILDPATCHVERSION: ${{github.run_number}}
CGO_ENABLED: 1
CGO_LDFLAGS: '-static'
run: |
source cicd/version.txt
export BUILDMAJORVERSION="$MajorVersion"
export BUILDMINORVERSION="$MinorVersion"
if [[ ! "$BUILDBRANCH" == "*develop" ]]
then
# shellcheck disable=2269
export BUILDPATCHVERSION="${BUILDPATCHVERSION}"
fi
BUILDSHORTCOMMITSHA="$(echo "${BUILDCOMMITSHA}" | cut -c 1-7)"
export BUILDSHORTCOMMITSHA
BUILDDATE="$(date)"
export BUILDDATE
echo "BUILDMAJORVERSION: ${BUILDMAJORVERSION}"
echo "BUILDMINORVERSION: ${BUILDMINORVERSION}"
echo "BUILDPATCHVERSION: ${BUILDPATCHVERSION}"
echo "BUILDBRANCH: ${BUILDBRANCH}"
echo "BUILDCOMMITSHA: ${BUILDCOMMITSHA}"
echo "BUILDSHORTCOMMITSHA: ${BUILDSHORTCOMMITSHA}"
echo "BUILDDATE: ${BUILDDATE}"
echo "BUILDPLATFORM: ${BUILDPLATFORM}"

{
echo "BUILDMAJORVERSION=${BUILDMAJORVERSION}"
echo "BUILDMINORVERSION=${BUILDMINORVERSION}"
echo "BUILDPATCHVERSION=${BUILDPATCHVERSION}"
} >> "${GITHUB_ENV}"
python cicd/python/build.py --verbose --build

- name: Test
if: success()
run: python cicd/python/build.py --verbose --test

- name: Benchmark
if: success()
run: |
outFile="cicd/log/current-bench-nocache.txt"
baselineRefFile="cicd/ref/bench/baseline-go-bench.log"
thresholdRefFile="cicd/ref/bench/max-threshold-go-bench.log"
go test -run='^$' -bench . -benchtime=100x -count=6 \
--tags "sqlite_stackql" --ldflags "-X stackql/internal/stackql/planbuilder.PlanCacheEnabled=false" \
./... | tee ${outFile}
# shellcheck disable=SC2181
if [ "$?" = "0" ]; then
echo "Benchmarking run completed successfully"
else
echo "Benchmarking run failed"
exit 1
fi
echo ""
echo "##### Raw benchstat on current run #####"
echo ""
benchstat ${outFile} | tee cicd/log/raw-benchstat.txt
echo ""
echo "##### Comparing to baseline #####"
echo ""
benchstat -row .name -table .config -ignore goos,goarch,cpu ${baselineRefFile} ${outFile} | tee cicd/log/comparison-benchstat.txt
echo ""
echo "##### Comparing to max threshold #####"
echo ""
benchstat -row .name -table .config -ignore goos,goarch,cpu ${thresholdRefFile} ${outFile} | tee cicd/log/threshold-comparison-benchstat.txt
# shellcheck disable=SC2002,SC2062
comparisons=$( cat cicd/log/threshold-comparison-benchstat.txt \
| sed 's/.*\([+-][0-9][0-9]\.[0-9][0-9]*[%]\).*/\1/' \
| grep [+-][0-9][0-9]*\.[0-9][0-9]*[%] \
)
echo ""
echo "##### Comparisons #####"
echo ""
echo "${comparisons}"
# shellcheck disable=SC2062
nonNegativeComparisons=$( echo "${comparisons}" \
| grep -v [-].* \
|| true
)
echo "completed comparison logic"
if [ -z "${nonNegativeComparisons}" ]; then
echo "All max threshold comparisons are negative: this is acceptable"
else
echo "Some max threshold comparisons are positive or zero: this is unacceptable"
echo ""
echo "##### Non-negative comparisons #####"
echo "${nonNegativeComparisons}"
echo ""
exit 1
fi

- name: Mock Server Download
run: |
mvn \
org.apache.maven.plugins:maven-dependency-plugin:3.0.2:copy \
-Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded \
-DoutputDirectory=test/downloads

- name: Create certificates for robot tests
run: |
openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365

- name: Run robot mocked functional tests
if: success()
run: |
python cicd/python/build.py --robot-test --config='{ "variables": { "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": "true" } }'

- name: Output from mocked functional tests
if: always()
run: |
cat ./test/robot/functional/output.xml

- name: Run robot mocked functional tests with aggressive concurrency
if: success()
run: |
python cicd/python/build.py --robot-test --config='{ "variables": { "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": "true", "CONCURRENCY_LIMIT": -1 } }'

- name: Output from mocked functional tests with aggressive concurrency
if: always()
run: |
cat ./test/robot/functional/output.xml

- name: Run robot integration tests
if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release')
env:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_INTEGRATION_TESTING_SUB_ID: ${{ secrets.AZURE_INTEGRATION_TESTING_SUB_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
run: |
python cicd/python/build.py --robot-test-integration

- name: Prepare Test DB
if: success()
run: cp test/db/db.sqlite test/db/tmp/python-tests-tmp-db.sqlite

- name: Test Script
if: success()
run: python3 "${TESTSCRIPT}"
env:
TESTSCRIPT: ${{env.TESTSCRIPT}}

- name: Upload Artifact
uses: actions/upload-artifact@v4.3.1
if: success()
with:
name: stackql_linux_arm64
path: build/stackql


wsltest:
name: WSL Test
runs-on: windows-latest
Expand Down