-
Notifications
You must be signed in to change notification settings - Fork 107
138 lines (121 loc) · 4.92 KB
/
build-rust.yml
File metadata and controls
138 lines (121 loc) · 4.92 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
name: Build Rust Sources
on:
workflow_call:
env:
CARGO_FLAGS: "--release --locked --all-targets"
CARGO_FEATURES_BASE: "fips,pubsub-emulator-test,iceberg-tests-fs,iceberg-tests-glue"
FELDERA_PLATFORM_VERSION_SUFFIX: ${{ github.sha }}
FELDERA_BUILD_ORIGIN: ci
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: ${{ vars.SCCACHE_CACHE_SIZE }}
SCCACHE_BUCKET: ${{ vars.SCCACHE_BUCKET }}
SCCACHE_ENDPOINT: ${{ vars.SCCACHE_ENDPOINT }}
SCCACHE_REGION: ${{ vars.SCCACHE_REGION }}
AWS_ACCESS_KEY_ID: "${{ secrets.CI_K8S_MINIO_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_K8S_MINIO_SECRET_ACCESS_KEY }}"
jobs:
build-rust:
name: Build Rust Binaries
# We run this on two different architectures (x86_64 and aarch64)
strategy:
matrix:
include:
- runner: [k8s-runners-amd64]
arch: x86_64
target: x86_64-unknown-linux-gnu
- runner: [k8s-runners-arm64]
arch: aarch64
target: aarch64-unknown-linux-gnu
runs-on: ${{ matrix.runner }}
container:
image: ghcr.io/feldera/feldera-dev:sha-db53e1302c944e29edd6d2af516fff7032b90e3b
steps:
- name: Show Kubernetes node
if: always()
run: |
echo "K8S node: ${K8S_NODE_NAME}"
- name: Checkout repository
uses: actions/checkout@v4
- name: Cache Cargo registry and index
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-registry-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
cargo-registry-${{ runner.os }}-${{ matrix.target }}-
# Thanks to rust cargo non-sense it's too hard to split this into test job
#
# limiting jobs with --test-threads 18: doc test call the linker for every test, on arm machines
# this can lead up to 128 parallel linkers being used OOM'ing the pod
# https://github.com/rust-lang/cargo/issues/10702
- name: Run Rust doc tests
if: matrix.arch == 'x86_64'
run: |
# Don't build the webconsole again for rust tests
export WEBCONSOLE_BUILD_DIR="$(mktemp -d)"
touch $WEBCONSOLE_BUILD_DIR/index.html
cargo test --locked --doc --workspace -- --test-threads 18
- name: Build Rust binaries
run: |
FEATURES="${{ env.CARGO_FEATURES_BASE }}"
cargo build ${{ env.CARGO_FLAGS }} --features "$FEATURES" --target=${{ matrix.target }}
- name: Print sccache stats
run: |
sccache --show-stats
# Get list of executables
- name: Collect executables
id: collect
run: |
FEATURES="${{ env.CARGO_FEATURES_BASE }}"
# Run again with --message-format=json to list out executables
# (No real recompile since nothing has changed).
# Then transform newlines to spaces for the artifact step.
EXES=$(cargo build ${{ env.CARGO_FLAGS }} --features "$FEATURES" --target=${{ matrix.target }} --message-format=json \
| jq -r '.executable | select(. != null)' | tr '\n' ' ')
echo "Found executables: $EXES"
# Save it as an output variable for subsequent steps
echo "executables=$EXES" >> $GITHUB_OUTPUT
# Copy all executables into a single directory because upload-artifact does not support
# multiple paths or `|` in glob patterns
- name: Copy executables
run: |
mkdir -p build-artifacts
for exe in ${{ steps.collect.outputs.executables }}; do
cp "$exe" build-artifacts/
done
rm build-artifacts/tutorial*
rm build-artifacts/galen-*
rm build-artifacts/fraud-*
rm build-artifacts/degrees
rm build-artifacts/coord
rm build-artifacts/ldbc_graphalytics-*
rm build-artifacts/logging_demo-*
rm build-artifacts/nexmark_gen-*
rm build-artifacts/orgchart
rm build-artifacts/path-*
rm build-artifacts/pool
mkdir -p build-release-artifacts
# Move the executables we ship to users to a separate directory
mv build-artifacts/fda build-release-artifacts/
mv build-artifacts/pipeline-manager build-release-artifacts/
# Upload test binaries as one artifact
- name: Upload test artifacts
uses: actions/upload-artifact@v4
with:
name: feldera-test-binaries-${{ matrix.target }}
path: build-artifacts
retention-days: 7
- name: Upload fda
uses: actions/upload-artifact@v4
with:
name: fda-${{ matrix.target }}
path: build-release-artifacts/fda
retention-days: 7
- name: Upload pipeline-manager
uses: actions/upload-artifact@v4
with:
name: pipeline-manager-${{ matrix.target }}
path: build-release-artifacts/pipeline-manager
retention-days: 7