-
Notifications
You must be signed in to change notification settings - Fork 109
182 lines (157 loc) · 6.26 KB
/
build-rust.yml
File metadata and controls
182 lines (157 loc) · 6.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
name: Build Rust Sources
on:
workflow_call:
env:
CARGO_FLAGS: "--release --locked --all-targets"
CARGO_FEATURES_BASE: "fips,pubsub-emulator-test,iceberg-tests-fs,iceberg-tests-glue"
FELDERA_PLATFORM_VERSION_SUFFIX: ${{ github.sha }}
FELDERA_BUILD_ORIGIN: ci
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: ${{ vars.SCCACHE_CACHE_SIZE }}
SCCACHE_BUCKET: ${{ vars.SCCACHE_BUCKET }}
SCCACHE_ENDPOINT: ${{ vars.SCCACHE_ENDPOINT }}
SCCACHE_REGION: ${{ vars.SCCACHE_REGION }}
AWS_ACCESS_KEY_ID: "${{ secrets.CI_K8S_MINIO_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_K8S_MINIO_SECRET_ACCESS_KEY }}"
jobs:
build-rust:
name: Build Rust Binaries
# We run this on two different architectures (x86_64 and aarch64)
strategy:
matrix:
include:
- runner: [k8s-runners-amd64]
arch: x86_64
target: x86_64-unknown-linux-gnu
- runner: [k8s-runners-arm64]
arch: aarch64
target: aarch64-unknown-linux-gnu
runs-on: ${{ matrix.runner }}
container:
image: ghcr.io/feldera/feldera-dev:sha-06a694400a6aea1f568da04eaf786bfe84e0a869
steps:
- name: Show Kubernetes node
if: always()
run: |
echo "K8S node: ${K8S_NODE_NAME}"
- name: Checkout repository
uses: actions/checkout@v6
- name: Cache Cargo registry and index
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-registry-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
cargo-registry-${{ runner.os }}-${{ matrix.target }}-
# Thanks to rust cargo non-sense it's too hard to split this into test job
#
# limiting jobs with --test-threads 18: doc test call the linker for every test, on arm machines
# this can lead up to 128 parallel linkers being used OOM'ing the pod
# https://github.com/rust-lang/cargo/issues/10702
- name: Run Rust doc tests
if: matrix.arch == 'x86_64'
run: |
# Don't build the webconsole again for rust tests
export WEBCONSOLE_BUILD_DIR="$(mktemp -d)"
touch $WEBCONSOLE_BUILD_DIR/index.html
cargo test --locked --doc --workspace -- --test-threads 18
- name: Build Rust binaries
run: |
FEATURES="${{ env.CARGO_FEATURES_BASE }}"
cargo build ${{ env.CARGO_FLAGS }} --features "$FEATURES" --target=${{ matrix.target }}
- name: Print sccache stats
run: |
sccache --show-stats
# Get list of executables
- name: Collect executables
id: collect
run: |
FEATURES="${{ env.CARGO_FEATURES_BASE }}"
# Run again with --message-format=json to list out executables
# (No real recompile since nothing has changed).
# Then transform newlines to spaces for the artifact step.
EXES=$(cargo build ${{ env.CARGO_FLAGS }} --features "$FEATURES" --target=${{ matrix.target }} --message-format=json \
| jq -r '.executable | select(. != null)' | tr '\n' ' ')
echo "Found executables: $EXES"
# Save it as an output variable for subsequent steps
echo "executables=$EXES" >> $GITHUB_OUTPUT
# Copy all executables into a single directory because upload-artifact does not support
# multiple paths or `|` in glob patterns
- name: Copy executables
run: |
mkdir -p build-artifacts
for exe in ${{ steps.collect.outputs.executables }}; do
cp "$exe" build-artifacts/
done
rm build-artifacts/tutorial*
rm build-artifacts/galen-*
rm build-artifacts/fraud-*
rm build-artifacts/degrees
rm build-artifacts/coord
rm build-artifacts/ldbc_graphalytics-*
rm build-artifacts/logging_demo-*
rm build-artifacts/nexmark_gen-*
rm build-artifacts/orgchart
rm build-artifacts/path-*
rm build-artifacts/pool
mkdir -p build-release-artifacts
# Move the executables we ship to users to a separate directory
mv build-artifacts/fda build-release-artifacts/
mv build-artifacts/pipeline-manager build-release-artifacts/
# Upload test binaries as one artifact
- name: Upload test artifacts
uses: actions/upload-artifact@v7
with:
name: feldera-test-binaries-${{ matrix.target }}
path: build-artifacts
retention-days: 7
- name: Upload fda
uses: actions/upload-artifact@v7
with:
name: fda-${{ matrix.target }}
path: build-release-artifacts/fda
retention-days: 7
- name: Upload pipeline-manager
uses: actions/upload-artifact@v7
with:
name: pipeline-manager-${{ matrix.target }}
path: build-release-artifacts/pipeline-manager
retention-days: 7
build-fda-native:
name: Build fda (Windows)
# Builds fda for Windows using GitHub-hosted larger runners.
# macOS arm64 will be added once self-hosted macOS runners are available.
# Windows arm64 is skipped for now — the runner lacks Rust and MSVC.
strategy:
matrix:
include:
- runner: windows-latest-amd64
target: x86_64-pc-windows-msvc
runs-on: ${{ matrix.runner }}
# sccache is not available on Windows GitHub-hosted runners
env:
RUSTC_WRAPPER: ""
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Add Rust target
run: rustup target add ${{ matrix.target }}
- name: Cache Cargo registry and index
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-registry-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
cargo-registry-${{ runner.os }}-${{ matrix.target }}-
- name: Build fda
run: cargo build --release --locked -p fda --target=${{ matrix.target }}
- name: Upload fda
uses: actions/upload-artifact@v7
with:
name: fda-${{ matrix.target }}
path: target/${{ matrix.target }}/release/fda.exe
retention-days: 7