Skip to content

bump datafusion to 35 and arrow to 50 #320

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
houqp merged 1 commit into from
Mar 24, 2024
Merged

bump datafusion to 35 and arrow to 50 #320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 20 additions & 51 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:

env:
# NOTE: the version is also defined in roapi_http_release.yml, columnq_cli_release.yml and Dockerfile
RUST_TC_NIGHTLY_VER: "2023-09-15"
RUST_TC_NIGHTLY_VER: "2024-03-01"

jobs:
build:
Expand All @@ -17,6 +17,7 @@ jobs:
RUSTFLAGS: "-C target-cpu=skylake"
steps:
- uses: actions/checkout@v2
- uses: rui314/setup-mold@v1
- uses: actions/cache@v2
with:
path: |
Expand All @@ -40,21 +41,21 @@ jobs:
- name: Format
run: cargo fmt --check
- name: Build
run: cargo build
run: mold -run cargo build
- name: Run tests
run: |
cargo test
run: mold -run cargo test
- name: Trim cache
run: |
which cargo-cache || cargo install cargo-cache
cargo cache trim -l 1G

simd_test:
database_test:
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C target-cpu=skylake"
steps:
- uses: actions/checkout@v2
- uses: rui314/setup-mold@v1
- uses: actions/cache@v2
with:
path: |
Expand All @@ -63,67 +64,34 @@ jobs:
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: simd-3-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
key: database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
restore-keys: |
simd-3-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
simd-3-${{ runner.os }}-cargo-
database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
database-1-${{ runner.os }}-cargo-
- name: Install nightly rust
uses: actions-rs/toolchain@v1
with:
profile: default
# toolchain: nightly
toolchain: nightly-${{ env.RUST_TC_NIGHTLY_VER }}
override: true
- name: Check
run: cargo clippy --features database
- name: Build
run: mold -run cargo build --features database
- name: Run tests
run: |
cargo test --features simd
run: mold -run cargo test --features database
- name: Trim cache
run: |
which cargo-cache || cargo install cargo-cache
cargo cache trim -l 1G

database_test:
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C target-cpu=skylake"
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
restore-keys: |
database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
database-1-${{ runner.os }}-cargo-
- name: Install nightly rust
uses: actions-rs/toolchain@v1
with:
profile: default
toolchain: nightly-${{ env.RUST_TC_NIGHTLY_VER }}
override: true
- name: Check
run: cargo clippy --features database
- name: Build
run: cargo build --features database
- name: Run tests
run: |
cargo test --features database
- name: Trim cache
run: |
which cargo-cache || cargo install cargo-cache
cargo cache trim -l 1G

object_store_memory_test:
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C target-cpu=skylake"
steps:
- uses: actions/checkout@v2
- uses: rui314/setup-mold@v1
- uses: actions/cache@v2
with:
path: |
Expand Down Expand Up @@ -171,8 +139,8 @@ jobs:
uses: BerniWittmann/background-server-action@v1.0.4
with:
command: /home/runner/work/roapi/roapi/test_end_to_end/query_blogs.sh
build: cargo build
start: cargo run --bin roapi -- -c test_end_to_end/test_object_store_memory.yml
build: mold -run cargo build
start: mold -run cargo run --bin roapi -- -c test_end_to_end/test_object_store_memory.yml
wait-on: "http://127.0.0.1:8000/api/schema"
# By default, wait-on will retry for 60 seconds. You can pass a custom timeout in seconds using wait-on-timeout.
# 10 minutes = 600 seconds
Expand All @@ -184,6 +152,7 @@ jobs:
RUSTFLAGS: "-C target-cpu=skylake"
steps:
- uses: actions/checkout@v2
- uses: rui314/setup-mold@v1
- uses: actions/cache@v2
with:
path: |
Expand Down Expand Up @@ -231,8 +200,8 @@ jobs:
uses: BerniWittmann/background-server-action@v1.0.4
with:
command: /home/runner/work/roapi/roapi/test_end_to_end/query_blogs.sh
build: cargo build
start: cargo run --bin roapi -- -c test_end_to_end/test_object_store_direct.yml
build: mold -run cargo build
start: mold -run cargo run --bin roapi -- -c test_end_to_end/test_object_store_direct.yml
wait-on: "http://127.0.0.1:8000/api/schema"
# By default, wait-on will retry for 60 seconds. You can pass a custom timeout in seconds using wait-on-timeout.
# 10 minutes = 600 seconds
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/columnq_cli_release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:

env:
# NOTE: the version is also defined in build.yml and Dockerfile
RUST_TC_NIGHTLY_VER: "2023-09-15"
RUST_TC_NIGHTLY_VER: "2024-03-01"

jobs:
# skip tag version validation on non-release branch run
Expand Down Expand Up @@ -51,7 +51,7 @@ jobs:
run: pip3 install 'maturin<0.12'
- name: Build wheels - x86_64
run: |
maturin build -m columnq-cli/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database-sqlite"
maturin build -m columnq-cli/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=database-sqlite"
pip install columnq-cli --no-index --find-links dist --force-reinstall
- name: Build wheels - universal2
env:
Expand All @@ -60,7 +60,7 @@ jobs:
run: |
# set SDKROOT for C dependencies
export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
maturin build -m columnq-cli/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database-sqlite"
maturin build -m columnq-cli/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=database-sqlite"
pip install columnq-cli --no-index --find-links dist --force-reinstall
columnq --help
- name: Upload wheels
Expand Down Expand Up @@ -89,7 +89,7 @@ jobs:
platform:
- python-architecture: "x64"
target: "x86_64-pc-windows-msvc"
features: "simd,database-sqlite"
features: "database-sqlite"
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
Expand Down Expand Up @@ -138,14 +138,14 @@ jobs:
- manylinux: "2010"
target: "x86_64-unknown-linux-musl"
image_tag: "x86_64-musl"
features: "simd,rustls,database-sqlite"
features: "rustls,database-sqlite"
rustflags: "-C target-cpu=skylake"
name_suffix: ""
upload: "true"
# - manylinux: "2010"
# target: "x86_64-unknown-linux-musl"
# image_tag: "x86_64-musl"
# features: "simd,native-tls,database-sqlite"
# features: "native-tls,database-sqlite"
# rustflags: "-C target-cpu=skylake"
# name_suffix: "-openssl"
# upload: "false"
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/roapi_release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:

env:
# NOTE: the version is also defined in build.yml and Dockerfile
RUST_TC_NIGHTLY_VER: "2023-09-15"
RUST_TC_NIGHTLY_VER: "2024-03-01"

jobs:
validate-release-tag:
Expand Down Expand Up @@ -53,7 +53,7 @@ jobs:
run: pip3 install 'maturin<0.12'
- name: Build wheels - x86_64
run: |
maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database-sqlite"
maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=database-sqlite"
pip install roapi --no-index --find-links dist --force-reinstall
- name: Build wheels - universal2
env:
Expand All @@ -62,7 +62,7 @@ jobs:
run: |
# set SDKROOT for C dependencies
export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database-sqlite"
maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=database-sqlite"
pip install roapi --no-index --find-links dist --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
Expand All @@ -88,7 +88,7 @@ jobs:
strategy:
matrix:
platform: [
{ python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd,database-sqlite" },
{ python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "database-sqlite" },
# { python-architecture: "x86", target: "i686-pc-windows-msvc", features: "" },
]
steps:
Expand Down Expand Up @@ -140,14 +140,14 @@ jobs:
- manylinux: "2010"
target: "x86_64-unknown-linux-musl"
image_tag: "x86_64-musl"
features: "simd,rustls,database-sqlite"
features: "rustls,database-sqlite"
name_suffix: ""
rustflags: "-C target-cpu=skylake"
upload: "true"
# - manylinux: "2010"
# target: "x86_64-unknown-linux-musl"
# image_tag: "x86_64-musl"
# features: "simd,native-tls"
# features: "native-tls"
# rustflags: "-C target-cpu=skylake"
# name_suffix: "-openssl"
# upload: "false"
Expand Down
Loading