Skip to content

Add benchmarking with CI job #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vvish wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add benchmarking with CI job #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 199 additions & 0 deletions .github/workflows/benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
name: Benchmarks

on:
push:
branches: [main]
pull_request:
branches: [main]

permissions:
contents: read

concurrency:
group: benchmarks-${{ github.ref }}
cancel-in-progress: true

jobs:
bench-linux:
name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
sys:
- { compiler: 'gcc', version: '12', flags: 'force_no_instr_set' }
- { compiler: 'gcc', version: '14', flags: 'avx' }
- { compiler: 'gcc', version: '13', flags: 'avx512' }
- { compiler: 'gcc', version: '13', flags: 'avx512pf' }
- { compiler: 'gcc', version: '13', flags: 'avx512vbmi' }
- { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' }
- { compiler: 'gcc', version: '13', flags: 'avx512vnni' }
- { compiler: 'clang', version: '17', flags: 'sse3' }
- { compiler: 'clang', version: '17', flags: 'avx' }

env:
PLATFORM: linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }}

defaults:
run:
shell: bash -l {0}

steps:
- name: Setup GCC compiler
if: ${{ matrix.sys.compiler == 'gcc' }}
run: |
GCC_VERSION=${{ matrix.sys.version }}
sudo apt-get update
sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION gcc-$GCC_VERSION
echo "CC=gcc-$GCC_VERSION" >> $GITHUB_ENV
echo "CXX=g++-$GCC_VERSION" >> $GITHUB_ENV

- name: Setup Clang compiler
if: ${{ matrix.sys.compiler == 'clang' }}
run: |
LLVM_VERSION=${{ matrix.sys.version }}
sudo apt-get update
sudo apt-get --no-install-suggests --no-install-recommends install clang-$LLVM_VERSION g++
sudo ln -sf /usr/include/asm-generic /usr/include/asm
echo "CC=clang-$LLVM_VERSION" >> $GITHUB_ENV
echo "CXX=clang++-$LLVM_VERSION" >> $GITHUB_ENV

- uses: actions/checkout@v4

- name: Set conda environment
uses: mamba-org/setup-micromamba@main
with:
environment-name: myenv
environment-file: environment-dev.yml
init-shell: bash
cache-downloads: true

- name: Setup SDE
if: startsWith(matrix.sys.flags, 'avx512')
run: sh install_sde.sh

- name: Configure CMake
env:
CC: ${{ env.CC }}
CXX: ${{ env.CXX }}
run: |
if [[ '${{ matrix.sys.flags }}' == 'avx' ]]; then CXX_FLAGS="$CXX_FLAGS -march=sandybridge"; fi
if [[ '${{ matrix.sys.flags }}' == 'sse3' ]]; then CXX_FLAGS="$CXX_FLAGS -march=nocona"; fi
if [[ '${{ matrix.sys.flags }}' == 'avx512' ]]; then CXX_FLAGS="$CXX_FLAGS -march=skylake-avx512"; fi
if [[ '${{ matrix.sys.flags }}' == 'avx512pf' ]]; then CXX_FLAGS="$CXX_FLAGS -march=knl"; fi
if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi' ]]; then CXX_FLAGS="$CXX_FLAGS -march=cannonlake"; fi
if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi2' ]]; then CXX_FLAGS="$CXX_FLAGS -march=icelake-server"; fi
if [[ '${{ matrix.sys.flags }}' == 'avx512vnni' ]]; then CXX_FLAGS="$CXX_FLAGS -march=knm"; fi
cmake -S . -B _bench_build \
-G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=$CC \
-DCMAKE_CXX_COMPILER=$CXX \
-DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
-DBUILD_BENCHMARKS=ON

- name: Build
run: ninja -C _bench_build bench_algorithms

- name: Run benchmarks
run: |
if echo '${{ matrix.sys.flags }}' | grep -q 'avx512'; then
./sde-external-9.48.0-2024-11-25-lin/sde64 -tgl -- \
./_bench_build/benchmark/bench_algorithms \
--benchmark_repetitions=10 --benchmark_min_time=0.1s \
--benchmark_out=benchmark-result.json
else
./_bench_build/benchmark/bench_algorithms \
--benchmark_repetitions=10 --benchmark_min_time=0.1s \
--benchmark_out=benchmark-result.json
fi

- name: Install compare tool dependencies
run: |
python3 -m venv _gbench_venv
_gbench_venv/bin/pip install --quiet -r "$CONDA_PREFIX/share/googlebenchmark/tools/requirements.txt"

- name: Publish results to job summary
run: |
{
echo "## Benchmarks: \`$PLATFORM\`"
echo
echo '```'
_gbench_venv/bin/python "$CONDA_PREFIX/share/googlebenchmark/tools/compare.py" --no-color -a \
filters benchmark-result.json BM_std_ BM_xsimd_
echo '```'
} >> "$GITHUB_STEP_SUMMARY"

- name: Upload benchmark artifact
uses: actions/upload-artifact@v4
with:
name: bench-${{ env.PLATFORM }}
path: benchmark-result.json
if-no-files-found: error


bench-macos:
name: 'macos-${{ matrix.os }}'
runs-on: macos-${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [14, 15]

env:
PLATFORM: macos-${{ matrix.os }}

defaults:
run:
shell: bash -e -l {0}

steps:
- uses: actions/checkout@v4

- name: Set conda environment
uses: mamba-org/setup-micromamba@main
with:
environment-name: myenv
environment-file: environment-dev.yml
init-shell: bash
cache-downloads: true

- name: Configure CMake
run: |
cmake -S . -B _bench_build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
-DBUILD_BENCHMARKS=ON

- name: Build
run: cmake --build _bench_build --target bench_algorithms --parallel 8

- name: Run benchmarks
run: |
./_bench_build/benchmark/bench_algorithms \
--benchmark_repetitions=10 --benchmark_min_time=0.1s \
--benchmark_out=benchmark-result.json

- name: Install compare tool dependencies
run: |
python3 -m venv _gbench_venv
_gbench_venv/bin/pip install --quiet -r "$CONDA_PREFIX/share/googlebenchmark/tools/requirements.txt"

- name: Publish results to job summary
run: |
{
echo "## Benchmarks: \`$PLATFORM\`"
echo
echo '```'
_gbench_venv/bin/python "$CONDA_PREFIX/share/googlebenchmark/tools/compare.py" --no-color -a \
filters benchmark-result.json BM_std_ BM_xsimd_
echo '```'
} >> "$GITHUB_STEP_SUMMARY"

- name: Upload benchmark artifact
uses: actions/upload-artifact@v4
with:
name: bench-${{ env.PLATFORM }}
path: benchmark-result.json
if-no-files-found: error

5 changes: 5 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,17 @@ target_compile_features(xsimd-algorithm INTERFACE cxx_std_20)
target_link_libraries(xsimd-algorithm INTERFACE xsimd)

OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF)
OPTION(BUILD_BENCHMARKS "xsimd-algorithm benchmarks" OFF)

if(BUILD_TESTS)
enable_testing()
add_subdirectory(test)
endif()

if(BUILD_BENCHMARKS)
add_subdirectory(benchmark)
endif()

# Installation
# ============

Expand Down
30 changes: 30 additions & 0 deletions benchmark/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
############################################################################
# Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and #
# Martin Renou #
# Copyright (c) QuantStack #
# Copyright (c) Serge Guelton #
# #
# Distributed under the terms of the BSD 3-Clause License. #
# #
# The full license is in the file LICENSE, distributed with this software. #
############################################################################

cmake_minimum_required(VERSION 3.8)

project(xsimd-algorithm-benchmark)

if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
find_package(xsimd-algorithm REQUIRED CONFIG)
endif ()

find_package(benchmark REQUIRED)

add_executable(bench_algorithms bench_algorithms.cpp)

target_link_libraries(bench_algorithms PRIVATE xsimd-algorithm benchmark::benchmark)

target_compile_options(bench_algorithms PRIVATE
$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-O3>)

set_target_properties(bench_algorithms PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/benchmark)
Loading
Loading