jpegli/.github/workflows/build_test.yml at main · google/jpegli

372 lines (345 loc) · 13.4 KB
# Copyright (c) the JPEG XL Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
# Workflow for building and running tests.
name: Build/Test *nix
  merge_group:
    branches:
      - main
      - v*.*.x
  pull_request:
    types: [opened, reopened, labeled, unlabeled, synchronize]
permissions:
  contents: read
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
  build_test:
    name: ${{ startsWith(matrix.os, 'macos-') && 'MacOS' || 'Ubuntu' }} Build ${{ matrix.name }}
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI:none') }}
    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        # We have one job per "name" in the matrix. Attributes are set on the
        # specific job names.
        name: [release, debug, asan, msan, tsan, scalar]
        include:
          - name: release
            mode: release
            run_bench: true
            test_in_pr: true
            cmake_args: >-
              -DJPEGLI_TEST_TOOLS=ON
              -DJPEGLI_LIBJPEG_LIBRARY_VERSION="8.2.2"
              -DJPEGLI_LIBJPEG_LIBRARY_SOVERSION="8"
            # Track static stack size on build and check it doesn't exceed 3 kB.
            emit_stack_sizes: 1
            max_stack: 2400
            # Conformance tooling test requires numpy.
            # jpegli tests require imagemagick and libjpeg-turbo-progs
            apt_pkgs: imagemagick libjpeg-turbo-progs python3-numpy
          - name: debug
          # Build scalar-only hwy instructions.
          - name: scalar
            mode: release
            cxxflags: -DHWY_COMPILE_ONLY_SCALAR
          # Disabling optional features to speed up MSAN build a little bit.
          - name: msan
            os: ubuntu-24.04
            skip_install: true
            cmake_args: >-
              -DJPEGLI_ENABLE_DEVTOOLS=OFF
            cc: clang-18
            cxx: clang++-18
          - name: asan
            skip_install: true
          - name: tsan
            skip_install: true
          - name: coverage
            skip_install: true
            disable_tests:
              - test_jpegli_jni_wrapper # TODO: why this does not work?
          # Build with jpegli_cms based on lcms2 library.
          - name: release-lcms2
            mode: release
            cmake_args: >-
              -DJPEGLI_ENABLE_SKCMS=OFF
          - name: release-system-lcms2
            mode: release
            cmake_args: >-
              -DJPEGLI_ENABLE_SKCMS=OFF
              -DJPEGLI_FORCE_SYSTEM_LCMS2=ON
            apt_pkgs: liblcms2-dev
            # static build is impossible
            skip_install: true
          # Build/test on arm64 CPU
          - name: release:arm64
            os: ubuntu-24.04-arm
            mode: release
            run_bench: true
            test_in_pr: true
          # Builds with gcc in release mode
          - name: release:gcc
            os: ubuntu-latest
            mode: release
            cc: gcc
            cxx: g++
            run_bench: true
            test_in_pr: true
          - name: release:gcc-old
            os: ubuntu-24.04
            mode: release
            cc: gcc
            cxx: g++
          - name: release:gcc-old-old
            os: ubuntu-22.04
            mode: release
            cc: gcc
            cxx: g++
          # Builds with old clang in release mode
          - name: release:clang-old
            os: ubuntu-24.04
            mode: release
            cc: clang
            cxx: clang++
          - name: release:clang-old-old
            os: ubuntu-22.04
            mode: release
            cc: clang
            cxx: clang++
          # LTO builds
          - name: release_lto
            os: ubuntu-latest
            mode: release
            cc: clang-17
            cxx: clang++-17
            apt_pkgs: lld
            cmake_args: >-
              -DJPEGLI_ENABLE_LTO=ON
              -DCMAKE_LINKER_TYPE=LLD
            run_bench: true
            test_in_pr: true
          - name: release_lto:gcc
            os: ubuntu-latest
            mode: release
            cc: gcc
            cxx: g++
            cmake_args: >-
              -DJPEGLI_ENABLE_LTO=ON
            run_bench: true
            test_in_pr: true
          # OSX builds
          - name: release:osx
            os: macos-latest
            mode: release
            cmake_args: >-
              -DCMAKE_FIND_FRAMEWORK=NEVER
          - name: release:osx-15
            os: macos-15
            mode: release
            test_stack_limit: 128
            cmake_args: >-
              -DCMAKE_FIND_FRAMEWORK=NEVER
          - name: debug:osx-15
            # The last "free" X86 OSX runner
            os: macos-15-intel
            mode: debug
            cmake_args: >-
              -DCMAKE_FIND_FRAMEWORK=NEVER
          # Android builds
          - name: release_android
            os: ubuntu-latest
            mode: release
            skip_install: true
            skip_stats: true
            skip_test: true # can't finish "linking" tests without emulator
            cmake_args: >-
              -DANDROID_ABI=arm64-v8a
              -DANDROID_PLATFORM=28
              -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_LATEST_HOME}/build/cmake/android.toolchain.cmake
              -DJPEGLI_ENABLE_BENCHMARK=OFF
              -DJPEGLI_ENABLE_OPENEXR=OFF
              -DSJPEG_ANDROID_NDK_PATH=${ANDROID_NDK_LATEST_HOME}
              -DJPEGLI_BUNDLE_LIBPNG=ON
          - name: release_ios
            os: macos-latest
            mode: release
            skip_install: true
            skip_stats: true
            skip_test: true # can't finish "linking" tests without emulator
            cmake_args: >-
              -DCMAKE_SYSTEM_NAME=iOS
              -DCMAKE_OSX_ARCHITECTURES=arm64
              -DJPEGLI_ENABLE_BENCHMARK=OFF
              -DJPEGLI_ENABLE_OPENEXR=OFF
              -DJPEGLI_BUNDLE_LIBPNG=ON
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      # Whether we track the stack size.
      EMIT_STACK_SIZES: ${{ matrix.emit_stack_sizes }}
      TEST_STACK_LIMIT: ${{ matrix.test_stack_limit }}
      WILL_TEST: ${{  github.event_name == 'push' || (github.event_name == 'pull_request' && matrix.name != 'coverage' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full'))) }}
      WILL_BUILD: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && matrix.name != 'coverage') }}
      WILL_BENCH: ${{ github.event_name != 'merge_group' && matrix.run_bench || false }}
      WILL_COV: ${{ github.event_name == 'push' && matrix.name == 'coverage' }}
      FASTER_MSAN_BUILD: 1
    - name: Harden Runner
      uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
      with:
        egress-policy: audit
    - name: Check workflow configuration
      run: |
        if [[ "${{ matrix.run_bench && !matrix.test_in_pr }}" == "true" ]]; then
          echo "::error title::Configuration Error::'run_bench' requires 'test_in_pr'"
          exit 1
    - name: Install build deps MacOS
      if: startsWith(matrix.os, 'macos-')
      run: |
        # Should be already installed:
        #  brew install giflib jpeg-turbo libpng zlib
        # Not required, since we skip building documentation
        #  brew install doxygen graphviz
        brew install binutils ccache coreutils googletest libavif ninja openexr sdl2 webp
        if [[ "${WILL_BENCH}" == "true" ]]; then
          brew install google-benchmark
    - name: Checkout the source
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        submodules: true
        fetch-depth: 2
    - name: Install build deps Ubuntu
      if: startsWith(matrix.os, 'macos-') == false
      run: |
        INSTALL_BENCHMARK=''
        if [[ "${WILL_BENCH}" == "true" ]]; then
          if [[ "${WILL_TEST}" == "true" ]]; then
            INSTALL_BENCHMARK=benchmark
          else
            exit 1
        sudo ./tools/scripts/install_deps.sh build extras plugins ${INSTALL_BENCHMARK} -- ${{ matrix.apt_pkgs }}
        echo "CC=${{ matrix.cc || 'clang' }}" >> $GITHUB_ENV
        echo "CXX=${{ matrix.cxx || 'clang++' }}" >> $GITHUB_ENV
    - name: Setup the Homebrew prefixes
      if: startsWith(matrix.os, 'macos-')
      run: |
          CMAKE_PREFIX_PATH=`brew --prefix giflib`:`brew --prefix google-benchmark`:`brew --prefix jpeg-turbo`:`brew --prefix libpng`:`brew --prefix sdl2`:`brew --prefix zlib`
          echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV
    - name: Setup the LLVM source path
      if: matrix.name == 'msan' && env.WILL_BUILD == 'true'
      run: |
        LLVM_ROOT=${GITHUB_WORKSPACE}/llvm_root
        mkdir -p ${LLVM_ROOT}
        echo "LLVM_ROOT=${LLVM_ROOT}" >> $GITHUB_ENV
    - name: Cache LLVM sources
      if: matrix.name == 'msan' && env.WILL_BUILD == 'true'
      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      with:
        path: ${{ env.LLVM_ROOT }}
        key: llvm
    - name: Checkout the LLVM source
      if: matrix.name == 'msan' && env.WILL_BUILD == 'true'
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        submodules: false
        repository: llvm/llvm-project
        ref: llvmorg-18.1.3 # NB: 15.0.0 does not build ¯\_(ツ)_/¯
        path: llvm_root
    - name: Git environment
      id: git-env
      run: |
        echo "parent=$(git rev-parse ${{ github.sha }}^)" >> $GITHUB_OUTPUT
      shell: bash
    - name: ccache
      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      with:
        path: ${{ env.CCACHE_DIR }}
        # When the cache hits the key it is not updated, so if this is a rebuild
        # of the same Pull Request it will reuse the cache if still around. For
        # either Pull Requests or new pushes to main, this will use the parent
        # hash as the starting point from the restore-keys entry.
        key: build-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }}
        restore-keys: |
          build-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }}
    - name: Build
      if: env.WILL_BUILD == 'true'
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        MODE="${{ matrix.mode }}"
        BUILD_TESTS=$([ "${WILL_TEST}" == "true" ] && echo "ON" || echo "OFF")
        [[ -n "${MODE}" ]] || MODE="${{ matrix.name }}"
        SKIP_TEST=1 TARGETS=all CMAKE_CXX_FLAGS='${{ matrix.cxxflags }}' \
        ./ci.sh ${MODE} \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
          -DBUILD_TESTING=${BUILD_TESTS} \
          -DJPEGLI_ENABLE_HWY_AVX3_DL=false \
          ${{ matrix.cmake_args }}
    - name: Build stats
      if: env.WILL_BUILD == 'true'
      run: |
        awk '!/^#/ {total[$4]+=($2-$1);cntr[$4]+=1} END {for (key in total) print total[key]/cntr[key] " " key}' build/.ninja_log | sort -n | tail -n 25
    - name: ccache stats
      run: ccache --show-stats
    #- name: Build stats ${{ matrix.name }}
    #  if: env.WILL_BUILD == 'true' && matrix.mode == 'release'
    #  run: |
    #    SHARED_LIB_EXT="${{ startsWith(matrix.os, 'macos-') && 'dylib' || 'so' }}"
    #    SELECT_BINUTILS="${{ startsWith(matrix.os, 'macos-') && '--binutils `brew --prefix binutils`/bin/' || '' }}"
    #    tools/scripts/build_stats.py --save build/stats.json \
    #      --max-stack ${{ matrix.max_stack || '0' }} ${SELECT_BINUTILS} \
    #      cjpegli djpegli libjpegli.${SHARED_LIB_EXT} libjpegli_dec.${SHARED_LIB_EXT}
    # Check that we can build the example project against the installed libs.
    #- name: Install and build examples
    #  if: env.WILL_BUILD == 'true' && matrix.mode == 'release' && !matrix.skip_install
    #  run: |
    #    set -x
    #    sudo cmake --build build -- install
    #    cmake -Bbuild-example -Hexamples -G Ninja
    #    cmake --build build-example
        # Test that the built binaries run.
    #    echo -e -n "PF\n1 1\n-1.0\n\0\0\x80\x3f\0\0\x80\x3f\0\0\x80\x3f" > test.pfm
    #    build-example/encode_oneshot test.pfm test.jxl
    #    build-example/decode_oneshot test.jxl dec.pfm dec.icc
    # Run the tests on push and when requested in pull_request.
    - name: Test ${{ matrix.mode }}
      if: env.WILL_TEST == 'true'
      run: |
        disabled_tests="${{ join(matrix.disable_tests, '|') }}"
        if [[ -z "${disabled_tests}" ]]; then
          disabled_tests="bash_test";
        else
          disabled_tests="bash_test|${disabled_tests}";
        ./ci.sh test -E "${disabled_tests}"
    # Print the running time summary for the slowest tests.
    - name: Test runtime stats
      if: env.WILL_TEST == 'true'
      run: |
        sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true
    - name: Coverage report
      if: env.WILL_COV == 'true'
      run: |
        ./ci.sh coverage_report
    - name: Coverage upload to Codecov
      if: env.WILL_COV == 'true'
      uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
      with:
        flags: unittests
        files: build/coverage.xml
    - name: Fast benchmark ${{ matrix.mode }}
      if: env.WILL_BENCH == 'true'
      run: |
        cat /proc/cpuinfo | grep MHz | sort | uniq
        lscpu
        BENCHMARK_NUM_THREADS=3 STORE_IMAGES=0 ./ci.sh fast_benchmark
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

build_test.yml

Latest commit

History

build_test.yml

File metadata and controls
