codegate/.github/workflows/integration-tests.yml at main · SuperCodeTool/codegate

332 lines (288 loc) · 12.1 KB
# This workflow will run the integration tests for the project
name: Tests - Integration
  workflow_call:
    inputs:
      artifact-name:
        description: 'The name of the artifact to download'
        required: true
        type: string
    secrets:
      copilot-key:
        description: 'The Copilot key to use for integration tests'
        required: true
      anthropic-key:
        description: 'The Anthropic key to use for integration tests'
        required: true
      openrouter-key:
        description: 'The Openrouter key to use for integration tests'
        required: true
  integration-tests:
    name: Test
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false # Continue running other tests if one fails
      matrix:
        python-version: [ "3.12" ]
        test-provider: [ "copilot", "openai", "anthropic", "ollama", "vllm", "llamacpp", "openrouter" ]
      ENV_COPILOT_KEY: ${{ secrets.copilot-key }}
      ENV_OPENAI_KEY: ${{ secrets.copilot-key }} # We use the same key for OpenAI as the Copilot tests
      ENV_ANTHROPIC_KEY: ${{ secrets.anthropic-key }}
      ENV_OPENROUTER_KEY: ${{ secrets.openrouter-key }}
      CA_CERT_FILE: "/home/runner/work/codegate/codegate/codegate_volume/certs/ca.crt"
      CODEGATE_CONTAINER_NAME: "codegate"
      CODEGATE_MOUNT_PATH_CERT_FILE: "/app/codegate_volume/certs/ca.crt"
      CODEGATE_LOG_LEVEL: "DEBUG"
      LOCAL_OLLAMA_URL: "http://localhost:11434"
      LOCAL_VLLM_URL: "http://localhost:8000"
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          lfs: true
      - name: Checkout LFS objects
        run: git lfs pull
      - name: Ensure file permissions for mounted volume
        run: |
          mkdir -p ./codegate_volume/certs ./codegate_volume/models ./codegate_volume/db
          chmod -R 777 ./codegate_volume
      - name: Download the CodeGate container image
        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
        with:
          name: ${{ inputs.artifact-name }}
      - name: Load the CodeGate container image
        run: |
          docker load -i image.tar
          echo "Loaded image:"
          docker images
      - name: Download the Qwen2.5-Coder-0.5B-Instruct-GGUF model (llamacpp only)
        if: ${{ matrix.test-provider == 'llamacpp' }} # This is only needed for llamacpp
        run: |
          wget -P ./codegate_volume/models https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF/resolve/main/qwen2.5-coder-0.5b-instruct-q5_k_m.gguf
      - name: Start the CodeGate container
        run: |
          # Get the image name
          DOCKER_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | head -n 1)
          echo "Running container from image: $DOCKER_IMAGE"
          # Run the container
          docker run --name $CODEGATE_CONTAINER_NAME -d --network host \
            -v "$(pwd)"/codegate_volume:/app/codegate_volume \
            -e CODEGATE_APP_LOG_LEVEL=$CODEGATE_LOG_LEVEL \
            -e CODEGATE_OLLAMA_URL=$LOCAL_OLLAMA_URL \
            -e CODEGATE_VLLM_URL=$LOCAL_VLLM_URL \
            -e CODEGATE_DEV_ENV=true \
            --restart unless-stopped $DOCKER_IMAGE
          # Confirm the container started
          echo "Container started:"
          docker ps
          # Verify container is running with correct ports
          docker ps -f name=$CODEGATE_CONTAINER_NAME
          # Check mount configuration
          docker inspect $CODEGATE_CONTAINER_NAME -f '{{ json .Mounts }}' | jq
      - name: Test the healthcheck endpoint
        timeout-minutes: 4
        run: |
          # Check the healthcheck endpoint is available
          while true; do
            echo "Checking for healthcheck endpoint in CodeGate..."
            if curl --silent --fail --get "http://localhost:8989/health" >/dev/null; then
              echo "Healthcheck endpoint is available"
              break
            else
              echo "Healthcheck endpoint not available. Retrying in 5 seconds..."
              sleep 5
          done
          curl "http://localhost:8989/health"
      - name: Ensure certificates are available in the container
        timeout-minutes: 4
        run: |
          # Wait for the cert file to be available in the container
          while true; do
            echo "Checking for $CODEGATE_MOUNT_PATH_CERT_FILE in container $CODEGATE_CONTAINER_NAME..."
            if docker exec "$CODEGATE_CONTAINER_NAME" test -f "$CODEGATE_MOUNT_PATH_CERT_FILE"; then
              echo "Cert file found: $CODEGATE_MOUNT_PATH_CERT_FILE"
              break
            else
              echo "Cert file not found. Retrying in 5 seconds..."
              sleep 5
          done
          # Verify volume contents are accessible
          docker exec $CODEGATE_CONTAINER_NAME ls -la /app/codegate_volume
      - name: Copy and install the CodeGate certificate
        run: |
          docker cp codegate:/app/codegate_volume/certs/ca.crt ./codegate.crt
          sudo cp ./codegate.crt /usr/local/share/ca-certificates/codegate.crt
          sudo update-ca-certificates
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Poetry
        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
        with:
          version: 2.0.1
          virtualenvs-create: true
          virtualenvs-in-project: true
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Install dependencies
        run: poetry install --with dev
      - name: Run the Ollama container (ollama-only)
        if: ${{ matrix.test-provider == 'ollama' }} # This is only needed for Ollama
        timeout-minutes: 15
        env:
          MAX_RETRIES: 3
        run: |
          function check_model_ready() {
            response=$(curl -s http://localhost:11434/api/generate -d '{
              "model": "qwen2.5-coder:1.5b",
              "prompt": "Hello",
              "stream": false
            }' 2>&1)
            if ! echo "$response" | grep -q "error"; then
              return 0  # Success
            return 1  # Not ready/error
          function cleanup_container() {
            docker stop ollama >/dev/null 2>&1 || true
            docker rm ollama >/dev/null 2>&1 || true
            sleep 2
          retry_count=0
          while [ $retry_count -lt $MAX_RETRIES ]; do
            # Cleanup any existing container
            cleanup_container
            echo "Starting Ollama container (Attempt $(($retry_count + 1))/$MAX_RETRIES)"
            docker run -d -v ollama:/root/.ollama --network host --name ollama -e OLLAMA_LOG_LEVEL=DEBUG ollama/ollama
            # Wait for endpoint to be available
            endpoint_wait=0
            while [ $endpoint_wait -lt 30 ]; do
              if curl --silent --fail --get "http://localhost:11434" >/dev/null; then
                echo "Ollama endpoint is available"
                break
              sleep 2
              endpoint_wait=$((endpoint_wait + 1))
            done
            if [ $endpoint_wait -eq 30 ]; then
              echo "Endpoint never became available, retrying..."
              retry_count=$((retry_count + 1))
              continue
            echo "Starting model download/initialization..."
            docker exec -d ollama ollama run qwen2.5-coder:1.5b
            # Monitor container and model status
            monitor_count=0
            while [ $monitor_count -lt 90 ]; do  # 7.5 minutes
              # Check if container is still running
              if ! docker ps | grep -q ollama; then
                echo "Container crashed, logs:"
                docker logs ollama
                retry_count=$((retry_count + 1))
                break
              # Check if model is ready
              if check_model_ready; then
                echo "Model is ready!"
                exit 0  # Success!
              echo "Model not ready yet. Waiting... ($(($monitor_count + 1))/90)"
              sleep 5
              monitor_count=$((monitor_count + 1))
            done
            if [ $monitor_count -eq 90 ]; then
              echo "Timeout waiting for model, container logs:"
              docker logs ollama
              retry_count=$((retry_count + 1))
          done
          echo "Failed after $MAX_RETRIES attempts"
          exit 1
      - name: Build and run the vllm container (vllm-only)
        if: ${{ matrix.test-provider == 'vllm' }} # This is only needed for VLLM
        timeout-minutes: 10
        run: |
          # We clone the VLLM repo and build the container because the CPU-mode container is not published
          git clone https://github.com/vllm-project/vllm.git
          cd vllm
          docker build -f Dockerfile.cpu -t vllm-cpu-env --shm-size=4g .
          docker run -d  --name vllm \
             --network="host" \
             vllm-cpu-env --model Qwen/Qwen2.5-Coder-0.5B-Instruct
          echo -e "\nVerify the vllm container is serving\n"
          docker ps -f name=vllm
          echo "Loop until the endpoint responds successfully"
          while ! curl --silent --fail --get "http://localhost:8000/ping" >/dev/null; do
            echo "Ping not available yet. Retrying in 2 seconds..."
            sleep 2
          done
          echo -e "\nPing is now available!\n"
          echo -e "\nVerify the completions endpoint works\n"
          curl http://localhost:8000/v1/completions -H "Content-Type: application/json"   -d '{
              "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
              "prompt": ["Hello"],
              "max_tokens": 100,
              "temperature": 0
          echo -e "\nVerify the chat/completions endpoint works\n"
          curl -X POST http://localhost:8000/v1/chat/completions \
              -H "Content-Type: application/json" \
              -d '{
                "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
                "messages": [
                  {"role": "system", "content": "You are a coding assistant."},
                  {"role": "user", "content": "Hello"}
                "temperature": 0,
                "max_tokens": 4096,
                "extra_body": {}
          echo -e "\nPrint the vllm container logs\n"
          docker logs vllm
      - name: Tests - ${{ matrix.test-provider }}
        timeout-minutes: 15
        env:
          CODEGATE_PROVIDERS: ${{ matrix.test-provider }}
        run: |
          poetry run --ansi python tests/integration/integration_tests.py
      - name: Logs - CodeGate container
        if: always()
        continue-on-error: true
        run: |
          docker logs $CODEGATE_CONTAINER_NAME
          echo "Models contents:"
          ls -la codegate_volume/models
          docker exec $CODEGATE_CONTAINER_NAME ls -la /app/codegate_volume/models
          echo "Certs contents:"
          ls -la codegate_volume/certs
          docker exec $CODEGATE_CONTAINER_NAME ls -la /app/codegate_volume/certs
          echo "DB contents:"
          ls -la codegate_volume/db
          docker exec $CODEGATE_CONTAINER_NAME ls -la /app/codegate_volume/db
      - name: Logs - vllm container (vllm-only)
        if: always()
        continue-on-error: true
        run: |
          if ${{ matrix.test-provider == 'vllm' }}; then
            docker logs vllm
          else
            echo "Skipping vllm logs, as this is not a VLLM test"
      - name: Logs - Ollama container (ollama-only)
        if: always()
        continue-on-error: true
        run: |
          if ${{ matrix.test-provider == 'ollama' }}; then
            docker logs ollama
          else
            echo "Skipping Ollama logs, as this is not an Ollama test"
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

integration-tests.yml

Latest commit

History

integration-tests.yml

File metadata and controls
