Add branding format support to JS and Python SDKs #2045
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Server Test Suite | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ci=${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| audit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10 | |
| - name: Audit NPM Packages | |
| run: | | |
| pnpm dlx audit-ci@^7 --directory apps/api --config apps/api/audit-ci.jsonc | |
| pnpm dlx audit-ci@^7 --directory apps/playwright-service-ts --config apps/playwright-service-ts/audit-ci.jsonc | |
| pnpm dlx audit-ci@^7 --directory apps/test-site --config apps/test-site/audit-ci.jsonc | |
| # Job seems unused, remove once confirmed | |
| # build-images: | |
| # name: Build images | |
| # runs-on: big-runner | |
| # needs: audit | |
| # steps: | |
| # - uses: actions/checkout@v5 | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v3 | |
| # - name: Build API image | |
| # run: | | |
| # docker buildx build \ | |
| # -t firecrawl/firecrawl:latest \ | |
| # --cache-from type=gha \ | |
| # --cache-to type=gha,mode=max \ | |
| # --load \ | |
| # ./apps/api | |
| # - name: Build Playwright image | |
| # run: | | |
| # docker buildx build \ | |
| # -t firecrawl/playwright-service:latest \ | |
| # --cache-from type=gha \ | |
| # --cache-to type=gha,mode=max \ | |
| # --load \ | |
| # ./apps/playwright-service-ts | |
| self-host: | |
| name: Self-hosted environment tests | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| engine: ["playwright", "fetch"] # unsure if we need both of these | |
| proxy: ["proxy", "no-proxy"] # proxy / no-proxy run different tests, keep both | |
| search: ["searxng"] # disabled google for now, should be fine with just Searxng | |
| ai: ["openai"] # AI only should be fine, as it simply adds tests, if non-AI fails, AI will fail. | |
| # legacy matrix: | |
| # engine: ["playwright", "fetch"] | |
| # proxy: ["proxy", "no-proxy"] | |
| # search: ["searxng", "google"] | |
| # ai: ["openai", "no-ai"] | |
| runs-on: big-runner | |
| needs: audit | |
| services: | |
| redis: | |
| image: redis | |
| ports: | |
| - 6379:6379 | |
| env: | |
| HOST: 0.0.0.0 | |
| TEST_SUITE_SELF_HOSTED: true | |
| TEST_SUITE_WEBSITE: ${{ matrix.proxy == 'no-proxy' && 'http://127.0.0.1:4321' || '' }} | |
| OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} | |
| SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }} | |
| PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }} | |
| PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} | |
| PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }} | |
| PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }} | |
| NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres | |
| USE_GO_MARKDOWN_PARSER: true | |
| ALLOW_LOCAL_WEBHOOKS: true | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10 | |
| - run: pnpm config set store-dir ~/.pnpm-store | |
| - run: mkdir -p ~/.pnpm-store | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 22 | |
| cache: pnpm | |
| cache-dependency-path: | | |
| apps/api/pnpm-lock.yaml | |
| apps/test-site/pnpm-lock.yaml | |
| apps/playwright-service-ts/pnpm-lock.yaml | |
| - run: pnpm fetch | |
| working-directory: apps/api | |
| - run: pnpm fetch | |
| working-directory: apps/test-site | |
| - name: Restore native lib | |
| id: napi_restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| apps/api/native/*.node | |
| apps/api/native/index.js | |
| apps/api/native/index.d.ts | |
| # note: this key is not ideal, need to find a better solution | |
| key: ${{ runner.os }}-napi-${{ hashFiles('apps/api/native/Cargo.toml', 'apps/api/native/package.json', 'apps/api/native/src/**') }} | |
| - name: Build native lib | |
| if: steps.napi_restore.outputs.cache-hit != 'true' | |
| run: pnpm install | |
| working-directory: apps/api/native | |
| - name: Cache native lib | |
| if: steps.napi_restore.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| apps/api/native/*.node | |
| apps/api/native/index.js | |
| apps/api/native/index.d.ts | |
| key: ${{ steps.napi_restore.outputs.cache-primary-key }} | |
| - uses: actions/setup-go@v6 | |
| with: | |
| go-version: 1.24 | |
| cache-dependency-path: apps/api/sharedLibs/go-html-to-md/go.sum | |
| - name: Restore Go lib | |
| id: golib_restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so | |
| key: ${{ runner.os }}-golib-${{ hashFiles('apps/api/sharedLibs/go-html-to-md/go.sum', 'apps/api/sharedLibs/go-html-to-md/*.go') }} | |
| - name: Build go-html-to-md | |
| if: steps.golib_restore.outputs.cache-hit != 'true' | |
| run: | | |
| cd apps/api/sharedLibs/go-html-to-md | |
| go mod tidy | |
| go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go | |
| - name: Cache Go lib | |
| if: steps.golib_restore.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so | |
| key: ${{ steps.golib_restore.outputs.cache-primary-key }} | |
| - name: Restore Playwright cache | |
| if: matrix.engine == 'playwright' | |
| id: pw_cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-${{ runner.arch }}-pw-chromium-${{ hashFiles('apps/playwright-service-ts/pnpm-lock.yaml', 'apps/playwright-service-ts/package.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-${{ runner.arch }}-pw-chromium- | |
| - name: Install Playwright dependencies | |
| if: matrix.engine == 'playwright' | |
| run: | | |
| pnpm install | |
| pnpm exec playwright install-deps | |
| pnpm exec playwright install chromium | |
| working-directory: ./apps/playwright-service-ts | |
| - name: Cache Playwright | |
| if: steps.pw_cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ steps.pw_cache.outputs.cache-primary-key }} | |
| - name: Set up SearXNG | |
| if: matrix.search == 'searxng' | |
| run: | | |
| mkdir searxng | |
| echo "use_default_settings: true | |
| search: | |
| formats: [html, json, csv] | |
| server: | |
| secret_key: 'fcsecret'" > searxng/settings.yml | |
| docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng | |
| pnpx wait-on tcp:3434 -t 30s | |
| working-directory: ./ | |
| - name: Install API dependencies | |
| run: pnpm install --frozen-lockfile --ignore-scripts | |
| working-directory: apps/api | |
| env: | |
| npm_config_ignore_scripts: "true" | |
| - name: Install test site dependencies | |
| run: pnpm install --frozen-lockfile | |
| working-directory: apps/test-site | |
| - name: Build + serve test site | |
| run: | | |
| pnpm build | |
| pnpm preview --port 4321 --strictPort --host 127.0.0.1 > test-site.log 2>&1 & | |
| pnpx wait-on tcp:4321 -t 20s | |
| working-directory: apps/test-site | |
| - name: Start playwright | |
| if: matrix.engine == 'playwright' | |
| run: | | |
| pnpm run dev > playwright.log 2>&1 & | |
| pnpx wait-on tcp:3003 -t 15s | |
| working-directory: ./apps/playwright-service-ts | |
| env: | |
| PORT: 3003 | |
| - name: Run Docker Postgres | |
| run: | | |
| docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres | |
| docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest | |
| - name: Run tests | |
| run: pnpm harness pnpm test:snips | |
| working-directory: apps/api | |
| env: | |
| npm_config_ignore_scripts: "true" # required currently to prevent re-building cached native lib | |
| - name: Copy log files | |
| if: always() | |
| run: | | |
| mkdir -p logs | |
| cp ./apps/api/firecrawl.log logs/firecrawl.log | |
| - name: Copy SearXNG logs | |
| if: always() && matrix.search == 'searxng' | |
| run: docker logs searxng > logs/searxng.log && docker kill searxng | |
| - name: Copy Playwright logs | |
| if: always() && matrix.engine == 'playwright' | |
| run: cp ./apps/playwright-service-ts/playwright.log logs/playwright.log | |
| - name: Zip logs | |
| if: always() | |
| run: | | |
| cd logs | |
| zip -r logs.zip ./* | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}) | |
| path: logs/logs.zip | |
| # temp disabled | |
| # prod-test: | |
| # name: Production environment tests | |
| # runs-on: big-runner | |
| # needs: audit | |
| # services: | |
| # redis: | |
| # image: redis | |
| # ports: | |
| # - 6379:6379 | |
| # rabbitmq: | |
| # image: rabbitmq | |
| # ports: | |
| # - 5672:5672 | |
| # env: | |
| # BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} | |
| # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # REDIS_URL: ${{ secrets.REDIS_URL }} | |
| # SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} | |
| # SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} | |
| # SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
| # SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }} | |
| # INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} | |
| # INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} | |
| # INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }} | |
| # TEST_API_KEY: ${{ secrets.TEST_API_KEY }} | |
| # TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }} | |
| # TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }} | |
| # TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }} | |
| # TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }} | |
| # TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }} | |
| # FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }} | |
| # FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }} | |
| # USE_DB_AUTHENTICATION: true | |
| # SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }} | |
| # ENV: ${{ secrets.ENV }} | |
| # RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }} | |
| # RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }} | |
| # RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }} | |
| # GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }} | |
| # GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }} | |
| # GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }} | |
| # GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }} | |
| # GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} | |
| # GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| # VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }} | |
| # USE_GO_MARKDOWN_PARSER: true | |
| # SENTRY_ENVIRONMENT: dev | |
| # IDMUX_URL: ${{ secrets.IDMUX_URL }} | |
| # LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }} | |
| # TEST_SUITE_WEBSITE: ${{ secrets.TEST_SUITE_WEBSITE }} | |
| # NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres | |
| # NUQ_RABBITMQ_URL: amqp://localhost:5672 | |
| # HOST: 0.0.0.0 | |
| # steps: | |
| # - uses: actions/checkout@v5 | |
| # - name: Tailscale | |
| # uses: tailscale/github-action@v4 | |
| # with: | |
| # oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} | |
| # oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} | |
| # tags: tag:ci | |
| # use-cache: 'true' | |
| # - uses: pnpm/action-setup@v4 | |
| # with: | |
| # version: 10 | |
| # - run: pnpm config set store-dir ~/.pnpm-store | |
| # - run: mkdir -p ~/.pnpm-store | |
| # - uses: actions/setup-node@v6 | |
| # with: | |
| # node-version: 22 | |
| # cache: pnpm | |
| # cache-dependency-path: | | |
| # apps/api/pnpm-lock.yaml | |
| # - run: pnpm fetch | |
| # working-directory: apps/api | |
| # - name: Restore native lib | |
| # id: napi_restore | |
| # uses: actions/cache/restore@v4 | |
| # with: | |
| # path: | | |
| # apps/api/native/*.node | |
| # apps/api/native/index.js | |
| # apps/api/native/index.d.ts | |
| # # note: this key is not ideal, need to find a better solution | |
| # key: ${{ runner.os }}-napi-${{ hashFiles('apps/api/native/Cargo.toml', 'apps/api/native/package.json', 'apps/api/native/src/**') }} | |
| # - name: Build native lib | |
| # if: steps.napi_restore.outputs.cache-hit != 'true' | |
| # run: pnpm install | |
| # working-directory: apps/api/native | |
| # - name: Cache native lib | |
| # if: steps.napi_restore.outputs.cache-hit != 'true' | |
| # uses: actions/cache/save@v4 | |
| # with: | |
| # path: | | |
| # apps/api/native/*.node | |
| # apps/api/native/index.js | |
| # apps/api/native/index.d.ts | |
| # key: ${{ steps.napi_restore.outputs.cache-primary-key }} | |
| # - uses: actions/setup-go@v6 | |
| # with: | |
| # go-version: 1.24 | |
| # cache-dependency-path: apps/api/sharedLibs/go-html-to-md/go.sum | |
| # - name: Restore Go lib | |
| # id: golib_restore | |
| # uses: actions/cache/restore@v4 | |
| # with: | |
| # path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so | |
| # key: ${{ runner.os }}-golib-${{ hashFiles('apps/api/sharedLibs/go-html-to-md/go.sum', 'apps/api/sharedLibs/go-html-to-md/*.go') }} | |
| # - name: Build go-html-to-md | |
| # if: steps.golib_restore.outputs.cache-hit != 'true' | |
| # run: | | |
| # cd apps/api/sharedLibs/go-html-to-md | |
| # go mod tidy | |
| # go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go | |
| # - name: Cache Go lib | |
| # if: steps.golib_restore.outputs.cache-hit != 'true' | |
| # uses: actions/cache/save@v4 | |
| # with: | |
| # path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so | |
| # key: ${{ steps.golib_restore.outputs.cache-primary-key }} | |
| # - name: Install API dependencies | |
| # run: pnpm install --frozen-lockfile --ignore-scripts | |
| # working-directory: apps/api | |
| # env: | |
| # npm_config_ignore_scripts: "true" | |
| # - name: Run Docker Postgres | |
| # run: | | |
| # docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres | |
| # docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest | |
| # - name: Run tests | |
| # run: pnpm harness pnpm test:snips | |
| # working-directory: apps/api | |
| # env: | |
| # npm_config_ignore_scripts: "true" # required currently to prevent re-building cached native lib | |
| # - name: Create logs directory | |
| # if: always() | |
| # run: | | |
| # mkdir -p logs | |
| # cp ./apps/api/firecrawl.log logs/firecrawl.log | |
| # cd logs | |
| # zip -r logs.zip ./* | |
| # echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip | |
| # rm logs.zip | |
| # - uses: actions/upload-artifact@v4 | |
| # if: always() | |
| # with: | |
| # name: Encrypted Logs | |
| # path: logs/logs.zip.gpg | |
| # retention-days: 5 |