Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,9 @@ jobs:
needs:
- linuxbuild
- test_python_package_build
permissions:
id-token: write
contents: read
runs-on: ubuntu-22-04-m
strategy:
matrix:
Expand Down Expand Up @@ -759,6 +762,71 @@ jobs:
run: |
cat ./test/robot/foreign-integration-traffic-lights/tmp/* || true

- name: ID Fed - configure AWS credentials (OIDC)
if: ((startsWith(github.ref_name, 'build-traffic-lights') && github.ref_type == 'tag') || (github.repository == 'stackql/stackql' && github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.registry == 'test/registry'
continue-on-error: true
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.STACKQL_ID_FED_AWS_ROLE_ARN }}
aws-region: us-east-1

- name: ID Fed - authenticate to Google Cloud (OIDC)
id: gcp_auth
if: ((startsWith(github.ref_name, 'build-traffic-lights') && github.ref_type == 'tag') || (github.repository == 'stackql/stackql' && github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.registry == 'test/registry'
continue-on-error: true
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.STACKQL_ID_FED_GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.STACKQL_ID_FED_GCP_SERVICE_ACCOUNT }}
token_format: 'access_token'

- name: ID Fed - login to Azure (OIDC)
if: ((startsWith(github.ref_name, 'build-traffic-lights') && github.ref_type == 'tag') || (github.repository == 'stackql/stackql' && github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.registry == 'test/registry'
continue-on-error: true
uses: azure/login@v2
with:
client-id: ${{ secrets.STACKQL_ID_FED_AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.STACKQL_ID_FED_AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_INTEGRATION_TESTING_SUB_ID }}

- name: Run id-fed traffic light robot integration tests
if: ((startsWith(github.ref_name, 'build-traffic-lights') && github.ref_type == 'tag') || (github.repository == 'stackql/stackql' && github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.registry == 'test/registry'
env:
PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python'
AZURE_TARGET_SUBSCRIPTION_ID: ${{ secrets.AZURE_INTEGRATION_TESTING_SUB_ID }}
GCP_ACCESS_TOKEN: ${{ steps.gcp_auth.outputs.access_token }}
GODEBUG: netdns=go
run: |
echo "## Stray flask apps to be killed before robot tests ##"
pgrep -f flask | xargs kill -9 || true
echo "## End ##"
if python cicd/python/build.py --robot-test-id-fed-traffic-lights-integration; then
echo "✅ ID-FED traffic light robot integration tests **all** passed"
else
rv="$?"
echo "🟡 **some** ID-FED traffic light robot integration tests failed code = $rv"
fi
{
echo "ID_FED_TRAFFIC_LIGHTS_COMPLETED=true"
} >> "$GITHUB_ENV"

- name: Output from id-fed traffic lights integration tests
if: env.ID_FED_TRAFFIC_LIGHTS_COMPLETED == 'true'
run: |
cat ./test/robot/reports-id-fed-traffic-lights/output.xml || true

- name: Output from id-fed traffic lights tmp dir
if: env.ID_FED_TRAFFIC_LIGHTS_COMPLETED == 'true'
run: |
cat ./test/robot/id-fed-traffic-lights/tmp/* || true

- name: ID Fed - clear cloud env vars before downstream tests
if: always()
run: |
for v in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_REGION AWS_DEFAULT_REGION AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE GOOGLE_APPLICATION_CREDENTIALS GOOGLE_GHA_CREDS_PATH CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE CLOUDSDK_CORE_PROJECT CLOUDSDK_PROJECT GCLOUD_PROJECT GCP_PROJECT GCP_ACCESS_TOKEN AZURE_HTTP_USER_AGENT AZUREPS_HOST_ENVIRONMENT; do
echo "${v}=" >> "$GITHUB_ENV"
done

- name: Generate hosts file and nginx materials
env:
BUILDCOMMITSHA: ${{github.sha}}
Expand Down
15 changes: 15 additions & 0 deletions cicd/python/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,16 @@ def run_robot_foreign_integration_traffic_lights_tests_stackql(*args, **kwargs)
shell=True
)

def run_robot_id_fed_integration_traffic_lights_tests_stackql(*args, **kwargs) -> int:
variables = ' '.join([f'--variable {key}:{sanitise_val(value)} ' for key, value in kwargs.get("variables", {}).items()])
return subprocess.call(
'robot '
f'{variables} '
'-d test/robot/reports-id-fed-traffic-lights '
'test/robot/id-fed-traffic-lights',
shell=True
)

def main():
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', action='store_true')
Expand All @@ -101,6 +111,7 @@ def main():
parser.add_argument('--robot-test', action='store_true')
parser.add_argument('--robot-test-integration', action='store_true')
parser.add_argument('--robot-test-traffic-lights-integration', action='store_true')
parser.add_argument('--robot-test-id-fed-traffic-lights-integration', action='store_true')
parser.add_argument('--robot-test-foreign-traffic-lights-integration', action='store_true')
parser.add_argument('--config', type=json.loads, default={})
args = parser.parse_args()
Expand Down Expand Up @@ -133,6 +144,10 @@ def main():
ret_code = run_robot_foreign_integration_traffic_lights_tests_stackql(**args.config)
if ret_code != 0:
exit(ret_code)
if args.robot_test_id_fed_traffic_lights_integration:
ret_code = run_robot_id_fed_integration_traffic_lights_tests_stackql(**args.config)
if ret_code != 0:
exit(ret_code)
exit(ret_code)


Expand Down
25 changes: 25 additions & 0 deletions docs/auth.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,28 @@ source cicd/vol/vendor-secrets/foreign_to_stackql_user.sh
stackql --auth '{"aws":{"type":"aws_assume_role","keyIDenvvar":"AWS_ACCESS_KEY_ID","credentialsenvvar":"AWS_SECRET_ACCESS_KEY","aws_role_arn":"'"${STACKQL_AUDIT_ROLE_ARN}"'"}}' shell
```

## OIDC from github

GHA → cloud federation via each provider's official action; stackql then uses its standard auth types — **no federation-specific `--auth` string needed**.

Workflow needs `permissions: id-token: write`, then per cloud:

**AWS** — [GitHub docs](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services). IAM OIDC provider + role; `aws-actions/configure-aws-credentials@v4` with `role-to-assume`. Session token (`AWS_SESSION_TOKEN`) is picked up from env automatically — no extra field needed.
```
--auth '{"aws":{"type":"aws_signing_v4","keyIDenvvar":"AWS_ACCESS_KEY_ID","credentialsenvvar":"AWS_SECRET_ACCESS_KEY"}}'
```

**GCP** — [GitHub docs](https://github.com/google-github-actions/auth#setup). Workload Identity **Pool** + Provider, SA bound via **both** `roles/iam.workloadIdentityUser` **and** `roles/iam.serviceAccountTokenCreator` (the access-token mint path needs Token Creator on top of WIU); `google-github-actions/auth@v2` with `workload_identity_provider` + `service_account` + `token_format: 'access_token'`. Capture the action's `access_token` output into an env var (e.g. `GCP_ACCESS_TOKEN`) for the stackql step. The action's default `external_account` credentials file isn't consumable by stackql's `service_account` type.
```
--auth '{"google":{"type":"bearer","credentialsenvvar":"GCP_ACCESS_TOKEN"}}'
```

**Azure** — [action docs](https://github.com/Azure/login#login-with-openid-connect-oidc-recommended). Entra app + federated credential (one per org — Azure won't accept `repository_owner` matching); `azure/login@v2` with client/tenant/subscription IDs.
```
--auth '{"azure":{"type":"azure_default"}}'
```

### Gotchas
- GCP: use **Workload** Identity Federation (project-scoped), not Workforce. Impersonation via `token_format: access_token` needs **both** `roles/iam.workloadIdentityUser` and `roles/iam.serviceAccountTokenCreator` on the SA. GH secret = full provider resource name (`projects/.../providers/...`).
- Azure: federated credentials for GHA disallow `repository_owner` matching → create one credential per org, or regex on `claims['sub']`.

4 changes: 4 additions & 0 deletions test/robot/id-fed-traffic-lights/__init__.robot
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
*** Settings ***
Resource ${CURDIR}/stackql.resource


21 changes: 21 additions & 0 deletions test/robot/id-fed-traffic-lights/stackql.resource
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
*** Variables ***
${LOCAL_LIB_HOME} ${CURDIR}${/}..${/}..${/}python
${REPOSITORY_ROOT} ${CURDIR}${/}..${/}..${/}..
${EXECUTION_PLATFORM} native # to be overridden from command line, eg "docker"
${SQL_BACKEND} sqlite_embedded # to be overridden from command line, eg "postgres_tcp"
${IS_WSL} false # to be overridden from command line, with string "true"
${USE_STACKQL_PREINSTALLED} false # to be overridden from command line, with string "true"
${SUNDRY_CONFIG} {} # to be overridden from command line, with string value
${STACKQL_INTERFACE_LIBRARY} stackql_test_tooling.StackQLInterfaces
${CLOUD_INTEGRATION_LIBRARY} stackql_test_tooling.CloudIntegration

*** Settings ***
Library Process
Library OperatingSystem
Variables ${LOCAL_LIB_HOME}/stackql_test_tooling/stackql_context.py ${REPOSITORY_ROOT} ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${USE_STACKQL_PREINSTALLED}
... ${SUNDRY_CONFIG}
Library Process
Library OperatingSystem
Library String
Library ${STACKQL_INTERFACE_LIBRARY} ${EXECUTION_PLATFORM} ${SQL_BACKEND}
Library ${CLOUD_INTEGRATION_LIBRARY}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@



*** Settings ***
Resource ${CURDIR}/stackql.resource

*** Test Cases ***

ID Fed AWS S3 Buckets List
Sleep 2s
${awsAuthCfg} = Catenate
... { "aws": { "type":"aws_signing_v4", "keyIDenvvar": "AWS_ACCESS_KEY_ID", "credentialsenvvar": "AWS_SECRET_ACCESS_KEY" } }
${bucketsListQuery} = Catenate
... select * from aws.pseudo_s3.buckets_list_only where region = 'ap-southeast-2';
${result} = Run Process
... ${STACKQL_EXE}
... \-\-auth
... ${awsAuthCfg}
... \-\-registry
... { "url": "file://${REPOSITORY_ROOT}/test/registry", "localDocRoot": "${REPOSITORY_ROOT}/test/registry", "verifyConfig": { "nopVerify": true } }
... exec
... ${bucketsListQuery}
... cwd=${REPOSITORY_ROOT}
... stdout=${CURDIR}/tmp/ID-Fed-AWS-S3-Buckets-List.tmp
... stderr=${CURDIR}/tmp/ID-Fed-AWS-S3-Buckets-List-stderr.tmp
Should Be Equal As Integers ${result.rc} 0
Should Be Empty ${result.stderr}
Should Contain ${result.stdout} stackql\-trial\-bucket\-02


ID Fed Azure VNETs List
Sleep 2s
${azureTargetSubscription} = OperatingSystem.Get Environment Variable AZURE_TARGET_SUBSCRIPTION_ID
Should Not Be Empty ${azureTargetSubscription}
${azureAuthCfg} = Catenate
... { "azure": { "type":"azure_default" } }
${bucketsListQuery} = Catenate
... select location, name from azure.network.virtual_networks where subscriptionId = '${azureTargetSubscription}';
${result} = Run Process
... ${STACKQL_EXE}
... \-\-auth
... ${azureAuthCfg}
... \-\-registry
... { "url": "file://${REPOSITORY_ROOT}/test/registry", "localDocRoot": "${REPOSITORY_ROOT}/test/registry", "verifyConfig": { "nopVerify": true } }
... exec
... ${bucketsListQuery}
... cwd=${REPOSITORY_ROOT}
... stdout=${CURDIR}/tmp/ID-Fed-Azure-VNETs-List.tmp
... stderr=${CURDIR}/tmp/ID-Fed-Azure-VNETs-List-stderr.tmp
Should Be Equal As Integers ${result.rc} 0
Should Be Empty ${result.stderr}
Should Contain ${result.stdout} inspector\-network


ID Fed Google Buckets List
Sleep 2s
${gcpAccessToken} = OperatingSystem.Get Environment Variable GCP_ACCESS_TOKEN
Should Not Be Empty ${gcpAccessToken}
${gcpAuthCfg} = Catenate
... { "google": { "type":"bearer", "credentialsenvvar": "GCP_ACCESS_TOKEN" } }
${bucketsListQuery} = Catenate
... select location, name from google.storage.buckets where project = 'stackql-demo';
${result} = Run Process
... ${STACKQL_EXE}
... \-\-auth
... ${gcpAuthCfg}
... \-\-registry
... { "url": "file://${REPOSITORY_ROOT}/test/registry", "localDocRoot": "${REPOSITORY_ROOT}/test/registry", "verifyConfig": { "nopVerify": true } }
... exec
... ${bucketsListQuery}
... cwd=${REPOSITORY_ROOT}
... stdout=${CURDIR}/tmp/ID-Fed-Google-Buckets-List.tmp
... stderr=${CURDIR}/tmp/ID-Fed-Google-Buckets-List-stderr.tmp
Should Be Equal As Integers ${result.rc} 0
Should Be Empty ${result.stderr}
Should Contain ${result.stdout} stackql\-demo\-bucket
2 changes: 2 additions & 0 deletions test/robot/id-fed-traffic-lights/tmp/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*
!.gitignore
2 changes: 2 additions & 0 deletions test/robot/reports-id-fed-traffic-lights/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*
!.gitignore
Loading