Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,19 @@
"show insert into google.iam.service_accounts;"
],
},
{
"name": "DEBUG show insert into google.bigquery.routines ;",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/stackql",
"args": [
"exec",
"--registry=${input:registryString}",
"--auth=${input:authString}",
"show insert into google.bigquery.routines;"
],
},
{
"name": "DEBUG filtered github.repos.commits... ;",
"type": "go",
Expand Down
85 changes: 85 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
FROM golang:1.18.4-bullseye AS builder

ENV SRC_DIR=/work/stackql/src

ENV BUILD_DIR=/work/stackql/build

RUN mkdir -p ${SRC_DIR} ${BUILD_DIR}

ADD internal ${SRC_DIR}/internal

ADD pkg ${SRC_DIR}/pkg

ADD stackql ${SRC_DIR}/stackql

ADD test ${SRC_DIR}/test

COPY go.mod go.sum ${SRC_DIR}/

RUN cd ${SRC_DIR} && ls && go get -v -t -d ./... && go test --tags "json1" ./... \
&& go build --tags "json1" -o ${BUILD_DIR}/stackql ./stackql


FROM ubuntu:22.04 AS integration

ENV TEST_ROOT_DIR=/opt/test/stackql

RUN mkdir -p ${TEST_ROOT_DIR}/build

COPY --from=builder /work/stackql/build/stackql ${TEST_ROOT_DIR}/build/

COPY --from=builder /work/stackql/src ${TEST_ROOT_DIR}/

RUN apt-get update \
&& apt-get install --yes --no-install-recommends \
default-jdk \
default-jre \
maven \
openssl \
postgresql-client \
python3 \
python3-pip \
&& pip3 install PyYaml robotframework \
&& mvn \
org.apache.maven.plugins:maven-dependency-plugin:3.0.2:copy \
-Dartifact=org.mock-server:mockserver-netty:5.12.0:jar:shaded \
-DoutputDirectory=${TEST_ROOT_DIR}/test/downloads \
&& python3 ${TEST_ROOT_DIR}/test/python/registry-rewrite.py \
&& openssl req -x509 -keyout ${TEST_ROOT_DIR}/test/server/mtls/credentials/pg_server_key.pem -out ${TEST_ROOT_DIR}/test/server/mtls/credentials/pg_server_cert.pem -config ${TEST_ROOT_DIR}/test/server/mtls/openssl.cnf -days 365 \
&& openssl req -x509 -keyout ${TEST_ROOT_DIR}/test/server/mtls/credentials/pg_client_key.pem -out ${TEST_ROOT_DIR}/test/server/mtls/credentials/pg_client_cert.pem -config ${TEST_ROOT_DIR}/test/server/mtls/openssl.cnf -days 365 \
&& openssl req -x509 -keyout ${TEST_ROOT_DIR}/test/server/mtls/credentials/pg_rubbish_key.pem -out ${TEST_ROOT_DIR}/test/server/mtls/credentials/pg_rubbish_cert.pem -config ${TEST_ROOT_DIR}/test/server/mtls/openssl.cnf -days 365 \
&& robot ${TEST_ROOT_DIR}/test/robot/functional

FROM ubuntu:22.04 AS app

ENV TEST_ROOT_DIR=/opt/test/stackql

ARG APP_DIR=/srv/stackql

ARG STACKQL_CFG_ROOT=/opt/stackql

ARG STACKQL_PG_PORT=5477

ENV APP_DIR="${APP_DIR}"

ENV STACKQL_CFG_ROOT="${STACKQL_CFG_ROOT}"

ENV STACKQL_PG_PORT="${STACKQL_PG_PORT}"

RUN mkdir -p ${APP_DIR} ${STACKQL_CFG_ROOT}

ENV PATH="${APP_DIR}:${PATH}"

COPY --from=integration ${TEST_ROOT_DIR}/build/stackql ${APP_DIR}/

RUN apt-get update \
&& apt-get install -y ca-certificates \
&& update-ca-certificates

EXPOSE ${STACKQL_PG_PORT}/tcp

WORKDIR ${STACKQL_CFG_ROOT}

CMD ["/bin/bash", "-c", "stackql --pgsrv.port=${STACKQL_PG_PORT} srv"]


59 changes: 58 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ Providers include:

## Build

### Native Build

Presuming you have all of [the system requirements](#system-requirements-for-local-devlopment-build-and-test), then build/test with cmake:

```bash
Expand All @@ -64,21 +66,76 @@ cmake --build .

Executable `build/stackql` will be created.

### System requirements for local development, build and test
#### System requirements for local development, build and test

- cmake>=3.22.3
- golang>=1.16
- openssl>=1.1.1
- python>=3.5

### Docker Build

```bash
docker build -t stackql:${STACKQL_TAG} -t stackql:latest .
```

## Run

### Native Run

#### Help message

```bash
./build/stackql --help

```

#### Shell

```bash

# Amend STACKQL_AUTH as required, angle bracketed strings must be replaced.
export STACKQL_AUTH='{ "google": { "credentialsfilepath": "</path/to/google/sa-key.json>", "type": "service_account" }, "okta": { "credentialsenvvar": "<OKTA_SECRET_KEY>", "type": "api_key" }, "github": { "type": "basic", "credentialsenvvar": "<GITHUB_CREDS>" }, "aws": { "type": "aws_signing_v4", "credentialsfilepath": "</path/to/aws/secret-key.txt>", "keyID": "<YOUR_AWS_KEY_NOT_A_SECRET>" }, "k8s": { "credentialsenvvar": "<K8S_TOKEN>", "type": "api_key", "valuePrefix": "Bearer " } }'

./build/stackql --auth="${STACKQL_AUTH}" shell

```

### Docker Run

**NOTE**: on some docker versions, the argument `--security-opt seccomp=unconfined` is required as a hack for a [known issue in docker](https://github.com/containers/skopeo/issues/1501).

#### Docker single query

```bash
docker run --rm stackql "bash" "-c" "stackql exec 'show providers;'"
```

#### Docker interactive shell

```bash

export AWS_KEY_ID='<YOUR_AWS_KEY_ID_NOT_A_SECRET>'

export DOCKER_AUTH_STR='{ "google": { "credentialsfilepath": "/opt/stackql/keys/sa-key.json", "type": "service_account" }, "okta": { "credentialsenvvar": "OKTA_SECRET_KEY", "type": "api_key" }, "github": { "type": "basic", "credentialsenvvar": "GITHUB_CREDS" }, "aws": { "type": "aws_signing_v4", "credentialsfilepath": "/opt/stackql/keys/integration/aws-secret-key.txt", "keyID": "'${AWS_KEY_ID}'" }, "k8s": { "credentialsenvvar": "K8S_TOKEN", "type": "api_key", "valuePrefix": "Bearer " } }'

export DOCKER_REG_CFG='{ "url": "https://cdn.statically.io/gh/stackql/stackql-provider-registry/dev/providers" }'

docker run --workdir=/opt/stackql -e "OKTA_SECRET_KEY=${OKTA_SECRET_KEY}" -e "GITHUB_CREDS=${GITHUB_CREDS}" -e "K8S_TOKEN=${K8S_TOKEN}" --volume=$(pwd)/keys:/opt/stackql/keys:ro --volume=$(pwd)/vol/stackql:/opt/stackql/.stackql:rw --rm -it stackql "bash" "-c" "stackql --auth='${DOCKER_AUTH_STR}' --registry='${DOCKER_REG_CFG}' --http.log.enabled shell"
```

#### Docker PG Server

```bash
docker run --rm -P stackql
```

Then, run `docker ps` to ascertain the local port on which the container is serving. Then...

```bash
psql -d "host=127.0.0.1 port=<LOCAL_PORT> user=myuser dbname=mydatabase"
```

## Examples

```
Expand Down
3 changes: 3 additions & 0 deletions internal/stackql/metadatavisitors/requestvisitors.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,9 @@ func (sv *SchemaRequestTemplateVisitor) retrieveTemplateVal(sc *openapistackql.S
propertyLocalSchemaVisitedMap[k] = v
}
if ss != nil && ((ss.Type != "array") || !sv.isVisited(ss.Title, propertyLocalSchemaVisitedMap)) {
if propertyLocalSchemaVisitedMap[ss.Title] {
return "\"{{ " + templateValName + " }}\"", nil
}
propertyLocalSchemaVisitedMap[ss.Title] = true
sv, err := sv.retrieveTemplateVal(ss, templateValName+"."+k, propertyLocalSchemaVisitedMap)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ INSERT INTO google.bigquery.datasets(
)
SELECT
'{{ .values.projectId }}',
'[ { "dataset": { "dataset": { "datasetId": "{{ .values.data__access[0].dataset.dataset.datasetId }}", "projectId": "{{ .values.data__access[0].dataset.dataset.projectId }}" } }, "domain": "{{ .values.data__access[0].domain }}", "groupByEmail": "{{ .values.data__access[0].groupByEmail }}", "iamMember": "{{ .values.data__access[0].iamMember }}", "role": "{{ .values.data__access[0].role }}", "routine": { "datasetId": "{{ .values.data__access[0].routine.datasetId }}", "projectId": "{{ .values.data__access[0].routine.projectId }}", "routineId": "{{ .values.data__access[0].routine.routineId }}" }, "specialGroup": "{{ .values.data__access[0].specialGroup }}", "userByEmail": "{{ .values.data__access[0].userByEmail }}", "view": { "datasetId": "{{ .values.data__access[0].view.datasetId }}", "projectId": "{{ .values.data__access[0].view.projectId }}", "tableId": "{{ .values.data__access[0].view.tableId }}" } } ]',
'[ { "dataset": "{{ .values.data__access[0].dataset }}", "domain": "{{ .values.data__access[0].domain }}", "groupByEmail": "{{ .values.data__access[0].groupByEmail }}", "iamMember": "{{ .values.data__access[0].iamMember }}", "role": "{{ .values.data__access[0].role }}", "routine": "{{ .values.data__access[0].routine }}", "specialGroup": "{{ .values.data__access[0].specialGroup }}", "userByEmail": "{{ .values.data__access[0].userByEmail }}", "view": "{{ .values.data__access[0].view }}" } ]',
'{ "datasetId": "{{ .values.data__datasetReference.datasetId }}", "projectId": "{{ .values.data__datasetReference.projectId }}" }',
'{ "kmsKeyName": "{{ .values.data__defaultEncryptionConfiguration.kmsKeyName }}" }',
'{{ .values.data__defaultPartitionExpirationMs }}',
Expand Down
Loading