secureCodeBox · o1oo11oo · Oct 20, 2023 · Sep 7, 2023 · Sep 14, 2023 · Sep 14, 2023
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -129,7 +129,11 @@ jobs:
         working-directory: ./operator
         run: make helm-unit-tests
 
-      - name: Helm-Chart Unit Tests | AutoDiscovery
+      - name: Helm-Chart Unit Tests | AutoDiscovery Cloud AWS
+        working-directory: ./auto-discovery/cloud-aws
+        run: make helm-unit-tests
+
+      - name: Helm-Chart Unit Tests | AutoDiscovery Kubernetes
         working-directory: ./auto-discovery/kubernetes
         run: make helm-unit-tests
 
@@ -203,7 +207,7 @@ jobs:
           path: ./operator/${{ matrix.component }}.tar
           retention-days: 1
 
-  # ---- Build Stage | AutoDiscovery ----
+  # ---- Build Stage | AutoDiscovery | Kubernetes ----
 
   auto-discovery-kubernetes:
     name: "AutoDiscovery | Kubernetes"
@@ -242,7 +246,7 @@ jobs:
           path: ./auto-discovery/kubernetes/auto-discovery-kubernetes.tar
           retention-days: 1
 
-  # ---- Build Stage | AutoDiscovery | PullSecretExtractor ----
+  # ---- Build Stage | AutoDiscovery | Kubernetes | PullSecretExtractor ----
   auto-discovery-kubernetes-secret-extraction-container:
     name: "Autodiscovery | Kubernetes | SecretExtractionInitContainer"
     runs-on: ubuntu-22.04
@@ -335,6 +339,44 @@ jobs:
         run: |
           make integration-test
 
+  # ---- Build Stage | AutoDiscovery | Cloud | AWS ----
+  auto-discovery-cloud-aws:
+    name: "AutoDiscovery | Cloud | AWS"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Go Setup
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Lint Go Code
+        working-directory: ./auto-discovery/cloud-aws
+        run: |
+          go fmt ./...
+          go vet ./...
+
+      - name: Test
+        working-directory: ./auto-discovery/cloud-aws/
+        run: make test
+
+      - name: Build Container Image
+        working-directory: ./auto-discovery/cloud-aws/
+        run: make docker-build
+
+      - name: Export Container Image
+        working-directory: ./auto-discovery/cloud-aws/
+        run: make docker-export
+
+      - name: Upload Image As Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: auto-discovery-cloud-aws-image
+          path: ./auto-discovery/cloud-aws/auto-discovery-cloud-aws.tar
+          retention-days: 1
+
   # ---- Build Stage | SDK Matrix ----
 
   sdk:

diff --git a/Makefile b/Makefile
@@ -56,6 +56,7 @@ operator-docs: ## Generate documentation for the operator.
 
 .PHONY: auto-discovery-docs
 auto-discovery-docs: ## Generate documentation for the auto-discovery.
+	"$(BIN_DIR)/generate-helm-docs.sh" --operator "$(AUTO_DISCOVERY_DIR)/cloud-aws/Chart.yaml" "$(HELM_DOCS_DIR)"
 	"$(BIN_DIR)/generate-helm-docs.sh" --operator "$(AUTO_DISCOVERY_DIR)/kubernetes/Chart.yaml" "$(HELM_DOCS_DIR)"
 
 .PHONY: demo-target-docs

diff --git a/auto-discovery/cloud-aws/.dockerignore b/auto-discovery/cloud-aws/.dockerignore
@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: the secureCodeBox authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore all files which are not go type
+*
+!**/*.go
+!**/*.mod
+!**/*.sum
diff --git a/auto-discovery/cloud-aws/.gitignore b/auto-discovery/cloud-aws/.gitignore
@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: the secureCodeBox authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin
+testbin/*
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# editor and IDE paraphernalia
+.idea
+*.swp
+*.swo
+*~
+
+# Debug files
+debug.env
diff --git a/auto-discovery/cloud-aws/.helm-docs.gotmpl b/auto-discovery/cloud-aws/.helm-docs.gotmpl
@@ -0,0 +1,196 @@
+{{- /*
+SPDX-FileCopyrightText: the secureCodeBox authors
+
+SPDX-License-Identifier: Apache-2.0
+*/ -}}
+
+{{- define "extra.docsSection" -}}
+---
+title: "secreCodeBox AutoDiscovery for AWS"
+category: "core"
+type: "AutoDiscovery"
+state: "developing"
+appVersion: "{{ template "chart.appVersion" . }}"
+usecase: "secureCodeBox AutoDiscovery for AWS discovers and starts scans for containers running in AWS ECS."
+---
+
+![auto-discovery logo](https://www.securecodebox.io/img/Logo_Color.svg)
+
+The secureCodeBox _AutoDiscovery_ is running on kubernetes (K8S) and is an optional component of the complete secureCodeBox stack.
+The AWS AutoDiscovery needs to be deployed along side the secureCodeBox Operator. It monitors security relevant resources inside the AWS Elastic Container Service and automatically creates scans to continuously monitor security aspects of the resources.
+
+<!-- end -->
+
+The AutoDiscovery controller will automatically detect these new resources (containers) and start secureCodeBox _scans_ for them:
+
+1. An image scan scanning for vulnerable libraries in the docker / container image of the deployment. (Using trivy)
+2. An image scan to create a Software Bill of Materials (SBOM) for the container. (Using trivy)
+
+The AutoDiscovery automatically tracks the lifecycle of the ECS containers and will automatically start new scans for new application versions.
+The scan type can be defined with `config.kubernetes.scanConfigs[0].scanType`.
+{{- end }}
+
+{{- define "extra.dockerDeploymentSection" -}}
+{{- end }}
+
+{{- define "extra.chartAboutSection" -}}
+> [!NOTE]
+> Even though the AWS Cloud AutoDiscovery monitors resources in AWS (currently only ECS), the AutoDiscovery itself is running in a Kubernetes cluster as part of the _secureCodeBox_.
+> While the _secureCodeBox_ can be deployed to AWS, for example by using the Elastic Kubernetes Service, it also works from anywhere outside of AWS.
+
+> [!WARNING]
+> The AWS Cloud AutoDiscovery is in an __early prerelease state__.
+> There is no initial state synchronization, no reordering for out-of-order events from eventbridge (which can rarely happen), and no retry when Kubernetes errors are encountered.
+> This might lead to the local state and the AWS state diverging and resources not getting scanned.
+
+## Prerequisites
+
+The AWS AutoDiscovery detects changes in AWS by reading change events from an SQS queue.
+To make sure these events are available there, EventBridge rules for the monitored resources need to be created.
+The queue needs to be a FIFO queue, because the AWS AutoDiscovery assumes the events are delivered in order.
+
+These instructions use the [AWS CLI](https://aws.amazon.com/cli/) to create the necessary resources and [jq](https://jqlang.github.io/jq/) to parse the responses.
+See the [AWS docs](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html) for configuration and authentication options.
+
+### Connect to AWS
+
+All values are optional, as long as the connection to AWS works.
+These values will be picked up by the AutoDiscovery, either through the Kubernetes secret defined below, by helm install or by the AutoDiscovery itself if it is running locally.
+For setting up the necessary resources on AWS you can also use different connection options, SSO for example.
+
+```bash
+# Connection options for the CLI
+export AWS_ACCESS_KEY_ID="<key id>"
+export AWS_SECRET_ACCESS_KEY="<access key>"
+export AWS_SESSION_TOKEN="<session token>"
+export AWS_REGION="<region>"
+
+# Create the Kubernetes secret for the service to read the credentials from (use -n to configure another namespace)
+# The names of the secret and the keys can be configured through helm values
+kubectl create secret generic aws-credentials --from-literal=aws-access-key-id=$AWS_ACCESS_KEY_ID --from-literal=aws-secret-access-key=$AWS_SECRET_ACCESS_KEY --from-literal=aws-session-token=$AWS_SESSION_TOKEN
+```
+
+### Create the rule and queue
+
+This sets up a high throughput FIFO queue, because the AutoDiscovery service assumes the messages will be delivered in order.
+EventBridge does not guarantee in order delivery though, so in rare cases the AutoDiscovery might delete scans too early or keep them around for too long.
+
+```bash
+# Configure the name both the queue and rule are going to use
+name="secureCodeBox-autodiscovery-events"
+
+# Create the queue
+queue_url="$(aws sqs create-queue --queue-name "${name}.fifo" --attributes FifoQueue=true,ReceiveMessageWaitTimeSeconds=20,ContentBasedDeduplication=true,DeduplicationScope=messageGroup,FifoThroughputLimit=perMessageGroupId --tags "SCB-AutoDiscovery=" | jq -r ".QueueUrl")"
+queue_arn="$(aws sqs get-queue-attributes --queue-url $queue_url --attribute-names QueueArn | jq -r ".Attributes.QueueArn")"
+
+# Create the rule
+rule_arn="$(aws events put-rule --name "${name}" --description "Gather events for the secureCodeBox AWS AutoDiscovery" --state ENABLED --tags "Key=SCB-AutoDiscovery,Value=" --event-pattern '{
+  "source": ["aws.ecs"],
+  "detail-type": ["ECS Task State Change", "ECS Container Instance State Change", "ECS Deployment State Change"]
+}' | jq -r ".RuleArn")"
+
+# Allow eventbridge to send messages to the queue
+# Depending on the way you give access to the queue to the AutoDiscovery service, you might also need to give permissions to that
+# The AutoDiscovery requires the sqs:ReceiveMessage and sqs:DeleteMessage permissions
+timestamp="$(date +%s)"
+policy='{
+  "Id": "Policy'"${timestamp}"'",
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "Stmt'"${timestamp}"'",
+      "Action": "sqs:SendMessage",
+      "Effect": "Allow",
+      "Resource": "'"${queue_arn}"'",
+      "Principal": {
+        "Service": "events.amazonaws.com"
+      },
+      "Condition": {
+        "ArnEquals": {
+          "aws:SourceArn": "'"${rule_arn}"'"
+        }
+      }
+    }
+  ]
+}'
+sqs_policy="$(echo $policy | jq -c -j | jq -R -s)"
+aws sqs set-queue-attributes --queue-url $queue_url --attributes '{"Policy": '"${sqs_policy}"'}'
+
+# Send the messages to the queue, if it shows "FailedEntryCount": 0 it worked
+aws --no-cli-pager events put-targets --rule $name --targets '[{"Id": "Id'"$(uuidgen | tr '[:upper:]' '[:lower:]')"'", "Arn": "'"${queue_arn}"'", "SqsParameters": {"MessageGroupId": "secureCodeBox-AutoDiscovery"}}]'
+
+# Set up variables for later use
+export SQS_QUEUE_URL="${queue_url}"
+```
+{{- end }}
+
+{{- define "extra.scannerConfigurationSection" -}}
+To directly deploy the auto-discovery-cloud-aws chart with the options for AWS configured, you can pass additional config values to helm:
+
+```bash
+# Install HelmChart (use -n to configure another namespace)
+helm upgrade --install {{ template "chart.name" . }} secureCodeBox/{{ template "chart.name" . }} --set="config.aws.queueUrl=${SQS_QUEUE_URL}" --set="config.aws.region=${AWS_REGION}"
+```
+{{- end }}
+
+{{- define "extra.chartConfigurationSection" -}}
+## Additional configuration
+
+### Installing the ScanType
+
+The AutoDiscovery creates _ScheduledScans_ for each resource it tracks.
+For these to work you need to install the correct scan types into the same namespace the AutoDiscovery is running in.
+The AutoDiscovery will create the scans in its own namespace.
+
+### Optional: In- / Excluding Resources from the AutoDiscovery
+
+The AutoDiscovery will create scans for everything it sees in the queue.
+You can limit the messages to specific ECS clusters by adding a filter to the EventBridge rule.
+The following command will either create or update the rule:
+
+```bash
+aws events put-rule --name "${name}" --description "Gather events for the secureCodeBox AWS AutoDiscovery" --state ENABLED --event-pattern '{
+  "source": ["aws.ecs"],
+  "detail-type": ["ECS Task State Change", "ECS Container Instance State Change", "ECS Deployment State Change"],
+  "detail": {
+    "clusterArn": ["<arn of the cluster you want to include>"]
+  }
+}'
+```
+
+## AWS Costs
+
+As of October 2023, the AWS AutoDiscovery incurs no or very little cost on AWS.
+EventBridge rules for internal events generated by AWS, which is all the AutoDiscovery uses, are [free of charge](https://aws.amazon.com/eventbridge/pricing/).
+The AutoDiscovery generates approximately 130,000 requests to SQS each month to poll the queue every 20 seconds using [long polling](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-short-and-long-polling.html).
+In addition to that there are two requests for each message, one to add it to the queue and one to delete it.
+The first million requests to SQS is free, after that [each million costs around $0.50](https://aws.amazon.com/sqs/pricing/) depending on the region.
+Data transfers are priced at $0.09 per 10TB.
+
+This means the AWS AutoDiscovery should either be free or cheaper than $1/month even for larger or busier setups.
+{{- end }}
+
+{{- define "extra.scannerLinksSection" -}}
+## Development
+
+### Run the AutoDiscovery locally
+
+To avoid having to build & deploy the AutoDiscovery every time you make a code change you can run it locally.
+It automatically connects to your current cluster configured in your kube config.
+To connect to AWS for development, you can either change the settings in `auto-discovery-cloud-aws-config.yaml` or set the `SQS_QUEUE_URL` environment variable and your preferred way of connecting to AWS.
+You can for example set `AWS_REGION` and `AWS_PROFILE` for using SSO, set `AWS_SDK_LOAD_CONFIG=true` to load everything from `.aws/` or set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` directly.
+You can check the [AWS SDK](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) docs for more options and details.
+```bash
+make run
+```
+
+### Running the tests
+
+```bash
+# execute the tests locally
+make test
+
+# view the test coverage
+go tool cover -html=cover.out
+```
+{{- end }}
diff --git a/auto-discovery/cloud-aws/.helmignore b/auto-discovery/cloud-aws/.helmignore
@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: the secureCodeBox authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# Debug files
+debug.env
+
+*.tar
+auto-discovery-cloud-aws-config.yaml
+bin/
+cmd/
+cover.out
+Dockerfile
+go.mod
+go.sum
+go.sum.license
+Makefile
+pkg/
diff --git a/auto-discovery/cloud-aws/.vscode/launch.json b/auto-discovery/cloud-aws/.vscode/launch.json
@@ -0,0 +1,24 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            // Make sure the AWS connection is configured either as env or in the yaml file
+            "name": "Debug Main",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceFolder}/cmd/service/main.go",
+            "args": ["--config", "${workspaceFolder}/auto-discovery-cloud-aws-config.yaml"],
+        },
+        {
+            "name": "Debug Integration Tests",
+            "type": "go",
+            "request": "launch",
+            "mode": "test",
+            "program": "${workspaceFolder}/cmd/service",
+            "args": ["-test.v"],
+            "preLaunchTask": "debug.env",
+            "envFile": "${workspaceFolder}/debug.env",
+        }
+    ]
+}
diff --git a/auto-discovery/cloud-aws/.vscode/launch.json.license b/auto-discovery/cloud-aws/.vscode/launch.json.license
@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: the secureCodeBox authors
+
+SPDX-License-Identifier: Apache-2.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		SPDX-FileCopyrightText: the secureCodeBox authors

		SPDX-License-Identifier: Apache-2.0