secureCodeBox · rfelber · Nov 25, 2020 · Nov 23, 2020 · Nov 23, 2020 · Nov 25, 2020
diff --git a/hooks/persistence-elastic/README.md b/hooks/persistence-elastic/README.md
@@ -19,6 +19,12 @@ Installing the Elasticsearch persistenceProvider hook will add a _ReadOnly Hook_
 helm upgrade --install elkh secureCodeBox/persistence-elastic
 ```
 
+## Elasticsearch Indexing
+
+For the elasticsearch `indexSuffix` you can provide a date format pattern. We use [Luxon](https://moment.github.io/luxon/) to format the date. So checkout
+the [Luxon documentation](https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) to see what kind of format patterns you can use for the
+`indexSuffix`. Default pattern is `yyyy-MM-dd`
+
 ## Chart Configuration
 
 | Key | Type | Default | Description |
@@ -40,7 +46,9 @@ helm upgrade --install elkh secureCodeBox/persistence-elastic
 | image.repository | string | `"docker.io/securecodebox/persistence-elastic"` | Image repository for the dashboard importer job |
 | image.tag | string | defaults to the charts version | Image tag for the dashboard importer job |
 | imagePullSecrets | list | `[]` |  |
+| indexAppendNamespace | bool | `true` | Define if the name of the namespace where this hook is deployed to must be added to the index name. The namespace can be used to separate index by tenants (namespaces). |
 | indexPrefix | string | `"scbv2"` | Define a specific index prefix used for all elasticsearch indices. |
+| indexSuffix | string | `"“yyyy-MM-dd”"` | Define a specific index suffix based on date pattern (YEAR (yyyy), MONTH (yyyy-MM), WEEK (yyyy-'W'W), DATE (yyyy-MM-dd)). We use Luxon for date formatting (https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) |
 | kibana | object | `{"enabled":true}` | Configures included Elasticsearch subchart |
 | kibana.enabled | bool | `true` | Enable if you want to deploy an kibana service (see: https://github.com/elastic/helm-charts/tree/master/kibana) |
 | nameOverride | string | `""` |  |
@@ -50,4 +58,4 @@ helm upgrade --install elkh secureCodeBox/persistence-elastic
 | securityContext | object | `{}` |  |
 | tolerations | list | `[]` |  |
 
-[elastic.io]: https://www.elastic.co/products/elasticsearch
+[elastic.io]: https://www.elastic.co/products/elasticsearch
diff --git a/hooks/persistence-elastic/README.md.gotmpl b/hooks/persistence-elastic/README.md.gotmpl
@@ -19,9 +19,16 @@ Installing the Elasticsearch persistenceProvider hook will add a _ReadOnly Hook_
 helm upgrade --install elkh secureCodeBox/persistence-elastic
 ```
 
+## Elasticsearch Indexing
+
+For the elasticsearch `indexSuffix` you can provide a date format pattern. We use [Luxon](https://moment.github.io/luxon/) to format the date. So checkout
+the [Luxon documentation](https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) to see what kind of format patterns you can use for the
+`indexSuffix`. Default pattern is `yyyy-MM-dd`
+
+
 ## Chart Configuration
 
 {{ template "chart.valuesTable" . }}
 
 
-[elastic.io]: https://www.elastic.co/products/elasticsearch
+[elastic.io]: https://www.elastic.co/products/elasticsearch
diff --git a/hooks/persistence-elastic/hook.js b/hooks/persistence-elastic/hook.js
@@ -3,13 +3,17 @@ const { Client } = require("@elastic/elasticsearch");
 const flatMap = require("lodash.flatmap");
 const chunk = require("lodash.chunk");
 
+const { DateTime } = require("luxon");
+
 const authParams = {};
 
 const username = process.env["ELASTICSEARCH_USERNAME"];
 const password = process.env["ELASTICSEARCH_PASSWORD"];
 const apiKeyId = process.env["ELASTICSEARCH_APIKEY_ID"];
 const apiKey = process.env["ELASTICSEARCH_APIKEY"];
 
+const defaultDateFormat = 'yyyy-MM-dd';
+
 if (apiKeyId && apiKey) {
   console.log("Using API Key for Authentication");
   authParams.auth = {
@@ -39,6 +43,8 @@ async function handle({
   now = new Date(),
   tenant = process.env["NAMESPACE"],
   indexPrefix = process.env["ELASTICSEARCH_INDEX_PREFIX"] || "scbv2",
+  indexSuffix = process.env["ELASTICSEARCH_INDEX_SUFFIX"] || defaultDateFormat,
+  appendNamespace = process.env['ELASTICSEARCH_INDEX_APPEND_NAMESPACE'] || false
 }) {
   const findings = await getFindings();
 
@@ -47,8 +53,8 @@ async function handle({
     `Using Elasticsearch Instance at "${process.env["ELASTICSEARCH_ADDRESS"]}"`
   );
 
-  const timeStamp = now.toISOString().substr(0, 10);
-  const indexName = `${indexPrefix}_${tenant}_${timeStamp}`;
+  let indexName = appendNamespace ? `${indexPrefix}_${tenant}_` : `${indexPrefix}_`;
+  indexName += DateTime.fromJSDate(now).toFormat(indexSuffix)
 
   await client.indices.create(
     {

diff --git a/hooks/persistence-elastic/hook.test.js b/hooks/persistence-elastic/hook.test.js
@@ -5,33 +5,33 @@ beforeEach(() => {
   elasticClient.bulk.mockClear();
 });
 
+const scan = {
+  metadata: {
+    uid: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
+    name: "demo-scan",
+    labels: {
+      company: "iteratec",
+    },
+  },
+  spec: {
+    scanType: "Nmap",
+    parameters: ["-Pn", "localhost"],
+  },
+};
+
+const testDate = new Date('2020-11-11');
+
 test("should only send scan summary document if no findings are passing in", async () => {
   const findings = [];
 
   const getFindings = async () => findings;
 
-  const scan = {
-    metadata: {
-      uid: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
-      name: "demo-scan",
-      labels: {
-        company: "iteratec",
-      },
-    },
-    spec: {
-      scanType: "Nmap",
-      parameters: ["-Pn", "localhost"],
-    },
-  };
-
-  const now = new Date();
-
-  await handle({ getFindings, scan, now, tenant: "default" });
+  await handle({ getFindings, scan, now: testDate, tenant: "default", appendNamespace: true });
 
   expect(elasticClient.index).toBeCalledTimes(1);
   expect(elasticClient.index).toBeCalledWith({
     body: {
-      "@timestamp": now,
+      "@timestamp": testDate,
       id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
       labels: {
         company: "iteratec",
@@ -41,12 +41,12 @@ test("should only send scan summary document if no findings are passing in", asy
       scan_type: "Nmap",
       type: "scan",
     },
-    index: `scbv2_default_${now.toISOString().substr(0, 10)}`,
+    index: `scbv2_default_2020-11-11`,
   });
   expect(elasticClient.bulk).not.toBeCalled();
 });
 
-test("should send findings to elasticsearch", async () => {
+test("should send findings to elasticsearch with given prefix", async () => {
   const findings = [
     {
       id: "4560b3e6-1219-4f5f-9b44-6579f5a32407",
@@ -57,28 +57,12 @@ test("should send findings to elasticsearch", async () => {
 
   const getFindings = async () => findings;
 
-  const scan = {
-    metadata: {
-      uid: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
-      name: "demo-scan",
-      labels: {
-        company: "iteratec",
-      },
-    },
-    spec: {
-      scanType: "Nmap",
-      parameters: ["-Pn", "localhost"],
-    },
-  };
-
-  const now = new Date();
-
-  await handle({ getFindings, scan, now, tenant: "default" });
+  await handle({ getFindings, scan, now: testDate, tenant: "default", indexPrefix: "myPrefix", appendNamespace: true });
 
   expect(elasticClient.index).toBeCalledTimes(1);
   expect(elasticClient.index).toBeCalledWith({
     body: {
-      "@timestamp": now,
+      "@timestamp": testDate,
       id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
       labels: {
         company: "iteratec",
@@ -88,7 +72,7 @@ test("should send findings to elasticsearch", async () => {
       scan_type: "Nmap",
       type: "scan",
     },
-    index: `scbv2_default_${now.toISOString().substr(0, 10)}`,
+    index: `myPrefix_default_2020-11-11`,
   });
 
   expect(elasticClient.bulk).toBeCalledTimes(1);
@@ -97,11 +81,11 @@ test("should send findings to elasticsearch", async () => {
     body: [
       {
         index: {
-          _index: `scbv2_default_${now.toISOString().substr(0, 10)}`,
+          _index: `myPrefix_default_2020-11-11`,
         },
       },
       {
-        "@timestamp": now,
+        "@timestamp": testDate,
         category: "Open Port",
         id: "4560b3e6-1219-4f5f-9b44-6579f5a32407",
         name: "Port 5601 is open",
@@ -116,3 +100,75 @@ test("should send findings to elasticsearch", async () => {
     ],
   });
 });
+
+test("should not append namespace if 'appendNamespace' is null", async () => {
+  const findings = [];
+
+  const getFindings = async () => findings;
+
+  await handle({ getFindings, scan, now: testDate, tenant: "default" });
+
+  expect(elasticClient.index).toBeCalledTimes(1);
+  expect(elasticClient.index).toBeCalledWith({
+    body: {
+      "@timestamp": testDate,
+      id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
+      labels: {
+        company: "iteratec",
+      },
+      name: "demo-scan",
+      parameters: ["-Pn", "localhost"],
+      scan_type: "Nmap",
+      type: "scan",
+    },
+    index: `scbv2_2020-11-11`,
+  });
+});
+
+test("should append date format yyyy", async () => {
+  const findings = [];
+
+  const getFindings = async () => findings;
+
+  await handle({ getFindings, scan, now: testDate, tenant: "default", indexSuffix: "yyyy" });
+
+  expect(elasticClient.index).toBeCalledTimes(1);
+  expect(elasticClient.index).toBeCalledWith({
+    body: {
+      "@timestamp": testDate,
+      id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
+      labels: {
+        company: "iteratec",
+      },
+      name: "demo-scan",
+      parameters: ["-Pn", "localhost"],
+      scan_type: "Nmap",
+      type: "scan",
+    },
+    index: `scbv2_2020`,
+  });
+});
+
+test("should append week format like yyyy/'W'W -> 2020/W46", async () => {
+  const findings = [];
+
+  const getFindings = async () => findings;
+
+  await handle({ getFindings, scan, now: testDate, tenant: "default", indexSuffix: "yyyy/'W'W" });
+
+  expect(elasticClient.index).toBeCalledTimes(1);
+  expect(elasticClient.index).toBeCalledWith({
+    body: {
+      "@timestamp": testDate,
+      id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
+      labels: {
+        company: "iteratec",
+      },
+      name: "demo-scan",
+      parameters: ["-Pn", "localhost"],
+      scan_type: "Nmap",
+      type: "scan",
+    },
+    index: `scbv2_2020/W46`,
+  });
+});
diff --git a/hooks/persistence-elastic/package-lock.json b/hooks/persistence-elastic/package-lock.json
diff --git a/hooks/persistence-elastic/package.json b/hooks/persistence-elastic/package.json
@@ -24,13 +24,13 @@
     "url": "https://www.iteratec.com"
   },
   "contributors": [
-    { 
-      "name" : "Jannik Hollenbach",
-      "url" : "https://github.com/J12934"
+    {
+      "name": "Jannik Hollenbach",
+      "url": "https://github.com/J12934"
     },
-    { 
-      "name" : "Robert Seedorff",
-      "url" : "https://github.com/rseedorff"
+    {
+      "name": "Robert Seedorff",
+      "url": "https://github.com/rseedorff"
     }
   ],
   "bugs": {
@@ -43,6 +43,7 @@
     "lodash.flatmap": "^4.5.0"
   },
   "devDependencies": {
-    "jest": "^25.1.0"
+    "jest": "^25.1.0",
+    "luxon": "^1.25.0"
   }
 }
diff --git a/hooks/persistence-elastic/templates/persistence-provider.yaml b/hooks/persistence-elastic/templates/persistence-provider.yaml
@@ -10,6 +10,10 @@ spec:
   env:
     - name: ELASTICSEARCH_INDEX_PREFIX
       value: {{ .Values.indexPrefix | quote }}
+    - name: ELASTICSEARCH_INDEX_SUFFIX
+      value: {{ .Values.indexSuffix | quote }}
+    - name: ELASTICSEARCH_INDEX_APPEND_NAMESPACE
+      value: {{ .Values.indexAppendNamespace | quote }}
 {{- if .Values.externalElasticStack.enabled }}
     - name: ELASTICSEARCH_ADDRESS
       value: {{ .Values.externalElasticStack.elasticsearchAddress | quote }}
@@ -39,4 +43,4 @@ spec:
         secretKeyRef:
           name: {{ .Values.authentication.apiKeySecret }}
           key: id
-{{- end }}
+{{- end }}
diff --git a/hooks/persistence-elastic/values.yaml b/hooks/persistence-elastic/values.yaml
@@ -12,6 +12,10 @@ image:
 
 # indexPrefix -- Define a specific index prefix used for all elasticsearch indices.
 indexPrefix: "scbv2"
+# indexSuffix -- Define a specific index suffix based on date pattern (YEAR (yyyy), MONTH (yyyy-MM), WEEK (yyyy-'W'W), DATE (yyyy-MM-dd)). We use Luxon for date formatting (https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens)
+indexSuffix: “yyyy-MM-dd”
+# indexAppendNamespace -- Define if the name of the namespace where this hook is deployed to must be added to the index name. The namespace can be used to separate index by tenants (namespaces).
+indexAppendNamespace: true
 
 externalElasticStack:
   # externalElasticStack.enabled -- Enable this when you already have an Elastic Stack running to which you want to send your results