Skip to content

Commit 6f8ccbb

Browse files
SebieFSebieF
committed
Changing parser to work with json instead of xml
Signed-off-by: Sebastian <sebastian.franz@iteratec.com>
1 parent c363eba commit 6f8ccbb

File tree

10 files changed

+148
-396
lines changed

10 files changed

+148
-396
lines changed

‎scanners/whatweb/parser/__testFiles__/example.com.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[
2+
{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7EA7)"]},"Country":{"string":["EUROPEAN UNION"],"module":["EU"]}}},
3+
{}
4+
]

‎scanners/whatweb/parser/__testFiles__/example.com.xml‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎scanners/whatweb/parser/__testFiles__/no-address.com.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[
2+
{}
3+
]

‎scanners/whatweb/parser/__testFiles__/securecodebox.io.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[
2+
{"target":"http://securecodebox.io","http_status":301,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["185.199.110.153"]},"Title":{"string":["301 Moved Permanently"]},"HTTPServer":{"string":["GitHub.com"]},"Via-Proxy":{"string":["1.1 varnish"]},"UncommonHeaders":{"string":["x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]},"RedirectLocation":{"string":["https://www.securecodebox.io/"]}}},
3+
{"target":"https://www.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["185.199.110.153"]},"Title":{"string":["secureCodeBox – Testing your Software Security"]},"Meta-Refresh-Redirect":{"string":["https://docs.securecodebox.io/"]},"HTML5":{},"Strict-Transport-Security":{"string":["max-age=31556952"]},"HTTPServer":{"string":["GitHub.com"]},"Via-Proxy":{"string":["1.1 varnish"]},"UncommonHeaders":{"string":["access-control-allow-origin,x-proxy-cache,x-github-request-id,x-served-by,x-cache-hits,x-timer,x-fastly-request-id"]}}},
4+
{"target":"https://docs.securecodebox.io/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["206.189.58.26"]},"Script":{},"Open-Graph-Protocol":{},"HTML5":{},"Strict-Transport-Security":{"string":["max-age=31536000"]},"HTTPServer":{"string":["Netlify"]},"UncommonHeaders":{"string":["x-nf-request-id"]},"Country":{"string":["UNITED STATES"],"module":["US"]},"MetaGenerator":{"string":["Docusaurus v2.0.0-beta.3"]}}},
5+
{}
6+
]

‎scanners/whatweb/parser/__testFiles__/securecodebox.io.xml‎

Lines changed: 0 additions & 87 deletions
This file was deleted.

‎scanners/whatweb/parser/__testFiles__/somedomain.com.xml‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎scanners/whatweb/parser/__testFiles__/two-domains.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[
2+
{"target":"http://example.com","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["93.184.216.34"]},"Title":{"string":["Example Domain"]},"HTML5":{},"HTTPServer":{"string":["ECS (dcb/7F14)"]},"Country":{"string":["EUROPEAN UNION"],"module":["EU"]}}},
3+
{"target":"http://scanme.nmap.org/","http_status":200,"request_config":{"headers":{"User-Agent":"WhatWeb/0.5.0"}},"plugins":{"IP":{"string":["45.33.32.156"]},"Script":{"string":["text/javascript"]},"Google-Analytics":{"version":["Universal"],"account":["UA-11009417-1"]},"Title":{"string":["Go ahead and ScanMe!"]},"HTTPServer":{"os":["Ubuntu Linux"],"string":["Apache/2.4.7 (Ubuntu)"]},"Country":{"string":["RESERVED"],"module":["ZZ"]},"Apache":{"version":["2.4.7"]}}},
4+
{}
5+
]

‎scanners/whatweb/parser/__testFiles__/two-domains.xml‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎scanners/whatweb/parser/parser.js‎

Lines changed: 45 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,15 @@
22
//
33
// SPDX-License-Identifier: Apache-2.0
44

5-
const xml2js = require('xml2js');
6-
75
async function parse(fileContent) {
8-
const hosts = await parseResultFile(fileContent);
9-
return transformToFindings(hosts);
6+
const targets = await parseResultFile(fileContent);
7+
return transformToFindings(targets);
108
}
119

1210
function transformToFindings(targets) {
1311

1412
const targetFindings = targets.map(target => {
15-
let tempFinding = {
13+
let finding = {
1614
name: target.uri,
1715
category: "WEB APPLICATION",
1816
description: target.title,
@@ -21,80 +19,66 @@ function transformToFindings(targets) {
2119
severity: 'INFORMATIONAL',
2220
attributes: {
2321
requestConfig: target.requestConfig,
24-
ipAddress: target.ipAddress
22+
ipAddress: target.ipAddress,
23+
country: target.country,
24+
HTML5: target.html5
2525
}
2626
};
2727

2828
target.additional.forEach(additional => {
29-
if (!tempFinding.attributes[additional.name[0]]) {
30-
tempFinding.attributes[additional.name[0]] =
31-
(("string" in additional) ? additional.string[0] : "") + (("module" in additional) ? " " + additional.module[0] : "");
29+
if (!finding.attributes[additional[0]]) { //Check if key already exists
30+
finding.attributes[additional[0]] =
31+
(("string" in additional[1]) ? additional[1].string[0] : "") +
32+
(("module" in additional[1]) ? "/" + additional[1].module[0] : "");
3233
}
3334
});
3435

35-
return tempFinding;
36+
if (!finding.attributes.HTML5) //Do not show in findings if undefined
37+
delete finding.attributes.HTML5;
38+
39+
return finding;
3640
});
3741

3842
return [...targetFindings];
3943
}
4044

4145
/**
42-
* Parses a given NMAP XML file to a smaller JSON represenation with the following object:
43-
* {
44-
* hostname: null,
45-
* ip: null,
46-
* mac: null,
47-
* openPorts: null,
48-
* osNmap: null,
49-
* scripts: null
50-
* }
51-
* @param {*} fileContent
46+
* Parses a given Whatweb JSON file and extracts all targets
47+
* @param {*} fileContent
5248
*/
5349
function parseResultFile(fileContent) {
54-
return new Promise((resolve, reject) => {
55-
xml2js.parseString(fileContent, (err, xmlInput) => {
56-
if (err) {
57-
reject(new Error('Error converting XML to JSON in xml2js: ' + err));
58-
} else {
59-
let tempTargetList = [];
60-
if (!xmlInput.log.target) {
61-
resolve([]);
62-
return;
50+
console.log(fileContent);
51+
let targetList = [];
52+
for(const rawTarget of fileContent) {
53+
if (rawTarget.target) { //Check for empty target
54+
let newTarget = {
55+
uri: rawTarget.target,
56+
httpStatus: rawTarget.http_status,
57+
requestConfig: rawTarget.request_config.headers["User-Agent"],
58+
ipAddress: null,
59+
title: null,
60+
html5: null,
61+
country: null,
62+
additional: []
6363
}
64-
65-
xmlInput = xmlInput.log.target;
66-
67-
tempTargetList = xmlInput.map(target => {
68-
let newTarget = {
69-
uri: target.uri[0],
70-
httpStatus: target['http-status'][0],
71-
requestConfig: {
72-
headerName: target['request-config'][0].header[0]["header-name"][0],
73-
headerValue: target['request-config'][0].header[0]["header-value"][0]
74-
},
75-
ipAddress: null,
76-
title: null,
77-
additional: []
78-
};
79-
80-
if(target.plugin) {
81-
for(const plugin of target.plugin) {
82-
if (plugin.name[0] === "IP")
83-
newTarget.ipAddress = plugin.string[0];
84-
else if (plugin.name[0] === "Title")
85-
newTarget.title = plugin.string[0];
86-
else
87-
newTarget.additional.push(plugin)
88-
}
64+
if(rawTarget.plugins) {
65+
for(const [key, value] of Object.entries(rawTarget.plugins)) {
66+
if (key === "IP")
67+
newTarget.ipAddress = value.string[0];
68+
else if (key === "Title")
69+
newTarget.title = value.string[0];
70+
else if (key === "HTML5")
71+
newTarget.html5 = true;
72+
else if (key === "Country")
73+
newTarget.country = value.string[0] + "/" + value.module[0];
74+
else
75+
newTarget.additional.push([key, value])
8976
}
90-
91-
return newTarget;
92-
});
93-
94-
resolve(tempTargetList);
77+
}
78+
targetList.push(newTarget);
9579
}
96-
});
97-
});
80+
}
81+
return targetList;
9882
}
9983

10084
module.exports.parse = parse;

0 commit comments

Comments
 (0)