Skip to content

Commit 385fb4c

Browse files
kersevankersevan
committed
Add temporary results and scorecards
1 parent 1c8bd06 commit 385fb4c

File tree

58 files changed

+1113710
-94298
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1113710
-94298
lines changed

‎results/CodeqlSemgrepSnyk.sarif‎

Lines changed: 232333 additions & 0 deletions
Large diffs are not rendered by default.

‎results/CodeqlSemgrepSnykAdvanced.sarif‎

Lines changed: 62380 additions & 0 deletions
Large diffs are not rendered by default.

‎results/CodeqlSemgrepSnykCategories.sarif‎

Lines changed: 47680 additions & 0 deletions
Large diffs are not rendered by default.

‎results/CodeqlSemgrepSnykHighPrecision.sarif‎

Lines changed: 7234 additions & 0 deletions
Large diffs are not rendered by default.

‎results/SemgrepCodeQLMerged.sarif‎

Lines changed: 0 additions & 93669 deletions
This file was deleted.

‎results/codeql.sarif‎

100755100644
Lines changed: 230 additions & 230 deletions
Large diffs are not rendered by default.

‎results/mergeResults.py‎

Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
import analyzeCodeQL
2+
import analyzeSnyk
3+
import analyzeSemgrep
4+
import Classes
5+
from pprint import pprint
6+
from jinja2 import FileSystemLoader, Environment
7+
import json
8+
import yaml
9+
10+
11+
def createMergedResultsSarifFile(rules, results, template, outputFileName):
12+
file_loader = FileSystemLoader('templates')
13+
env = Environment(loader=file_loader)
14+
jinja2Template = env.get_template(template)
15+
16+
output = jinja2Template.render(rules=rules, results=results)
17+
18+
# use yaml to clean json - remove excess commas, then turn back to json
19+
outputJson = json.dumps(yaml.safe_load(output))
20+
outputJson = json.loads(outputJson)
21+
22+
# change fields for benchmark to distinguish between tools
23+
outputJson["runs"][0]["tool"]["driver"]["name"] = outputFileName.split("/")[-1]
24+
outputJson["runs"][0]["tool"]["driver"]["semanticVersion"] = outputFileName.split("/")[-1]
25+
26+
outputFile = open(outputFileName, "w+")
27+
outputFile.write(json.dumps(outputJson, indent=1))
28+
29+
def mergeCodeQLwithSnykByLocation(codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
30+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
31+
snykResults = analyzeSnyk.extractResults(snykFileName)
32+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
33+
snykRules = analyzeSnyk.extractRules(snykFileName)
34+
mergedResults, mergedRules = [], []
35+
36+
for codeqlElement in codeqlResults:
37+
codeqlCwes = codeqlElement.cwes
38+
codeqlLocation = codeqlElement.location
39+
40+
for snykElement in snykResults:
41+
# if snykElement.containsCwe(codeqlCwes):
42+
snykLocation = snykElement.location
43+
if codeqlElement.containsLocation(snykLocation):
44+
mergedResults.append(codeqlElement)
45+
46+
createMergedResultsSarifFile(codeqlRules, mergedResults, jinja2TemplateFileName, outputFileName)
47+
48+
def mergeCodeQLwithSnykAdvanced(codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
49+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
50+
snykResults = analyzeSnyk.extractResults(snykFileName)
51+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
52+
snykRules = analyzeSnyk.extractRules(snykFileName)
53+
mergedResults, mergedRules = [], []
54+
55+
for codeqlElement in codeqlResults:
56+
codeqlCwes = codeqlElement.cwes
57+
codeqlLocation = codeqlElement.location
58+
59+
for snykElement in snykResults:
60+
# if snykElement.containsCwe(codeqlCwes):
61+
snykLocation = snykElement.location
62+
if codeqlElement.containsLocation(snykLocation):
63+
mergedResults.append(codeqlElement)
64+
65+
createMergedResultsSarifFile(codeqlRules, mergedResults, jinja2TemplateFileName, outputFileName)
66+
67+
def mergeCodeQLwithSnyk(codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
68+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
69+
snykResults = analyzeSnyk.extractResults(snykFileName)
70+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
71+
snykRules = analyzeSnyk.extractRules(snykFileName)
72+
73+
createMergedResultsSarifFile(codeqlRules + snykRules, codeqlResults + snykResults, jinja2TemplateFileName, outputFileName)
74+
75+
def mergeSemgrepWithCodeQL(semgrepFileName, codeqlFileName, jinja2TemplateFileName, outputFileName):
76+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
77+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
78+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
79+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
80+
81+
createMergedResultsSarifFile(semgrepRules + codeqlRules, semgrepResults + codeqlResults, jinja2TemplateFileName, outputFileName)
82+
83+
def mergeSemgrepWithCodeQLPrecisionHigh(semgrepFileName, codeqlFileName, jinja2TemplateFileName, outputFileName):
84+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
85+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
86+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
87+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
88+
89+
mergedResults, mergedRules = [], codeqlRules + semgrepRules
90+
91+
highPrecisionRules = {}
92+
93+
for rule in mergedRules:
94+
pprint(rule.precision)
95+
# pprint(vars(rule))
96+
if rule.precision == "HIGH":
97+
highPrecisionRules[rule.ruleId] = "HIGH"
98+
if rule.precision == "MEDIUM":
99+
highPrecisionRules[rule.ruleId] = "MEDIUM"
100+
101+
for element in codeqlResults + semgrepResults:
102+
if element.ruleId in highPrecisionRules:
103+
mergedResults.append(element)
104+
continue
105+
106+
createMergedResultsSarifFile(semgrepRules + codeqlRules, mergedResults, jinja2TemplateFileName, outputFileName)
107+
108+
def mergeSemgrepWithCodeQLAdvanced(semgrepFileName, codeqlFileName, jinja2TemplateFileName, outputFileName):
109+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
110+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
111+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
112+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
113+
114+
mergedResults, mergedRules = [], codeqlRules + semgrepRules
115+
116+
highPrecisionRules = {}
117+
118+
for rule in mergedRules:
119+
pprint(rule.precision)
120+
# pprint(vars(rule))
121+
if rule.precision == "HIGH":
122+
highPrecisionRules[rule.ruleId] = "HIGH"
123+
if rule.precision == "MEDIUM":
124+
highPrecisionRules[rule.ruleId] = "MEDIUM"
125+
126+
for element in codeqlResults + semgrepResults:
127+
if element.ruleId in highPrecisionRules:
128+
mergedResults.append(element)
129+
continue
130+
131+
132+
createMergedResultsSarifFile(mergedRules, mergedResults, jinja2TemplateFileName, outputFileName)
133+
134+
def mergeSemgrepWithCodeQLWithSnyk(semgrepFileName, codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
135+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
136+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
137+
snykResults = analyzeSnyk.extractResults(snykFileName)
138+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
139+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
140+
snykRules = analyzeSnyk.extractRules(snykFileName)
141+
142+
mergedResults, mergedRules = codeqlResults + semgrepResults + snykResults, codeqlRules + semgrepRules + snykRules
143+
createMergedResultsSarifFile(mergedRules, mergedResults, jinja2TemplateFileName, outputFileName)
144+
145+
def mergeSemgrepWithCodeQLWithSnykAdvanced(semgrepFileName, codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
146+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
147+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
148+
snykResults = analyzeSnyk.extractResults(snykFileName)
149+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
150+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
151+
snykRules = analyzeSnyk.extractRules(snykFileName)
152+
153+
mergedResults, mergedRules = [], codeqlRules + semgrepRules + snykRules
154+
highPrecisionRules = {}
155+
156+
for rule in codeqlRules + semgrepRules:
157+
# pprint(rule.precision)
158+
# pprint(vars(rule))
159+
if rule.precision == "HIGH":
160+
highPrecisionRules[rule.ruleId] = "HIGH"
161+
# if rule.precision == "MEDIUM":
162+
# highPrecisionRules[rule.ruleId] = "MEDIUM"
163+
164+
allResults = codeqlResults + semgrepResults + snykResults
165+
166+
seenOnce = set()
167+
seenTwice = set()
168+
resultsAppearingTwice = []
169+
resultsAppearingTrice = []
170+
171+
for result in allResults:
172+
if result.ruleId in highPrecisionRules:
173+
resultsAppearingTwice.append(result)
174+
continue
175+
location = f"uri: {result.location.uri}"
176+
print(location)
177+
# location = f"uri: {result.location.uri}, startLine: {result.location.startLine}"
178+
# print(location)
179+
if location in seenOnce:
180+
resultsAppearingTwice.append(result)
181+
if location in seenTwice:
182+
resultsAppearingTrice.append(result)
183+
else:
184+
seenTwice.add(location)
185+
else:
186+
seenOnce.add(location)
187+
188+
189+
190+
createMergedResultsSarifFile(mergedRules, resultsAppearingTwice, jinja2TemplateFileName, outputFileName)
191+
192+
def mergeSemgrepWithCodeQLWithSnykByHighPrecision(semgrepFileName, codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
193+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
194+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
195+
snykResults = analyzeSnyk.extractResults(snykFileName)
196+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
197+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
198+
snykRules = analyzeSnyk.extractRules(snykFileName)
199+
200+
mergedResults, mergedRules = [], codeqlRules + semgrepRules + snykRules
201+
highPrecisionRules = {}
202+
203+
for rule in codeqlRules + semgrepRules:
204+
# pprint(rule.precision)
205+
# pprint(vars(rule))
206+
if rule.precision == "HIGH":
207+
highPrecisionRules[rule.ruleId] = "HIGH"
208+
209+
allResults = codeqlResults + semgrepResults + snykResults
210+
211+
seenOnce = set()
212+
seenTwice = set()
213+
resultsAppearingTwice = []
214+
resultsAppearingTrice = []
215+
216+
for result in allResults:
217+
if result.ruleId in highPrecisionRules:
218+
resultsAppearingTwice.append(result)
219+
continue
220+
location = f"uri: {result.location.uri}, startLine: {result.location.startLine}"
221+
print(location)
222+
if location in seenOnce:
223+
resultsAppearingTwice.append(result)
224+
if location in seenTwice:
225+
resultsAppearingTrice.append(result)
226+
else:
227+
seenTwice.add(location)
228+
else:
229+
seenOnce.add(location)
230+
231+
232+
233+
createMergedResultsSarifFile(mergedRules, resultsAppearingTwice, jinja2TemplateFileName, outputFileName)
234+
235+
def mergeSemgrepWithCodeQLWithSnykByCategories(semgrepFileName, codeqlFileName, snykFileName, jinja2TemplateFileName, outputFileName):
236+
codeqlResults = analyzeCodeQL.extractResults(codeqlFileName)
237+
semgrepResults = analyzeSemgrep.extractResults(semgrepFileName)
238+
snykResults = analyzeSnyk.extractResults(snykFileName)
239+
codeqlRules = analyzeCodeQL.extractRules(codeqlFileName)
240+
semgrepRules = analyzeSemgrep.extractRules(semgrepFileName)
241+
snykRules = analyzeSnyk.extractRules(snykFileName)
242+
243+
mergedRules, mergedResults = codeqlRules + semgrepRules + snykRules, []
244+
allResults = codeqlResults + semgrepResults + snykResults
245+
allRules = codeqlRules + semgrepRules + snykRules
246+
ruleIds= []
247+
248+
# switch = {
249+
# '22' : 'pathtraver',
250+
# '78' : 'cmdi',
251+
# '79' : 'xss',
252+
# '89' : 'sqli',
253+
# '90' : 'ldapi',
254+
# '327' : 'crypto',
255+
# '328' : 'hash',
256+
# '330' : 'weakrand',
257+
# '501' : 'trustbound',
258+
# '614' : 'securecookie',
259+
# '643' : 'xpathi',
260+
# }
261+
262+
# for rule in codeqlRules:
263+
# print(rule.categories)
264+
# print(rule.cwes)
265+
# if "securecookie" in rule.categories:
266+
# print(rule.ruleId)
267+
# ruleIds.append(rule.ruleId)
268+
# if "xss" in rule.categories:
269+
# print(rule.ruleId)
270+
# ruleIds.append(rule.ruleId)
271+
# if "sqli" in rule.categories or "ldapi" in rule.categories or "pathtraver" in rule.categories:
272+
# print(rule.ruleId)
273+
# ruleIds.append(rule.ruleId)
274+
# for result in codeqlResults:
275+
# if result.ruleId in ruleIds:
276+
# mergedResults.append(result)
277+
278+
# ruleIds= []
279+
280+
# for rule in semgrepRules:
281+
# if "weakrand" in rule.categories:
282+
# ruleIds.append(rule.ruleId)
283+
# if "crypto" in rule.categories:
284+
# ruleIds.append(rule.ruleId)
285+
# if "hash" in rule.categories:
286+
# ruleIds.append(rule.ruleId)
287+
# for result in semgrepResults:
288+
# if result.ruleId in ruleIds:
289+
# mergedResults.append(result)
290+
291+
ruleIds = []
292+
for rule in allRules:
293+
if "xpathi" in rule.categories or "pathtraver" in rule.categories or "cmdi" in rule.categories or "sqli" in rule.categories or "ldapi" in rule.categories:
294+
ruleIds.append(rule.ruleId)
295+
296+
seen = set()
297+
resultsAppearingTwice = []
298+
299+
for result in allResults:
300+
if result.ruleId in ruleIds:
301+
location = f"uri: {result.location.uri}, startLine: {result.location.startLine}"
302+
if location in seen:
303+
resultsAppearingTwice.append(result)
304+
else:
305+
seen.add(location)
306+
mergedResults += resultsAppearingTwice
307+
308+
309+
createMergedResultsSarifFile(mergedRules, mergedResults, jinja2TemplateFileName, outputFileName)
310+
311+
# COMPARE:
312+
# 0. all results of different tools merged
313+
# - does removing duplicates help?
314+
# 1. all results with precision HIGH
315+
# 2. all results that appear in results of 2 different tools
316+
# 3. all results that appear in results of 3 different tools
317+
# 4. na podlagi expectedresults-1.2.csv pridobi TP, TN ...
318+
# - če naredim to lahko tudi SARD

‎results/semgrep-results.json‎

100755100644
File mode changed.

0 commit comments

Comments
 (0)