Skip to content

Commit 69f747b

Browse files
konklonekonklone
authored
BOD 18-01 (web) analysis (GSA#247)
* making a sub-dir of compliance for m-15-13 analysis work * starting bod-18-01 analysis directory with copies of m-15-13 code * remove scratch space for m-15-13 * handle base domain detection better, header row detection better * update CFO Act agency names as represented in new .gov data * first pass at calculating compliance stats for a single scan event * remove unneeded files from a scan event * ignore libreoffice lock files * proper boolean parsing, ignore unperformed sslyze scans * use parent's preload status regardless of pshtt status for parent * downloading script for pre-bod and post-bod scans * moving data out of version control * finalize download script * removing data from version control, see Releases for this repo * generating first new CSVs for modern reports * some more reports, tweak the column headers some more
1 parent f820a5d commit 69f747b

101 files changed

Lines changed: 1023 additions & 6 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
.~lock*
12
.sass-cache
23
_site
34
.DS_Store
@@ -10,3 +11,6 @@ node_modules
1011
/assets/vendor
1112
/assets/css
1213
/vendor
14+
/compliance/*/cache
15+
/compliance/*/data
16+
/compliance/*/*.gz

‎compliance/bod-18-01/compliance.py‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
#!/usr/bin/env python
2+
3+
import utils
4+
import glob
5+
6+
## Todo for DHS:
7+
# * compare data across all dates
8+
9+
## Todo for me:
10+
# * pre-bod path and post-bod path
11+
# * parents-only data going back to 2015?
12+
# * cross-reference with DAP?
13+
14+
# get mapping of domains to agency
15+
base_domains = utils.domains_to_agencies()
16+
17+
# get current and pending preloaded domains
18+
# Downloads live from remote sources. (clear cache/ to re-download)
19+
preloaded = set(utils.fetch_preloaded()) & set(base_domains)
20+
21+
# See download.py for which dates are downloaded for each phase.
22+
23+
# pre-BOD measurements (no 3DES checking, old dir structure)
24+
pre_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/pre-bod/*")]
25+
pre_bod.sort()
26+
27+
# post-BOD measurements (bod_crypto measured, new dir structure)
28+
post_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/post-bod/*")]
29+
post_bod.sort()
30+
31+
32+
# post-bod paths
33+
def pshtts_and_sslyzes_for(date):
34+
pshtts = []
35+
sslyzes = []
36+
37+
if date in post_bod:
38+
pshtts.append("data/pulse/post-bod/%s/parents/results/pshtt.csv" % date)
39+
pshtts.append("data/pulse/post-bod/%s/subdomains/scan/results/pshtt.csv" % date)
40+
sslyzes.append("data/pulse/post-bod/%s/parents/results/sslyze.csv" % date)
41+
sslyzes.append("data/pulse/post-bod/%s/subdomains/scan/results/sslyze.csv" % date)
42+
elif date in pre_bod:
43+
pshtts.append("data/pulse/pre-bod/%s/scan/pshtt.csv" % date)
44+
pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/pshtt.csv" % date)
45+
pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/pshtt.csv" % date)
46+
sslyzes.append("data/pulse/pre-bod/%s/scan/sslyze.csv" % date)
47+
sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/sslyze.csv" % date)
48+
sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/sslyze.csv" % date)
49+
50+
return pshtts, sslyzes
51+
52+
def pct(num, denom):
53+
return round((num / denom) * 100)
54+
55+
def compliance_stats(name, date, filter):
56+
pshtts, sslyzes = pshtts_and_sslyzes_for(date)
57+
data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
58+
totals = utils.compliance_totals(data)
59+
60+
count = len(data.keys())
61+
62+
if count == 0:
63+
print("No rows returned, no data to display.")
64+
return
65+
66+
print()
67+
print("=====================================================")
68+
print(" [%s] %s" % (date, name))
69+
print("=====================================================")
70+
print()
71+
print("Total domains: %i" % count)
72+
print()
73+
print("== Direct enforcement ==")
74+
print("Enforces HTTPS: %i (%i%%)" % (totals['enforces'], pct(totals['enforces'], count)))
75+
print("Strong HSTS: %i (%i%%)" % (totals['hsts'], pct(totals['hsts'], count)))
76+
print("RC4 support: %i (%i%%)" % (totals['rc4'], pct(totals['rc4'], count)))
77+
78+
if date in post_bod:
79+
print("3DES support: %i (%i%%)" % (totals['3des'], pct(totals['3des'], count)))
80+
print("Free of Known-weak Crypto: %i (%i%%)" % (totals['bod_crypto'], pct(totals['bod_crypto'], count)))
81+
print("Compliant with BOD 18-01: %i (%i%%)" % (totals['compliant'], pct(totals['compliant'], count)))
82+
83+
elif date in pre_bod:
84+
print("Compliant with M-15-13: %i (%i%%)" % (totals['m1513'], pct(totals['m1513'], count)))
85+
86+
print()
87+
88+
def compliance_csv(dates, when, filter, path):
89+
if when == "pre":
90+
header = [
91+
"Date", "Total Hostnames",
92+
"Enforces HTTPS", "HSTS",
93+
"M-15-13",
94+
"RC4",
95+
"Enforces HTTPS (%)", "HSTS (%)",
96+
"M-15-13 (%)",
97+
"RC4 (%)",
98+
]
99+
elif when == "post":
100+
header = [
101+
"Date", "Total Hostnames",
102+
"Enforces HTTPS", "HSTS",
103+
"M-15-13",
104+
"RC4", "3DES", "Free of SSLv2/SSLv3/RC4/3DES",
105+
"BOD 18-01",
106+
"Enforces HTTPS (%)", "HSTS (%)",
107+
"M-15-13 (%)",
108+
"RC4 (%)", "3DES (%)", "Free of SSLv2/SSLv3/RC4/3DES (%)",
109+
"BOD 18-01 (%)",
110+
]
111+
112+
rows = []
113+
for date in dates:
114+
rows.append(compliance_csv_row(date, filter))
115+
116+
utils.save_csv(header, rows, path)
117+
118+
def compliance_csv_row(date, filter):
119+
print("[%s] Running report for row..." % date)
120+
pshtts, sslyzes = pshtts_and_sslyzes_for(date)
121+
data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
122+
totals = utils.compliance_totals(data)
123+
124+
count = len(data.keys())
125+
126+
if date in pre_bod:
127+
return [
128+
date,
129+
count,
130+
totals['enforces'],
131+
totals['hsts'],
132+
totals['m1513'],
133+
totals['rc4'],
134+
pct(totals['enforces'], count),
135+
pct(totals['hsts'], count),
136+
pct(totals['m1513'], count),
137+
pct(totals['rc4'], count),
138+
]
139+
elif date in post_bod:
140+
return [
141+
date,
142+
count,
143+
totals['enforces'],
144+
totals['hsts'],
145+
totals['m1513'],
146+
totals['rc4'],
147+
totals['3des'],
148+
totals['bod_crypto'],
149+
totals['compliant'],
150+
pct(totals['enforces'], count),
151+
pct(totals['hsts'], count),
152+
pct(totals['m1513'], count),
153+
pct(totals['rc4'], count),
154+
pct(totals['3des'], count),
155+
pct(totals['bod_crypto'], count),
156+
pct(totals['compliant'], count),
157+
]
158+
159+
160+
# # All executive hostnames pre-BOD.
161+
# compliance_csv(pre_bod, "pre",
162+
# utils.executive_only,
163+
# "cache/pre-bod-executive.csv"
164+
# )
165+
# # All executive hostnames post-BOD.
166+
# compliance_csv(post_bod, "post",
167+
# utils.executive_only,
168+
# "cache/post-bod-executive.csv"
169+
# )
170+
171+
172+
# CFO Act
173+
compliance_csv(pre_bod, "pre",
174+
utils.cfo_act_only,
175+
"cache/pre-bod-cfo.csv"
176+
)
177+
compliance_csv(post_bod, "post",
178+
utils.cfo_act_only,
179+
"cache/post-bod-cfo.csv"
180+
)
181+
182+
# Non-CFO Act
183+
compliance_csv(pre_bod, "pre",
184+
utils.executive_non_cfo_act,
185+
"cache/pre-bod-non-cfo.csv"
186+
)
187+
compliance_csv(post_bod, "post",
188+
utils.executive_non_cfo_act,
189+
"cache/post-bod-non-cfo.csv"
190+
)
191+
192+
# # CFO Act (minus DoD)
193+
# compliance_csv(pre_bod, "pre",
194+
# utils.cfo_act_only_sans_dod,
195+
# "cache/pre-bod-cfo-no-dod.csv"
196+
# )
197+
# compliance_csv(post_bod, "post",
198+
# utils.cfo_act_only_sans_dod,
199+
# "cache/post-bod-cfo-no-dod.csv"
200+
# )
201+
202+
# # DoD only
203+
# compliance_csv(pre_bod, "pre",
204+
# utils.for_agencies(["Department of Defense"]),
205+
# "cache/pre-bod-dod.csv"
206+
# )
207+
# compliance_csv(post_bod, "post",
208+
# utils.for_agencies(["Department of Defense"]),
209+
# "cache/post-bod-dod.csv"
210+
# )
211+
212+

‎compliance/bod-18-01/download.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
##
2+
# Download data from the Pulse S3 bucket to measure scans over time.
3+
# Must be run from a machine where "aws s3 cp" has credentials for the bucket.
4+
5+
import os
6+
import subprocess
7+
8+
AWS_REGION = "us-gov-west-1"
9+
BUCKET_NAME = "cg-4adefb86-dadb-4ecf-be3e-f1c7b4f6d084"
10+
DATA_DIR = "data/pulse"
11+
12+
# Before BOD 18-01 was issued and before Pulse data was restructured
13+
pre_bod = ["2017-02-10", "2017-02-12", "2017-02-15", "2017-02-17", "2017-02-20", "2017-02-22", "2017-02-24", "2017-02-27", "2017-03-01", "2017-03-03", "2017-03-06", "2017-03-08", "2017-03-10", "2017-03-13", "2017-03-15", "2017-03-17", "2017-03-20", "2017-03-22", "2017-03-24", "2017-03-27", "2017-03-30", "2017-03-31", "2017-04-04", "2017-04-06", "2017-04-08", "2017-04-11", "2017-04-13", "2017-04-20", "2017-04-22", "2017-04-25", "2017-04-27", "2017-04-29", "2017-05-02", "2017-05-04", "2017-05-06", "2017-05-09", "2017-05-11", "2017-05-13", "2017-05-16", "2017-05-18", "2017-05-20", "2017-05-23", "2017-05-25", "2017-05-27", "2017-05-30", "2017-06-01", "2017-06-03", "2017-06-05", "2017-07-02", "2017-07-03", "2017-07-05", "2017-07-07", "2017-07-10", "2017-07-12", "2017-07-14", "2017-07-17", "2017-07-19", "2017-07-21", "2017-07-25", "2017-07-26", "2017-07-28", "2017-07-31", "2017-08-02", "2017-08-07", "2017-08-09", "2017-08-11", "2017-08-14", "2017-08-16", "2017-08-18", "2017-08-21", "2017-08-23", "2017-08-25", "2017-09-28"]
14+
15+
# After BOD 18-01 was issued and after Pulse data was restructured
16+
# Exceptions:
17+
# 2018-01-02: no subdomains/scan/results
18+
post_bod = ["2017-11-20", "2017-11-25", "2017-12-11", "2017-12-13", "2017-12-15", "2017-12-16", "2017-12-17", "2017-12-18", "2017-12-19", "2017-12-20", "2017-12-21", "2017-12-22", "2017-12-23", "2017-12-24", "2017-12-25", "2017-12-26", "2017-12-27", "2017-12-28", "2017-12-29", "2017-12-30", "2017-12-31", "2018-01-01", "2018-01-03", "2018-01-04", "2018-01-05", "2018-01-06", "2018-01-07", "2018-01-08", "2018-01-09", "2018-01-10", "2018-01-11", "2018-01-12", "2018-01-13", "2018-01-14", "2018-01-15", "2018-01-16", "2018-01-17", "2018-01-18", "2018-01-19", "2018-01-20", "2018-01-21", "2018-01-22", "2018-01-23", "2018-01-24", "2018-01-25", "2018-01-26", "2018-01-27", "2018-01-28", "2018-01-29", "2018-01-30", "2018-01-31", "2018-02-01", "2018-02-02", "2018-02-03", "2018-02-04", "2018-02-05", "2018-02-06", "2018-02-07", "2018-02-08", "2018-02-09", "2018-02-10", "2018-02-11", "2018-02-12", "2018-02-13", "2018-02-14", "2018-02-15", "2018-02-16", "2018-02-17", "2018-02-18", "2018-02-19", "2018-02-20", "2018-02-21", "2018-02-22", "2018-02-23", "2018-02-24", "2018-02-25", "2018-02-26", "2018-02-27", "2018-02-28", "2018-03-01", "2018-03-02", "2018-03-03", "2018-03-04", "2018-03-05", "2018-03-06", "2018-03-07", "2018-03-08", "2018-03-09", "2018-03-10", "2018-03-11", "2018-03-15", "2018-03-16", "2018-03-17", "2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21", "2018-03-23", "2018-03-24", "2018-03-25", "2018-03-26", "2018-03-27", "2018-03-28", "2018-03-29", "2018-03-30", "2018-03-31", "2018-04-01", "2018-04-02", "2018-04-03", "2018-04-09", "2018-04-10", "2018-04-11", "2018-04-12", "2018-04-13", "2018-04-14", "2018-04-15", "2018-04-16", "2018-04-25", "2018-04-26", "2018-04-27", "2018-04-28"]
19+
20+
def shell_out(command, env=None):
21+
response = subprocess.check_output(command, shell=False, env=env)
22+
output = str(response, encoding='UTF-8')
23+
return output
24+
25+
def mkdir_p(path):
26+
try:
27+
os.makedirs(path)
28+
except OSError as exc: # Python >2.5
29+
if exc.errno == errno.EEXIST:
30+
pass
31+
else:
32+
raise
33+
34+
35+
# when = "pre" or "post"
36+
def download_date(date, when="post"):
37+
38+
def download(date, file):
39+
url = "s3://%s/archive/%s/%s" % (BUCKET_NAME, date, file)
40+
41+
# local destinations are relative to data/
42+
path = os.path.join(DATA_DIR, "%s-bod" % when, date, file)
43+
44+
shell_out(["aws", "s3", "cp", url, path])
45+
46+
try:
47+
# Newer streamlined directory structure
48+
if when == "post":
49+
download(date, "parents/results/pshtt.csv")
50+
download(date, "parents/results/sslyze.csv")
51+
download(date, "parents/results/meta.json")
52+
download(date, "subdomains/scan/results/pshtt.csv")
53+
download(date, "subdomains/scan/results/sslyze.csv")
54+
download(date, "subdomains/scan/results/meta.json")
55+
56+
# Older, more laborious directory structure
57+
elif when == "pre":
58+
download(date, "scan/pshtt.csv")
59+
download(date, "scan/sslyze.csv")
60+
download(date, "scan/meta.json")
61+
download(date, "subdomains/scan/url/results/pshtt.csv")
62+
download(date, "subdomains/scan/url/results/sslyze.csv")
63+
download(date, "subdomains/scan/url/results/meta.json")
64+
download(date, "subdomains/scan/censys/results/pshtt.csv")
65+
download(date, "subdomains/scan/censys/results/sslyze.csv")
66+
download(date, "subdomains/scan/censys/results/meta.json")
67+
except subprocess.CalledProcessError:
68+
print("[%s] MISSING SOMETHING" % date)
69+
70+
71+
# for date in post_bod:
72+
# download_date(date, "post")
73+
74+
# for date in pre_bod:
75+
# download_date(date, "pre")

0 commit comments

Comments
 (0)