GSA · konklone · Jun 20, 2018 · Apr 28, 2018 · Apr 28, 2018 · Apr 28, 2018
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.~lock*
 .sass-cache
 _site
 .DS_Store
@@ -10,3 +11,6 @@ node_modules
 /assets/vendor
 /assets/css
 /vendor
+/compliance/*/cache
+/compliance/*/data
+/compliance/*/*.gz
diff --git a/compliance/bod-18-01/compliance.py b/compliance/bod-18-01/compliance.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+
+import utils
+import glob
+
+## Todo for DHS:
+# * compare data across all dates
+
+## Todo for me:
+# * pre-bod path and post-bod path
+# * parents-only data going back to 2015?
+# * cross-reference with DAP?
+
+# get mapping of domains to agency
+base_domains = utils.domains_to_agencies()
+
+# get current and pending preloaded domains
+# Downloads live from remote sources. (clear cache/ to re-download)
+preloaded = set(utils.fetch_preloaded()) & set(base_domains)
+
+# See download.py for which dates are downloaded for each phase.
+
+# pre-BOD measurements (no 3DES checking, old dir structure)
+pre_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/pre-bod/*")]
+pre_bod.sort()
+
+# post-BOD measurements (bod_crypto measured, new dir structure)
+post_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/post-bod/*")]
+post_bod.sort()
+
+
+# post-bod paths
+def pshtts_and_sslyzes_for(date):
+  pshtts = []
+  sslyzes = []
+
+  if date in post_bod:
+    pshtts.append("data/pulse/post-bod/%s/parents/results/pshtt.csv" % date)
+    pshtts.append("data/pulse/post-bod/%s/subdomains/scan/results/pshtt.csv" % date)
+    sslyzes.append("data/pulse/post-bod/%s/parents/results/sslyze.csv" % date)
+    sslyzes.append("data/pulse/post-bod/%s/subdomains/scan/results/sslyze.csv" % date)
+  elif date in pre_bod:
+    pshtts.append("data/pulse/pre-bod/%s/scan/pshtt.csv" % date)
+    pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/pshtt.csv" % date)
+    pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/pshtt.csv" % date)
+    sslyzes.append("data/pulse/pre-bod/%s/scan/sslyze.csv" % date)
+    sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/sslyze.csv" % date)
+    sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/sslyze.csv" % date)
+
+  return pshtts, sslyzes
+
+def pct(num, denom):
+  return round((num / denom) * 100)
+
+def compliance_stats(name, date, filter):
+  pshtts, sslyzes = pshtts_and_sslyzes_for(date)
+  data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
+  totals = utils.compliance_totals(data)
+
+  count = len(data.keys())
+
+  if count == 0:
+    print("No rows returned, no data to display.")
+    return
+
+  print()
+  print("=====================================================")
+  print(" [%s] %s" % (date, name))
+  print("=====================================================")
+  print()
+  print("Total domains: %i" % count)
+  print()
+  print("== Direct enforcement ==")
+  print("Enforces HTTPS: %i (%i%%)" % (totals['enforces'], pct(totals['enforces'], count)))
+  print("Strong HSTS: %i (%i%%)" % (totals['hsts'], pct(totals['hsts'], count)))
+  print("RC4 support: %i (%i%%)" % (totals['rc4'], pct(totals['rc4'], count)))
+
+  if date in post_bod:
+    print("3DES support: %i (%i%%)" % (totals['3des'], pct(totals['3des'], count)))
+    print("Free of Known-weak Crypto: %i (%i%%)" % (totals['bod_crypto'], pct(totals['bod_crypto'], count)))
+    print("Compliant with BOD 18-01: %i (%i%%)" % (totals['compliant'], pct(totals['compliant'], count)))
+
+  elif date in pre_bod:
+    print("Compliant with M-15-13: %i (%i%%)" % (totals['m1513'], pct(totals['m1513'], count)))
+
+  print()
+
+def compliance_csv(dates, when, filter, path):
+  if when == "pre":
+    header = [
+      "Date", "Total Hostnames",
+      "Enforces HTTPS", "HSTS",
+      "M-15-13",
+      "RC4",
+      "Enforces HTTPS (%)", "HSTS (%)",
+      "M-15-13 (%)",
+      "RC4 (%)",
+    ]
+  elif when == "post":
+    header = [
+      "Date", "Total Hostnames",
+      "Enforces HTTPS", "HSTS",
+      "M-15-13",
+      "RC4", "3DES", "Free of SSLv2/SSLv3/RC4/3DES",
+      "BOD 18-01",
+      "Enforces HTTPS (%)", "HSTS (%)",
+      "M-15-13 (%)",
+      "RC4 (%)", "3DES (%)", "Free of SSLv2/SSLv3/RC4/3DES (%)",
+      "BOD 18-01 (%)",
+    ]
+
+  rows = []
+  for date in dates:
+    rows.append(compliance_csv_row(date, filter))
+
+  utils.save_csv(header, rows, path)
+
+def compliance_csv_row(date, filter):
+  print("[%s] Running report for row..." % date)
+  pshtts, sslyzes = pshtts_and_sslyzes_for(date)
+  data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
+  totals = utils.compliance_totals(data)
+
+  count = len(data.keys())
+
+  if date in pre_bod:
+    return [
+      date,
+      count,
+      totals['enforces'],
+      totals['hsts'],
+      totals['m1513'],
+      totals['rc4'],
+      pct(totals['enforces'], count),
+      pct(totals['hsts'], count),
+      pct(totals['m1513'], count),
+      pct(totals['rc4'], count),
+    ]
+  elif date in post_bod:
+    return [
+      date,
+      count,
+      totals['enforces'],
+      totals['hsts'],
+      totals['m1513'],
+      totals['rc4'],
+      totals['3des'],
+      totals['bod_crypto'],
+      totals['compliant'],
+      pct(totals['enforces'], count),
+      pct(totals['hsts'], count),
+      pct(totals['m1513'], count),
+      pct(totals['rc4'], count),
+      pct(totals['3des'], count),
+      pct(totals['bod_crypto'], count),
+      pct(totals['compliant'], count),
+    ]
+
+
+# # All executive hostnames pre-BOD.
+# compliance_csv(pre_bod, "pre",
+#   utils.executive_only,
+#   "cache/pre-bod-executive.csv"
+# )
+# # All executive hostnames post-BOD.
+# compliance_csv(post_bod, "post",
+#   utils.executive_only,
+#   "cache/post-bod-executive.csv"
+# )
+
+
+# CFO Act
+compliance_csv(pre_bod, "pre",
+  utils.cfo_act_only,
+  "cache/pre-bod-cfo.csv"
+)
+compliance_csv(post_bod, "post",
+  utils.cfo_act_only,
+  "cache/post-bod-cfo.csv"
+)
+
+# Non-CFO Act
+compliance_csv(pre_bod, "pre",
+  utils.executive_non_cfo_act,
+  "cache/pre-bod-non-cfo.csv"
+)
+compliance_csv(post_bod, "post",
+  utils.executive_non_cfo_act,
+  "cache/post-bod-non-cfo.csv"
+)
+
+# # CFO Act (minus DoD)
+# compliance_csv(pre_bod, "pre",
+#   utils.cfo_act_only_sans_dod,
+#   "cache/pre-bod-cfo-no-dod.csv"
+# )
+# compliance_csv(post_bod, "post",
+#   utils.cfo_act_only_sans_dod,
+#   "cache/post-bod-cfo-no-dod.csv"
+# )
+
+# # DoD only
+# compliance_csv(pre_bod, "pre",
+#   utils.for_agencies(["Department of Defense"]),
+#   "cache/pre-bod-dod.csv"
+# )
+# compliance_csv(post_bod, "post",
+#   utils.for_agencies(["Department of Defense"]),
+#   "cache/post-bod-dod.csv"
+# )
+
+
diff --git a/compliance/bod-18-01/download.py b/compliance/bod-18-01/download.py
@@ -0,0 +1,75 @@
+##
+# Download data from the Pulse S3 bucket to measure scans over time.
+# Must be run from a machine where "aws s3 cp" has credentials for the bucket.
+
+import os
+import subprocess
+
+AWS_REGION = "us-gov-west-1"
+BUCKET_NAME = "cg-4adefb86-dadb-4ecf-be3e-f1c7b4f6d084"
+DATA_DIR = "data/pulse"
+
+# Before BOD 18-01 was issued and before Pulse data was restructured
+pre_bod = ["2017-02-10", "2017-02-12", "2017-02-15", "2017-02-17", "2017-02-20", "2017-02-22", "2017-02-24", "2017-02-27", "2017-03-01", "2017-03-03", "2017-03-06", "2017-03-08", "2017-03-10", "2017-03-13", "2017-03-15", "2017-03-17", "2017-03-20", "2017-03-22", "2017-03-24", "2017-03-27", "2017-03-30", "2017-03-31", "2017-04-04", "2017-04-06", "2017-04-08", "2017-04-11", "2017-04-13", "2017-04-20", "2017-04-22", "2017-04-25", "2017-04-27", "2017-04-29", "2017-05-02", "2017-05-04", "2017-05-06", "2017-05-09", "2017-05-11", "2017-05-13", "2017-05-16", "2017-05-18", "2017-05-20", "2017-05-23", "2017-05-25", "2017-05-27", "2017-05-30", "2017-06-01", "2017-06-03", "2017-06-05", "2017-07-02", "2017-07-03", "2017-07-05", "2017-07-07", "2017-07-10", "2017-07-12", "2017-07-14", "2017-07-17", "2017-07-19", "2017-07-21", "2017-07-25", "2017-07-26", "2017-07-28", "2017-07-31", "2017-08-02", "2017-08-07", "2017-08-09", "2017-08-11", "2017-08-14", "2017-08-16", "2017-08-18", "2017-08-21", "2017-08-23", "2017-08-25", "2017-09-28"]
+
+# After BOD 18-01 was issued and after Pulse data was restructured
+# Exceptions:
+#  2018-01-02: no subdomains/scan/results
+post_bod = ["2017-11-20", "2017-11-25", "2017-12-11", "2017-12-13", "2017-12-15", "2017-12-16", "2017-12-17", "2017-12-18", "2017-12-19", "2017-12-20", "2017-12-21", "2017-12-22", "2017-12-23", "2017-12-24", "2017-12-25", "2017-12-26", "2017-12-27", "2017-12-28", "2017-12-29", "2017-12-30", "2017-12-31", "2018-01-01", "2018-01-03", "2018-01-04", "2018-01-05", "2018-01-06", "2018-01-07", "2018-01-08", "2018-01-09", "2018-01-10", "2018-01-11", "2018-01-12", "2018-01-13", "2018-01-14", "2018-01-15", "2018-01-16", "2018-01-17", "2018-01-18", "2018-01-19", "2018-01-20", "2018-01-21", "2018-01-22", "2018-01-23", "2018-01-24", "2018-01-25", "2018-01-26", "2018-01-27", "2018-01-28", "2018-01-29", "2018-01-30", "2018-01-31", "2018-02-01", "2018-02-02", "2018-02-03", "2018-02-04", "2018-02-05", "2018-02-06", "2018-02-07", "2018-02-08", "2018-02-09", "2018-02-10", "2018-02-11", "2018-02-12", "2018-02-13", "2018-02-14", "2018-02-15", "2018-02-16", "2018-02-17", "2018-02-18", "2018-02-19", "2018-02-20", "2018-02-21", "2018-02-22", "2018-02-23", "2018-02-24", "2018-02-25", "2018-02-26", "2018-02-27", "2018-02-28", "2018-03-01", "2018-03-02", "2018-03-03", "2018-03-04", "2018-03-05", "2018-03-06", "2018-03-07", "2018-03-08", "2018-03-09", "2018-03-10", "2018-03-11", "2018-03-15", "2018-03-16", "2018-03-17", "2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21", "2018-03-23", "2018-03-24", "2018-03-25", "2018-03-26", "2018-03-27", "2018-03-28", "2018-03-29", "2018-03-30", "2018-03-31", "2018-04-01", "2018-04-02", "2018-04-03", "2018-04-09", "2018-04-10", "2018-04-11", "2018-04-12", "2018-04-13", "2018-04-14", "2018-04-15", "2018-04-16", "2018-04-25", "2018-04-26", "2018-04-27", "2018-04-28"]
+
+def shell_out(command, env=None):
+    response = subprocess.check_output(command, shell=False, env=env)
+    output = str(response, encoding='UTF-8')
+    return output
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST:
+            pass
+        else:
+            raise
+
+
+# when = "pre" or "post"
+def download_date(date, when="post"):
+
+  def download(date, file):
+    url = "s3://%s/archive/%s/%s" % (BUCKET_NAME, date, file)
+
+    # local destinations are relative to data/
+    path = os.path.join(DATA_DIR, "%s-bod" % when, date, file)
+
+    shell_out(["aws", "s3", "cp", url, path])
+
+  try:
+    # Newer streamlined directory structure
+    if when == "post":
+      download(date, "parents/results/pshtt.csv")
+      download(date, "parents/results/sslyze.csv")
+      download(date, "parents/results/meta.json")
+      download(date, "subdomains/scan/results/pshtt.csv")
+      download(date, "subdomains/scan/results/sslyze.csv")
+      download(date, "subdomains/scan/results/meta.json")
+
+    # Older, more laborious directory structure
+    elif when == "pre":
+      download(date, "scan/pshtt.csv")
+      download(date, "scan/sslyze.csv")
+      download(date, "scan/meta.json")
+      download(date, "subdomains/scan/url/results/pshtt.csv")
+      download(date, "subdomains/scan/url/results/sslyze.csv")
+      download(date, "subdomains/scan/url/results/meta.json")
+      download(date, "subdomains/scan/censys/results/pshtt.csv")
+      download(date, "subdomains/scan/censys/results/sslyze.csv")
+      download(date, "subdomains/scan/censys/results/meta.json")
+  except subprocess.CalledProcessError:
+    print("[%s] MISSING SOMETHING" % date)
+
+
+# for date in post_bod:
+#   download_date(date, "post")
+
+# for date in pre_bod:
+#   download_date(date, "pre")