import os, sys, re, datetime, argparse, traceback, json, subprocess

import html5lib

import random

basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))

if not basedir in sys.path:

sys.path.insert(0, basedir)

parser = argparse.ArgumentParser(

description="""Perform a test crawl of the project. For each found URL, the HTTP

printed - in case of errors, a stacktrace is also included.""")

parser.add_argument('urls', metavar='URL', nargs='*',

help='One or more URLs to start the crawl from')

parser.add_argument('--urls', '-u', dest='url_file',

help='File with URLs to start the crawl from')

parser.add_argument('--slow', dest='slow_threshold', type=float, default=1.0,

help='Responses taking longer than this (in seconds) results in SLOW being printed')

parser.add_argument('--settings', help='Custom settings file')

parser.add_argument('--logfile', help='Write to logfile')

parser.add_argument('--user', help='Crawl logged in as this user', default=None)

parser.add_argument('--no-follow', dest='follow', action='store_false', default=True,

help='Do not follow URLs found in fetched pages, just check the given URLs')

parser.add_argument('--validator-nu', dest='validator_nu', action='store_true',

help='Use validator.nu instead of html5lib for HTML validation')

parser.add_argument('--pedantic', action='store_true',

help='Stop the crawl on the first HTML validation issue')

parser.add_argument('--random', action='store_true',

help='Crawl URLs randomly')

parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False,

help='Run html 5 validation on all pages, without skipping similar urls. '

'(The default is to only run validation on one of /foo/1/, /foo/2/, /foo/3/, etc.)')

parser.add_argument('-v', '--verbose', action='store_true', default=False,

help='Be more verbose')

args = parser.parse_args()

os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl")

import django

import django.test

import django.core.checks

from django.conf import settings

django.setup()

import debug # pyflakes:ignore

from ietf.name.models import DocTypeName

from ietf.utils.html import unescape

from ietf.utils.test_utils import unicontent

MAX_URL_LENGTH = 500

def note(s):

if args.verbose:

sys.stderr.write(s)

sys.stderr.write('\n')

def strip_url(url):

return url

def extract_html_urls(content):

yield unescape(url)

def extract_tastypie_urls(content):

yield uri

def check_html_valid(url, response, args):

warnings += 1

def skip_extract_from(url):

return False

def skip_url(url):

return False

def log(s):

logfile.write('\n')

def get_referrers(url):

return ref_list

slow_threshold = args.slow_threshold

visited = set()

urls = {} # url -> referrer

referrers = {}

initial_urls = []

initial_urls.extend(args.urls)

if args.url_file:

with open(args.url_file) as f:

for line in f:

line = line.partition("#")[0].strip()

if line:

initial_urls.append(line)

if not initial_urls:

initial_urls.append("/")

initial_urls.append("/api/v1")

for url in initial_urls:

urls[url] = "[initial]"

parser = html5lib.HTMLParser(strict=True)

validated_urls = {'/meeting/nn/agenda/foo/': True, }

doc_types = [ t.slug for t in DocTypeName.objects.all() ]

errors = 0

warnings = 0

count = 0

start_time = datetime.datetime.now()

client = django.test.Client(Accept='text/html,text/plain,application/json')

logfile = None

if args.logfile:

logfile = open(args.logfile, "w")

if __name__ == "__main__":

if (args.user):

# log in as user, to have the respective HTML generated by the templates

response = client.post('/accounts/login/',

{'username': args.user, 'password': 'password'},

secure=True, follow=True)

if (response.status_code != 200):

log("Could not log in as %s, HTML response %d" %

(args.user, response.status_code))

sys.exit(1)

# Run django system checks and checks from ietf.checks:

error_list = django.core.checks.run_checks()

silenced = []

for i in range(len(error_list)):

if error_list[i].id in settings.SILENCED_SYSTEM_CHECKS:

silenced.append(i)

silenced.sort(reverse=True)

for i in silenced:

del error_list[i]

if error_list:

print("")

for entry in error_list:

print(entry)

while urls:

if args.random:

# popitem() is documented to be random, but really isn't

url = random.choice(urls.keys())

referrer = urls.pop(url)

else:

url, referrer = urls.popitem()

visited.add(url)

if skip_url(url):

continue

timestamp = datetime.datetime.now()

acc_time = (timestamp - start_time).total_seconds()

acc_secs = (timestamp - start_time).total_seconds()

hrs = acc_secs // (60*60)

min = (acc_secs % (60*60)) // 60

sec = acc_secs % 60

try:

request_start = datetime.datetime.now()

if args.verbose:

sys.stderr.write(url+'\n')

r = client.get(url, secure=True)

elapsed = datetime.datetime.now() - request_start

except KeyboardInterrupt:

log(" ... was fetching %s" % url)

sys.exit(1)

except:

elapsed = datetime.datetime.now() - request_start

tags = [ u"FAIL (from [ %s ])" % (",\n\t".join(get_referrers(url))) ]

log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), 500, elapsed.total_seconds(), url, " ".join(tags)))

log("=============")

log(traceback.format_exc())

log("=============")

errors += 1

else:

tags = []

if r.status_code in (301, 302):

u = strip_url(r["Location"])

if u not in visited and u not in urls:

urls[u] = referrer # referrer is original referrer, not redirected url

referrers[u] = referrer

elif r.status_code == 200:

ctype = r["Content-Type"]

if ";" in ctype:

ctype = ctype[:ctype.index(";")]

if ctype == "text/html":

try:

if args.follow and not skip_extract_from(url):

for u in extract_html_urls(unicontent(r)):

if u not in visited and u not in urls:

urls[u] = url

referrers[u] = url

check_html_valid(url, r, args)

except:

log("error extracting HTML urls from %s" % url)

log("=============")

log(traceback.format_exc())

log("=============")

elif ctype == "application/json":

try:

if args.follow:

for u in extract_tastypie_urls(unicontent(r)):

if u not in visited and u not in urls:

urls[u] = url

referrers[u] = url

except:

log("error extracting urls from %s" % url)

log("=============")

log(traceback.format_exc())

log("=============")

else:

tags.append(u"FAIL (from %s)" % (referrer, ))

errors += 1

if elapsed.total_seconds() > slow_threshold:

tags.append("SLOW")

acc_time = (timestamp - start_time).total_seconds()

acc_secs = (timestamp - start_time).total_seconds()

hrs = acc_secs // (60*60)

min = (acc_secs % (60*60)) // 60

sec = acc_secs % 60

if (len(visited) % 100) == 1:

log("\nElapsed Visited Queue Code Time Url ... Notes")

log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags)))

if ((errors or warnings) and args.pedantic):

sys.exit(1)

if logfile:

logfile.close()

sys.stderr.write("Output written to %s\n\n" % logfile.name)

if errors > 0:

sys.stderr.write("Found %s errors, grep output for FAIL for details\n" % errors)

sys.exit(1)

else:

sys.stderr.write("Found no errors.\n")

if warnings > 0:

sys.stderr.write("Found %s warnings, grep output for WARN for details\n" % warnings)

else:

sys.stderr.write("Found no warnings.\n")