Merge pull request mattmakai#198 from huangsam/add-bad-locations

mattmakai · web-flow · commit 646e1d6640e6 · 2019-01-08T10:25:57.000-05:00
Add support for bad url locations
diff --git a/check_urls.py b/check_urls.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python
-from concurrent import futures
 import multiprocessing as mp
 import os
 import json
 import uuid
+from concurrent import futures
+from collections import defaultdict
 
 from bs4 import BeautifulSoup
 from markdown import markdown
@@ -18,20 +19,23 @@
 URL_BOT_ID = f'Bot {str(uuid.uuid4())}'
 
 
-def extract_urls_from_html(content, all_urls):
+def extract_urls_from_html(content):
     soup = BeautifulSoup(content, 'html.parser')
+    html_urls = set()
     for a in soup.find_all('a', href=True):
         url = a['href']
         if url.startswith('http'):
-            all_urls.add(url)
+            html_urls.add(url)
+    return html_urls
 
 
 def extract_urls(discover_path):
     exclude = ['.git', '.vscode']
-    all_urls = set()
+    all_urls = defaultdict(list)
     max_strlen = -1
     for root, dirs, files in os.walk(discover_path, topdown=True):
         dirs[:] = [d for d in dirs if d not in exclude]
+        short_root = root.lstrip(discover_path)
         for file in files:
             output = f'Currently checking: file={file}'
             file_path = os.path.join(root, file)
@@ -40,10 +44,14 @@ def extract_urls(discover_path):
             print(output.ljust(max_strlen), end='\r')
             if file_path.endswith('.html'):
                 content = open(file_path)
-                extract_urls_from_html(content, all_urls)
+                extract_urls_from_html(content)
             elif file_path.endswith('.markdown'):
                 content = markdown(open(file_path).read())
-                extract_urls_from_html(content, all_urls)
+            else:
+                continue
+            html_urls = extract_urls_from_html(content)
+            for url in html_urls:
+                all_urls[url].append(os.path.join(short_root, file))
     return all_urls
 
 
@@ -87,20 +95,27 @@ def bad_url(url_status):
 
 def main():
     print('Extract urls...')
-    all_urls = extract_urls(os.getcwd())
+    all_urls = extract_urls(os.getcwd() + os.path.sep + 'content')
     print('\nCheck urls...')
-    bad_urls = {}
+    bad_url_status = {}
     url_id = 1
     max_strlen = -1
-    for url_path, url_status in run_workers(get_url_status, all_urls):
+    for url_path, url_status in run_workers(get_url_status, all_urls.keys()):
         output = f'Currently checking: id={url_id} host={urllib3.util.parse_url(url_path).host}'
         if max_strlen < len(output):
             max_strlen = len(output)
         print(output.ljust(max_strlen), end='\r')
         if bad_url(url_status) is True:
-            bad_urls[url_path] = url_status
+            bad_url_status[url_path] = url_status
         url_id += 1
-    print(f'\nBad urls: {json.dumps(bad_urls, indent=4)}')
+    bad_url_location = {
+        bad_url: all_urls[bad_url]
+        for bad_url in bad_url_status
+    }
+    status_content = json.dumps(bad_url_status, indent=4)
+    location_content = json.dumps(bad_url_location, indent=4)
+    print(f'\nBad url status: {status_content}')
+    print(f'\nBad url locations: {location_content}')
 
 
 if __name__ == '__main__':
