Merge pull request p12tic#65 from p12tic/concurrent-preprocess

p12tic · web-flow · commit 8b1e5fab51b7 · 2018-10-01T20:26:58.000-07:00
Preprocess: Run HTML preprocessing concurrently
diff --git a/commands/preprocess.py b/commands/preprocess.py
@@ -18,6 +18,7 @@
 #   along with this program.  If not, see http://www.gnu.org/licenses/.
 
 import fnmatch
+import io
 from lxml import etree
 import re
 import os
@@ -313,6 +314,7 @@ def remove_ads(html):
 def preprocess_html_file(root, fn, rename_map):
     parser = etree.HTMLParser()
     html = etree.parse(fn, parser)
+    output = io.StringIO()
 
     # remove external links to unused resources
     for el in html.xpath('/html/head/link'):
@@ -332,9 +334,10 @@ def preprocess_html_file(root, fn, rename_map):
             el.set('href', transform_link(rename_map, el.get('href'), fn, root))
 
     for err in parser.error_log:
-        print("HTML WARN: {0}".format(err))
+        print("HTML WARN: {0}".format(err), file=output)
 
     html.write(fn, encoding='utf-8', method='html')
+    return output.getvalue()
 
 def preprocess_css_file(fn):
     f = open(fn, "r", encoding='utf-8')
diff --git a/preprocess.py b/preprocess.py
@@ -19,6 +19,7 @@
 
 from commands import preprocess
 import argparse
+import concurrent.futures
 import os
 import shutil
 
@@ -41,6 +42,18 @@ def main():
     preprocess.rename_files(rename_map)
 
     # clean the html files
+    file_list = preprocess.find_html_files(root)
+
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        futures = [executor.submit(preprocess.preprocess_html_file, root, fn,
+                                   rename_map)
+                   for fn in enumerate(file_list)]
+
+        for future in futures:
+            output = future.result()
+            if len(output) > 0:
+                print(output)
+
     for fn in preprocess.find_html_files(root):
         preprocess.preprocess_html_file(root, fn, rename_map)
 
