Makefiles: Use wget instead of httrack

p12tic · p12tic · commit 708fa3412038 · 2014-07-01T00:40:01.000+03:00
Httrack does not name downloaded files consistently. Also, it occasionally
manages to miss several files. Wget has needed filtering features since
1.15, so there no longer are reasons that prevent its usage.
diff --git a/Makefile b/Makefile
@@ -169,22 +169,17 @@ source:
 	mkdir "reference"
 
 	pushd "reference" > /dev/null; \
-	httrack http://en.cppreference.com/w/ -k --near --include-query-string \
-	  -* +en.cppreference.com/* +upload.cppreference.com/* -*index.php\?* \
-	  -*/Special:* -*/Talk:* -*/Help:* -*/File:* -*/Cppreference:* -*/WhatLinksHere:* \
-	  -*/Template:* -*/Category:* -*action=* -*printable=* \
-	  -en.cppreference.com/book/* -en.cppreference.com/book \
-	  +*MediaWiki:Common.css* +*MediaWiki:Print.css* +*MediaWiki:Vector.css* \
-	  -*MediaWiki:Geshi.css* "+*title=-&action=raw*" --timeout=180 --retries=10 ;\
+	regex=".*index\\.php.*|.*/Special:.*|.*/Talk:.*" \
+	regex+="|.*/Help:.*|.*/File:.*|.*/Cppreference:.*" \
+	regex+="|.*/WhatLinksHere:.*|.*/Template:.*|.*/Category:.*" \
+	regex+="|.*action=.*|.*printable=.*|.*en.cppreference.com/book.*" ; \
+	echo $$regex ; \
+	wget --adjust-extension --page-requisites --convert-links \
+	  --force-directories --recursive --level=15 \
+	  --span-hosts --domains=en.cppreference.com,upload.cppreference.com \
+	  --reject-regex $$regex \
+	  --timeout=180 --no-verbose \
+	  --retry-connrefused --waitretry=1 --read-timeout=20 \
+	  http://en.cppreference.com/w/ ; \
 	popd > /dev/null
 
-	#delete useless files
-	rm -rf "reference/hts-cache"
-	rm -f "reference/backblue.gif"
-	rm -f "reference/fade.gif"
-	rm -f "reference/hts-log.txt"
-	rm -f "reference/index.html"
-
-	#download files that httrack has forgotten
-	./httrack-workarounds.py
-
diff --git a/httrack-workarounds.py b/httrack-workarounds.py
diff --git a/preprocess.py b/preprocess.py
@@ -75,9 +75,6 @@
     for filename in fnmatch.filter(filenames, '*.css'):
         css_files.append(os.path.join(root, filename))
 
-#
-r1 = re.compile('<!-- Added by HTTrack -->.*?<!-- \/Added by HTTrack -->')
-r2 = re.compile('<!-- Mirrored from .*?-->')
 
 #temporary fix
 r3 = re.compile('<style[^<]*?<[^<]*?MediaWiki:Geshi\.css[^<]*?<\/style>', re.MULTILINE)
