11#! /usr/bin/env python
22# -*- coding: utf-8 -*-
33
4- import threading
54import urllib2
65import re
76import gzip
109from StringIO import StringIO
1110from BeautifulSoup import BeautifulSoup
1211
13- class GetHtml (threading .Thread ):
14- def __init__ (self , queue_url , dict_downloaded , db , args , logging ):
15- threading .Thread .__init__ (self )
16- self .queue_url = queue_url
17- self .dict_downloaded = dict_downloaded
18- self .db = db
19- self .deep = args .deep
20- self .key = args .key
21- self .encoding = args .encoding
22- self .logging = logging
23- self .daemon = True
24- self .start ()
25-
26- def run (self ):
27- while True :
28- url = self .queue_url .get ()
29- try :
30- response = urllib2 .urlopen (url [1 ], timeout = 20 )
31- if response .info ().get ('Content-Encoding' ) == 'gzip' :
32- buf = StringIO (response .read ())
33- f = gzip .GzipFile (fileobj = buf )
34- html = f .read ()
35- else :
36- html = response .read ()
37- except urllib2 .URLError as e :
38- self .logging .error ("URLError:{0} {1}" .format (url [1 ].encode ("utf8" ), e .reason ))
39- except urllib2 .HTTPError as e :
40- self .logging .error ("HTTPError:{0} {1}" .format (url [1 ].encode ("utf8" ), e .code ))
41- except Exception as e :
42- self .logging .error ("Unexpected:{0} {1}" .format (url [1 ].encode ("utf8" ), str (e )))
43- else :
44- if self .key == "" :
45- # 下载所有页面
46- self .getLink (url , html )
47- self .saveHtml (url , html )
48- else :
49- # 下载匹配关键字的页面
50- if not self .encoding :
51- charset = chardet .detect (html )
52- self .encoding = charset ['encoding' ]
53-
54- match = re .search (re .compile (self .key ), html .decode (self .encoding , "ignore" ))
55- if match :
56- self .getLink (url , html )
57- self .saveHtml (url , html )
58- else :
59- self .logging .debug ("{0} ignore {1} key not match" .format (self .getName (), url [1 ].encode ("utf8" )))
60-
61- self .queue_url .task_done ()
12+ def GetHtml (args ):
6213
6314 # 分析页面,获取链接
6415 def getLink (self , url , html ):
@@ -73,9 +24,38 @@ def getLink(self, url, html):
7324 self .queue_url .put ([url [0 ]+ 1 , href , url_hash ])
7425 self .logging .debug ("{0} add href {1} to queue" .format (self .getName (), href .encode ("utf8" )))
7526
76- # 保存下载的页面
77- def saveHtml (self , url , html ):
78- self .db .save (url [1 ], url [2 ], url [0 ], html )
79- self .dict_downloaded [url [2 ]] = url [1 ]
80- self .logging .debug ("{0} downloaded {1}" .format (self .getName (), url [1 ].encode ("utf8" )))
27+ url = self .queue_url .get ()
28+ try :
29+ response = urllib2 .urlopen (url [1 ], timeout = 20 )
30+ if response .info ().get ('Content-Encoding' ) == 'gzip' :
31+ buf = StringIO (response .read ())
32+ f = gzip .GzipFile (fileobj = buf )
33+ html = f .read ()
34+ else :
35+ html = response .read ()
36+ except urllib2 .URLError as e :
37+ self .logging .error ("URLError:{0} {1}" .format (url [1 ].encode ("utf8" ), e .reason ))
38+ except urllib2 .HTTPError as e :
39+ self .logging .error ("HTTPError:{0} {1}" .format (url [1 ].encode ("utf8" ), e .code ))
40+ except Exception as e :
41+ self .logging .error ("Unexpected:{0} {1}" .format (url [1 ].encode ("utf8" ), str (e )))
42+ else :
43+ if self .key == "" :
44+ # 下载所有页面
45+ self .getLink (url , html )
46+ else :
47+ # 下载匹配关键字的页面
48+ if not self .encoding :
49+ charset = chardet .detect (html )
50+ self .encoding = charset ['encoding' ]
51+
52+ match = re .search (re .compile (self .key ), html .decode (self .encoding , "ignore" ))
53+ if match :
54+ self .getLink (url , html )
55+ self .saveHtml (url , html )
56+ else :
57+ self .logging .debug ("{0} ignore {1} key not match" .format (self .getName (), url [1 ].encode ("utf8" )))
58+
59+ self .queue_url .task_done ()
60+
8161
0 commit comments