@@ -47,13 +47,14 @@ def close(self):
4747 self .conn .close ()
4848
4949class GetHtml (threading .Thread ):
50- def __init__ (self , queue_url , dict_downloaded , db , deep , key ):
50+ def __init__ (self , queue_url , dict_downloaded , db , deep , key , encoding ):
5151 threading .Thread .__init__ (self )
5252 self .queue_url = queue_url
5353 self .dict_downloaded = dict_downloaded
5454 self .db = db
5555 self .deep = deep
5656 self .key = key
57+ self .encoding = encoding
5758
5859 def run (self ):
5960 while True :
@@ -77,18 +78,20 @@ def run(self):
7778 self .getLink (url , html )
7879 self .saveHtml (url , html )
7980 else :
80- charset = chardet .detect (html )
81- soup = BeautifulSoup (html .decode (charset ['encoding' ], "ignore" ))
81+ if not self .encoding :
82+ charset = chardet .detect (html )
83+ self .encoding = charset ['encoding' ]
8284
83- if soup .findAll ('meta' , content = re .compile (self .key )):
84- self .getLink (url , html , soup )
85+ match = re .search (re .compile (self .key ), html .decode (self .encoding , "ignore" ))
86+ if match :
87+ self .getLink (url , html )
8588 self .saveHtml (url , html )
8689 else :
8790 logging .debug ("{0} ignore {1} key not match" .format (self .getName (), url [1 ].encode ("utf8" )))
8891
8992 self .queue_url .task_done ()
9093
91- def getLink (self , url , html , soup = None ):
94+ def getLink (self , url , html ):
9295 if url [0 ] < self .deep :
9396 if not soup :
9497 soup = BeautifulSoup (html )
@@ -151,7 +154,7 @@ def test_sqlite(dbfile):
151154 conn .close ()
152155 return True
153156
154- def main (url , deep , thread , dbfile , logfile , loglevel , key ):
157+ def main (url , deep , thread , dbfile , logfile , loglevel , key , encoding ):
155158 start = time .time ()
156159
157160 # logging初始化,设定日志文件名和记录级别
@@ -173,7 +176,7 @@ def main(url, deep, thread, dbfile, logfile, loglevel, key):
173176 dict_downloaded = {}
174177
175178 for i in range (thread ):
176- thread_job = GetHtml (queue_url , dict_downloaded , db , deep , key )
179+ thread_job = GetHtml (queue_url , dict_downloaded , db , deep , key , encoding )
177180 thread_job .setDaemon (True )
178181 thread_job .start ()
179182
@@ -196,6 +199,7 @@ def main(url, deep, thread, dbfile, logfile, loglevel, key):
196199 parser .add_argument ('-f' , dest = "logfile" , default = "spider.log" , help = "" )
197200 parser .add_argument ('-l' , dest = "loglevel" , default = "5" , type = int , help = "" )
198201 parser .add_argument ('--key' , dest = "key" , default = "" , help = "" )
202+ parser .add_argument ('--encoding' , dest = "encoding" , default = None , help = "" )
199203 parser .add_argument ('--testself' , action = "store_true" , dest = "testself" , default = "" , help = "" )
200204 args = parser .parse_args ()
201205
@@ -205,4 +209,4 @@ def main(url, deep, thread, dbfile, logfile, loglevel, key):
205209 doctest .testmod (verbose = True )
206210 else :
207211 args .key = args .key .decode ("utf-8" )
208- main (args .url , args .deep , args .thread , args .dbfile , args .logfile , args .loglevel , args .key )
212+ main (args .url , args .deep , args .thread , args .dbfile , args .logfile , args .loglevel , args .key , args . encoding )
0 commit comments