Skip to content

Commit b104d17

Browse files
author
dqw
committed
--key修改为使用正则匹配全文。使用chardet检测页面编码,由于chardet速度比较慢,增加--encoding参数可以手工制定编码
1 parent 0f2f669 commit b104d17

1 file changed

Lines changed: 13 additions & 9 deletions

File tree

‎spider.py‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,14 @@ def close(self):
4747
self.conn.close()
4848

4949
class GetHtml(threading.Thread):
50-
def __init__(self, queue_url, dict_downloaded, db, deep, key):
50+
def __init__(self, queue_url, dict_downloaded, db, deep, key, encoding):
5151
threading.Thread.__init__(self)
5252
self.queue_url = queue_url
5353
self.dict_downloaded = dict_downloaded
5454
self.db = db
5555
self.deep = deep
5656
self.key = key
57+
self.encoding = encoding
5758

5859
def run(self):
5960
while True:
@@ -77,18 +78,20 @@ def run(self):
7778
self.getLink(url, html)
7879
self.saveHtml(url, html)
7980
else:
80-
charset = chardet.detect(html)
81-
soup = BeautifulSoup(html.decode(charset['encoding'], "ignore"))
81+
if not self.encoding:
82+
charset = chardet.detect(html)
83+
self.encoding = charset['encoding']
8284

83-
if soup.findAll('meta', content=re.compile(self.key)):
84-
self.getLink(url, html, soup)
85+
match = re.search(re.compile(self.key), html.decode(self.encoding, "ignore"))
86+
if match:
87+
self.getLink(url, html)
8588
self.saveHtml(url, html)
8689
else:
8790
logging.debug("{0} ignore {1} key not match".format(self.getName(), url[1].encode("utf8")))
8891

8992
self.queue_url.task_done()
9093

91-
def getLink(self, url, html, soup = None):
94+
def getLink(self, url, html):
9295
if url[0] < self.deep:
9396
if not soup:
9497
soup = BeautifulSoup(html)
@@ -151,7 +154,7 @@ def test_sqlite(dbfile):
151154
conn.close()
152155
return True
153156

154-
def main(url, deep, thread, dbfile, logfile, loglevel, key):
157+
def main(url, deep, thread, dbfile, logfile, loglevel, key, encoding):
155158
start = time.time()
156159

157160
# logging初始化,设定日志文件名和记录级别
@@ -173,7 +176,7 @@ def main(url, deep, thread, dbfile, logfile, loglevel, key):
173176
dict_downloaded = {}
174177

175178
for i in range(thread):
176-
thread_job = GetHtml(queue_url, dict_downloaded, db, deep, key)
179+
thread_job = GetHtml(queue_url, dict_downloaded, db, deep, key, encoding)
177180
thread_job.setDaemon(True)
178181
thread_job.start()
179182

@@ -196,6 +199,7 @@ def main(url, deep, thread, dbfile, logfile, loglevel, key):
196199
parser.add_argument('-f', dest="logfile", default="spider.log", help="")
197200
parser.add_argument('-l', dest="loglevel", default="5", type=int, help="")
198201
parser.add_argument('--key', dest="key", default="", help="")
202+
parser.add_argument('--encoding', dest="encoding", default=None, help="")
199203
parser.add_argument('--testself', action="store_true", dest="testself", default="", help="")
200204
args = parser.parse_args()
201205

@@ -205,4 +209,4 @@ def main(url, deep, thread, dbfile, logfile, loglevel, key):
205209
doctest.testmod(verbose=True)
206210
else:
207211
args.key = args.key.decode("utf-8")
208-
main(args.url, args.deep, args.thread, args.dbfile, args.logfile, args.loglevel, args.key)
212+
main(args.url, args.deep, args.thread, args.dbfile, args.logfile, args.loglevel, args.key, args.encoding)

0 commit comments

Comments
 (0)