Skip to content

Commit 0c12dd5

Browse files
author
dqw
committed
添加注释
1 parent 2f8427d commit 0c12dd5

1 file changed

Lines changed: 5 additions & 0 deletions

File tree

‎utils/spider.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,11 @@ def run(self):
4242
self.logging.error("Unexpected:{0} {1}".format(url[1].encode("utf8"), str(e)))
4343
else:
4444
if self.key == "":
45+
# 下载所有页面
4546
self.getLink(url, html)
4647
self.saveHtml(url, html)
4748
else:
49+
# 下载匹配关键字的页面
4850
if not self.encoding:
4951
charset = chardet.detect(html)
5052
self.encoding = charset['encoding']
@@ -58,17 +60,20 @@ def run(self):
5860

5961
self.queue_url.task_done()
6062

63+
# 分析页面,获取链接
6164
def getLink(self, url, html):
6265
if url[0] < self.deep:
6366
soup = BeautifulSoup(html)
6467
for link in soup.findAll('a',
6568
attrs={'href': re.compile("^http://")}):
6669
href = link.get('href')
70+
# 判断该链接是否已经下载过
6771
url_hash = md5.new(href.encode("utf8")).hexdigest()
6872
if not self.dict_downloaded.has_key(url_hash):
6973
self.queue_url.put([url[0]+1, href, url_hash])
7074
self.logging.debug("{0} add href {1} to queue".format(self.getName(), href.encode("utf8")))
7175

76+
# 保存下载的页面
7277
def saveHtml(self, url, html):
7378
self.db.save(url[1], url[2], url[0], html)
7479
self.dict_downloaded[url[2]] = url[1]

0 commit comments

Comments
 (0)