Skip to content

Commit ca5f8ff

Browse files
author
dqw
committed
新版线程池
1 parent 0c12dd5 commit ca5f8ff

3 files changed

Lines changed: 108 additions & 82 deletions

File tree

‎main.py‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
import time
1010
from utils.parser import get_args
1111
from utils.log import PrintLog
12-
from utils.save import SaveToSqlite
13-
from utils.pool import WorkManager
12+
from utils.pool import ThreadPool
13+
from utils.spider import GetHtml
1414

1515
# 测试网络连接
1616
def test_network(url):
@@ -68,21 +68,12 @@ def test_sqlite(dbfile):
6868
level = LEVELS[args.loglevel]
6969
logging.basicConfig(filename=args.logfile, level=level)
7070

71-
db = SaveToSqlite(args.dbfile, logging)
72-
73-
queue_url = Queue.Queue()
7471
queue_url.put([0, args.url, md5.new(args.url).hexdigest()])
7572

76-
dict_downloaded = {}
77-
78-
thread_log = PrintLog(queue_url, dict_downloaded)
79-
thread_log.setDaemon(True)
80-
thread_log.start()
81-
82-
work_manager = WorkManager(queue_url, dict_downloaded, db, args, logging)
83-
queue_url.join()
84-
85-
db.close()
73+
thread_pool = ThreadPool(3)
74+
thread_pool.add_job(GetHtml, args.url)
75+
thread_pool.start_job()
76+
thread_pool.wait_allcomplete()
8677

8778
print "downloaded: {0} Elapsed Time: {1}".format(len(dict_downloaded), time.time()-start)
8879

‎utils/pool.py‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,75 @@
11
#! /usr/bin/env python
22
# -*- coding: utf-8 -*-
33

4-
from utils.spider import GetHtml
4+
class ThreadPool(object):
5+
def __init__(self, thread_num=2):
56

6-
class WorkManager(object):
7-
def __init__(self, queue_url, dict_downloaded, db, args, logging):
7+
self.work_queue = Queue.Queue()
88
self.threads = []
9-
self.queue_url = queue_url
10-
self.dict_downloaded = dict_downloaded
11-
self.db = db
12-
self.args = args
13-
self.logging = logging
14-
self.__init_thread_pool(args.thread)
9+
self.running = 0
10+
self.__init_thread_pool(thread_num)
1511

16-
def __init_thread_pool(self, thread):
17-
for i in range(thread):
18-
self.threads.append(GetHtml(self.queue_url, self.dict_downloaded, self.db, self.args, self.logging))
12+
def __init_thread_pool(self, thread_num):
13+
for i in range(thread_num):
14+
self.threads.append(WorkThread(self))
1915

16+
def add_job(self, func, url):
17+
self.work_queue.put((func, url))
18+
19+
def get_task(self):
20+
#只使用一个线程,能正常退出
21+
job = self.work_queue.get(block=False)
22+
#多线程但是不退出
23+
#job = self.work_queue.get()
24+
25+
return job
26+
27+
def task_done(self):
28+
self.work_queue.task_done()
29+
30+
def start_job(self):
31+
for item in self.threads:
32+
item.start()
33+
34+
def increase_running(self):
35+
self.running += 1
36+
37+
def decrease_running(self):
38+
self.running -= 1
39+
40+
def get_running(self):
41+
return self.running
42+
43+
def check_queue(self):
44+
return self.work_queue.qsize()
45+
46+
def wait_allcomplete(self):
47+
for item in self.threads:
48+
if item.isAlive():
49+
item.join()
50+
51+
class WorkThread(threading.Thread):
52+
def __init__(self, thread_pool):
53+
threading.Thread.__init__(self)
54+
self.thread_pool = thread_pool
55+
self.stop = False
56+
57+
def run(self):
58+
while True:
59+
try:
60+
do, url = self.thread_pool.get_task()
61+
self.thread_pool.increase_running()
62+
new_jobs = do(url)
63+
if new_jobs:
64+
for url in new_jobs:
65+
self.thread_pool.add_job(do, url)
66+
67+
self.thread_pool.decrease_running()
68+
self.thread_pool.task_done()
69+
except Queue.Empty:
70+
if self.thread_pool.get_running() <= 0:
71+
break
72+
except Exception,e:
73+
print str(e)
74+
break
2075

‎utils/spider.py‎

Lines changed: 35 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#! /usr/bin/env python
22
# -*- coding: utf-8 -*-
33

4-
import threading
54
import urllib2
65
import re
76
import gzip
@@ -10,55 +9,7 @@
109
from StringIO import StringIO
1110
from BeautifulSoup import BeautifulSoup
1211

13-
class GetHtml(threading.Thread):
14-
def __init__(self, queue_url, dict_downloaded, db, args, logging):
15-
threading.Thread.__init__(self)
16-
self.queue_url = queue_url
17-
self.dict_downloaded = dict_downloaded
18-
self.db = db
19-
self.deep = args.deep
20-
self.key = args.key
21-
self.encoding = args.encoding
22-
self.logging = logging
23-
self.daemon = True
24-
self.start()
25-
26-
def run(self):
27-
while True:
28-
url = self.queue_url.get()
29-
try:
30-
response = urllib2.urlopen(url[1], timeout=20)
31-
if response.info().get('Content-Encoding') == 'gzip':
32-
buf = StringIO(response.read())
33-
f = gzip.GzipFile(fileobj=buf)
34-
html = f.read()
35-
else:
36-
html = response.read()
37-
except urllib2.URLError as e:
38-
self.logging.error("URLError:{0} {1}".format(url[1].encode("utf8"), e.reason))
39-
except urllib2.HTTPError as e:
40-
self.logging.error("HTTPError:{0} {1}".format(url[1].encode("utf8"), e.code))
41-
except Exception as e:
42-
self.logging.error("Unexpected:{0} {1}".format(url[1].encode("utf8"), str(e)))
43-
else:
44-
if self.key == "":
45-
# 下载所有页面
46-
self.getLink(url, html)
47-
self.saveHtml(url, html)
48-
else:
49-
# 下载匹配关键字的页面
50-
if not self.encoding:
51-
charset = chardet.detect(html)
52-
self.encoding = charset['encoding']
53-
54-
match = re.search(re.compile(self.key), html.decode(self.encoding, "ignore"))
55-
if match:
56-
self.getLink(url, html)
57-
self.saveHtml(url, html)
58-
else:
59-
self.logging.debug("{0} ignore {1} key not match".format(self.getName(), url[1].encode("utf8")))
60-
61-
self.queue_url.task_done()
12+
def GetHtml(args):
6213

6314
# 分析页面,获取链接
6415
def getLink(self, url, html):
@@ -73,9 +24,38 @@ def getLink(self, url, html):
7324
self.queue_url.put([url[0]+1, href, url_hash])
7425
self.logging.debug("{0} add href {1} to queue".format(self.getName(), href.encode("utf8")))
7526

76-
# 保存下载的页面
77-
def saveHtml(self, url, html):
78-
self.db.save(url[1], url[2], url[0], html)
79-
self.dict_downloaded[url[2]] = url[1]
80-
self.logging.debug("{0} downloaded {1}".format(self.getName(), url[1].encode("utf8")))
27+
url = self.queue_url.get()
28+
try:
29+
response = urllib2.urlopen(url[1], timeout=20)
30+
if response.info().get('Content-Encoding') == 'gzip':
31+
buf = StringIO(response.read())
32+
f = gzip.GzipFile(fileobj=buf)
33+
html = f.read()
34+
else:
35+
html = response.read()
36+
except urllib2.URLError as e:
37+
self.logging.error("URLError:{0} {1}".format(url[1].encode("utf8"), e.reason))
38+
except urllib2.HTTPError as e:
39+
self.logging.error("HTTPError:{0} {1}".format(url[1].encode("utf8"), e.code))
40+
except Exception as e:
41+
self.logging.error("Unexpected:{0} {1}".format(url[1].encode("utf8"), str(e)))
42+
else:
43+
if self.key == "":
44+
# 下载所有页面
45+
self.getLink(url, html)
46+
else:
47+
# 下载匹配关键字的页面
48+
if not self.encoding:
49+
charset = chardet.detect(html)
50+
self.encoding = charset['encoding']
51+
52+
match = re.search(re.compile(self.key), html.decode(self.encoding, "ignore"))
53+
if match:
54+
self.getLink(url, html)
55+
self.saveHtml(url, html)
56+
else:
57+
self.logging.debug("{0} ignore {1} key not match".format(self.getName(), url[1].encode("utf8")))
58+
59+
self.queue_url.task_done()
60+
8161

0 commit comments

Comments
 (0)