Skip to content

Commit 5280e78

Browse files
author
dqw
committed
线程调整
1 parent 6426007 commit 5280e78

4 files changed

Lines changed: 4 additions & 11 deletions

File tree

‎main.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,21 +73,14 @@ def test_sqlite(dbfile):
7373
queue_url = Queue.Queue()
7474
queue_url.put([0, args.url, md5.new(args.url).hexdigest()])
7575

76-
#url1 = "http://www.sina.com.cn"
77-
#url2 = "http://www.sohu.com"
78-
#url3 = "http://www.163.com"
79-
#queue_url.put([0, url1, md5.new(url1).hexdigest()])
80-
#queue_url.put([0, url2, md5.new(url2).hexdigest()])
81-
#queue_url.put([0, url3, md5.new(url3).hexdigest()])
82-
8376
dict_downloaded = {}
8477

8578
thread_log = PrintLog(queue_url, dict_downloaded)
8679
thread_log.setDaemon(True)
8780
thread_log.start()
8881

8982
work_manager = WorkManager(queue_url, dict_downloaded, db, args, logging)
90-
work_manager.wait_allcomplete()
83+
queue_url.join()
9184

9285
db.close()
9386

‎utils/log.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ def __init__(self, queue_url, dict_downloaded):
1212

1313
def run(self):
1414
while True:
15+
if self.queue_url.empty():
16+
break
1517
time.sleep(1)
1618
queue = self.queue_url.qsize()
1719
downloaded = len(self.dict_downloaded)

‎utils/pool.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,4 @@ def __init_thread_pool(self, thread):
1717
for i in range(thread):
1818
self.threads.append(GetHtml(self.queue_url, self.dict_downloaded, self.db, self.args, self.logging))
1919

20-
def wait_allcomplete(self):
21-
for item in self.threads:
22-
if item.isAlive():item.join()
2320

‎utils/spider.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ def __init__(self, queue_url, dict_downloaded, db, args, logging):
2020
self.key = args.key
2121
self.encoding = args.encoding
2222
self.logging = logging
23+
self.daemon = True
2324
self.start()
2425

2526
def run(self):

0 commit comments

Comments
 (0)