Skip to content

Commit 34f8f79

Browse files
author
dqw
committed
代码整理
1 parent c5ca47b commit 34f8f79

2 files changed

Lines changed: 9 additions & 15 deletions

File tree

‎main.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
import sqlite3
99
import time
1010
from utils.parser import get_args
11-
from utils.pool import WorkManager
12-
from utils.save import SaveToSqlite
1311
from utils.log import PrintLog
12+
from utils.save import SaveToSqlite
13+
from utils.spider import GetHtml
14+
from utils.pool import WorkManager
1415

1516
# 测试网络连接
1617
def test_network(url):
@@ -79,7 +80,8 @@ def test_sqlite(dbfile):
7980
thread_log.setDaemon(True)
8081
thread_log.start()
8182

82-
work_manager = WorkManager(args, queue_url, dict_downloaded, db, logging)
83+
work = GetHtml(queue_url, dict_downloaded, db, args, logging)
84+
work_manager = WorkManager(work, args.thread)
8385
work_manager.wait_allcomplete()
8486

8587
db.close()

‎utils/pool.py‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,15 @@
11
#! /usr/bin/env python
22
# -*- coding: utf-8 -*-
3-
from spider import GetHtml
43

54
class WorkManager(object):
6-
def __init__(self, args, queue_url, dict_downloaded, db, logging):
5+
def __init__(self, work, thread):
76
self.threads = []
8-
self.args = args
9-
self.queue_url = queue_url
10-
self.dict_downloaded = dict_downloaded
11-
self.db = db
12-
self.logging = logging
13-
self.__init_thread_pool(args.thread)
7+
self.work = work
8+
self.__init_thread_pool(thread)
149

1510
def __init_thread_pool(self, thread):
1611
for i in range(thread):
17-
self.threads.append(GetHtml(self.queue_url, self.dict_downloaded, self.db, self.args, self.logging))
18-
19-
def check_queue(self):
20-
return self.queue_url.qsize()
12+
self.threads.append(self.work)
2113

2214
def wait_allcomplete(self):
2315
for item in self.threads:

0 commit comments

Comments
 (0)