Skip to content

Commit a9fe423

Browse files
author
dqw
committed
代码结构整理
1 parent 87b9857 commit a9fe423

4 files changed

Lines changed: 77 additions & 66 deletions

File tree

‎main.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import sqlite3
88
import time
99
from utils.parser import get_args
10-
from utils.log import PrintLog
1110
from utils.pool import ThreadPool
1211
from utils.spider import spider
1312

‎utils/pool.py‎

Lines changed: 2 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
import Queue
66
import md5
77
import logging
8-
import time
9-
import sqlite3
8+
from utils.progress import PrintProgress
9+
from utils.save import SaveToSqlite
1010

1111
class ThreadPool(object):
1212
def __init__(self, thread_num, args):
@@ -127,66 +127,3 @@ def run(self):
127127
except Exception,e:
128128
print str(e)
129129
break
130-
131-
132-
# 打印进度信息
133-
class PrintProgress(threading.Thread):
134-
def __init__(self, thread_pool):
135-
threading.Thread.__init__(self)
136-
self.thread_pool = thread_pool
137-
self.start()
138-
139-
def run(self):
140-
while True:
141-
thread_number = self.thread_pool.get_running()
142-
if thread_number <= 0:
143-
break
144-
145-
progress_info = self.thread_pool.get_progress_info()
146-
147-
print '总任务数:', progress_info['tasks_number']
148-
print '下载中:', thread_number
149-
print '待下载:', progress_info['work_queue_number']
150-
print '待保存:', progress_info['save_queue_number']
151-
print '---------------------------------------'
152-
153-
time.sleep(10)
154-
155-
# 保存html
156-
class SaveToSqlite(threading.Thread):
157-
def __init__(self, thread_pool, dbfile):
158-
threading.Thread.__init__(self)
159-
self.thread_pool = thread_pool
160-
self.conn = sqlite3.connect(dbfile, check_same_thread = False)
161-
#设置支持中文存储
162-
self.conn.text_factory = str
163-
self.cmd = self.conn.cursor()
164-
self.cmd.execute('''
165-
create table if not exists data(
166-
id INTEGER PRIMARY KEY AUTOINCREMENT,
167-
url text,
168-
html text
169-
)
170-
''')
171-
self.conn.commit()
172-
self.start()
173-
174-
def run(self):
175-
while True:
176-
try:
177-
url, html = self.thread_pool.get_save_task()
178-
try:
179-
self.cmd.execute("insert into data (url, html) values (?,?)", (url, html))
180-
self.conn.commit()
181-
except Exception as e:
182-
logging.error("Unexpected error:{0}".format(str(e)))
183-
except Queue.Empty:
184-
thread_number = self.thread_pool.get_running()
185-
if thread_number <= 0:
186-
self.conn.close()
187-
break
188-
except Exception,e:
189-
print str(e)
190-
break
191-
192-

‎utils/progress.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#! /usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
4+
import threading
5+
import time
6+
7+
# 打印进度信息
8+
class PrintProgress(threading.Thread):
9+
def __init__(self, thread_pool):
10+
threading.Thread.__init__(self)
11+
self.thread_pool = thread_pool
12+
self.start()
13+
14+
def run(self):
15+
while True:
16+
thread_number = self.thread_pool.get_running()
17+
if thread_number <= 0:
18+
break
19+
20+
progress_info = self.thread_pool.get_progress_info()
21+
22+
print '总任务数:', progress_info['tasks_number']
23+
print '下载中:', thread_number
24+
print '待下载:', progress_info['work_queue_number']
25+
print '待保存:', progress_info['save_queue_number']
26+
print '---------------------------------------'
27+
28+
time.sleep(10)
29+
30+

‎utils/save.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#! /usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
4+
import threading
5+
import Queue
6+
import sqlite3
7+
8+
# 保存html
9+
class SaveToSqlite(threading.Thread):
10+
def __init__(self, thread_pool, dbfile):
11+
threading.Thread.__init__(self)
12+
self.thread_pool = thread_pool
13+
self.conn = sqlite3.connect(dbfile, check_same_thread = False)
14+
#设置支持中文存储
15+
self.conn.text_factory = str
16+
self.cmd = self.conn.cursor()
17+
self.cmd.execute('''
18+
create table if not exists data(
19+
id INTEGER PRIMARY KEY AUTOINCREMENT,
20+
url text,
21+
html text
22+
)
23+
''')
24+
self.conn.commit()
25+
self.start()
26+
27+
def run(self):
28+
while True:
29+
try:
30+
url, html = self.thread_pool.get_save_task()
31+
try:
32+
self.cmd.execute("insert into data (url, html) values (?,?)", (url, html))
33+
self.conn.commit()
34+
except Exception as e:
35+
logging.error("Unexpected error:{0}".format(str(e)))
36+
except Queue.Empty:
37+
thread_number = self.thread_pool.get_running()
38+
if thread_number <= 0:
39+
self.conn.close()
40+
break
41+
except Exception,e:
42+
print str(e)
43+
break
44+
45+

0 commit comments

Comments
 (0)