|
5 | 5 | import Queue |
6 | 6 | import md5 |
7 | 7 | import logging |
8 | | -import time |
9 | | -import sqlite3 |
| 8 | +from utils.progress import PrintProgress |
| 9 | +from utils.save import SaveToSqlite |
10 | 10 |
|
11 | 11 | class ThreadPool(object): |
12 | 12 | def __init__(self, thread_num, args): |
@@ -127,66 +127,3 @@ def run(self): |
127 | 127 | except Exception,e: |
128 | 128 | print str(e) |
129 | 129 | break |
130 | | - |
131 | | - |
132 | | -# 打印进度信息 |
133 | | -class PrintProgress(threading.Thread): |
134 | | - def __init__(self, thread_pool): |
135 | | - threading.Thread.__init__(self) |
136 | | - self.thread_pool = thread_pool |
137 | | - self.start() |
138 | | - |
139 | | - def run(self): |
140 | | - while True: |
141 | | - thread_number = self.thread_pool.get_running() |
142 | | - if thread_number <= 0: |
143 | | - break |
144 | | - |
145 | | - progress_info = self.thread_pool.get_progress_info() |
146 | | - |
147 | | - print '总任务数:', progress_info['tasks_number'] |
148 | | - print '下载中:', thread_number |
149 | | - print '待下载:', progress_info['work_queue_number'] |
150 | | - print '待保存:', progress_info['save_queue_number'] |
151 | | - print '---------------------------------------' |
152 | | - |
153 | | - time.sleep(10) |
154 | | - |
155 | | -# 保存html |
156 | | -class SaveToSqlite(threading.Thread): |
157 | | - def __init__(self, thread_pool, dbfile): |
158 | | - threading.Thread.__init__(self) |
159 | | - self.thread_pool = thread_pool |
160 | | - self.conn = sqlite3.connect(dbfile, check_same_thread = False) |
161 | | - #设置支持中文存储 |
162 | | - self.conn.text_factory = str |
163 | | - self.cmd = self.conn.cursor() |
164 | | - self.cmd.execute(''' |
165 | | - create table if not exists data( |
166 | | - id INTEGER PRIMARY KEY AUTOINCREMENT, |
167 | | - url text, |
168 | | - html text |
169 | | - ) |
170 | | - ''') |
171 | | - self.conn.commit() |
172 | | - self.start() |
173 | | - |
174 | | - def run(self): |
175 | | - while True: |
176 | | - try: |
177 | | - url, html = self.thread_pool.get_save_task() |
178 | | - try: |
179 | | - self.cmd.execute("insert into data (url, html) values (?,?)", (url, html)) |
180 | | - self.conn.commit() |
181 | | - except Exception as e: |
182 | | - logging.error("Unexpected error:{0}".format(str(e))) |
183 | | - except Queue.Empty: |
184 | | - thread_number = self.thread_pool.get_running() |
185 | | - if thread_number <= 0: |
186 | | - self.conn.close() |
187 | | - break |
188 | | - except Exception,e: |
189 | | - print str(e) |
190 | | - break |
191 | | - |
192 | | - |
0 commit comments