Skip to content

Commit cd0bfbd

Browse files
author
dqw
committed
日志整理
1 parent a9fe423 commit cd0bfbd

5 files changed

Lines changed: 28 additions & 16 deletions

File tree

‎main.py‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,19 @@ def test_sqlite(dbfile):
6666
level = LEVELS[args.loglevel]
6767
logging.basicConfig(filename=args.logfile, level=level)
6868

69-
thread_pool = ThreadPool(args.thread, args)
70-
thread_pool.add_task(spider, args.url, 0)
71-
thread_pool.start_task()
72-
thread_pool.wait_all_complete()
69+
if args.url != '':
70+
thread_pool = ThreadPool(args.thread, args)
71+
thread_pool.add_task(spider, args.url, 0)
72+
thread_pool.start_task()
73+
thread_pool.wait_all_complete()
7374

74-
progress_info = thread_pool.get_progress_info()
75-
print "总任务数:",progress_info['tasks_number']
76-
print "成功下载:",progress_info['success']
77-
print "下载失败:",progress_info['failure']
78-
print "花费时间: {0} 秒".format(time.time()-start)
75+
progress_info = thread_pool.get_progress_info()
76+
print "总任务数:",progress_info['tasks_number']
77+
print "成功下载:",progress_info['success']
78+
print "下载失败:",progress_info['failure']
79+
print "花费时间: {0} 秒".format(time.time()-start)
80+
else:
81+
logging.critical("请使用-u参数指定初始url")
82+
print "请使用-u参数指定初始url"
83+
7984

‎utils/parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
def get_args():
77
# 参数处理
88
parser = argparse.ArgumentParser()
9-
parser.add_argument('-u', dest="url", default="http://www.baidu.com.cn", help="爬虫起始地址")
9+
parser.add_argument('-u', dest="url", default="", help="爬虫起始地址")
1010
parser.add_argument("-d", type=int, dest="deep", default=0, help="爬取深度,起始地址为第0级")
1111
parser.add_argument("--thread", type=int, dest="thread", default=10, help="爬虫使用的线程数")
1212
parser.add_argument('--dbfile', dest="dbfile", default="spider.db", help="存放数据库(sqlite)文件名")

‎utils/pool.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ def __init__(self, thread_num, args):
1919
self.failure = 0
2020
self.success = 0
2121
self.tasks = {}
22+
self.thread_name = threading.current_thread().getName()
2223
self.__init_thread_pool(thread_num)
2324

2425
def __init_thread_pool(self, thread_num):
@@ -31,6 +32,7 @@ def add_task(self, func, url, deep):
3132
if not self.tasks.has_key(url_hash):
3233
self.tasks[url_hash] = url
3334
self.work_queue.put((func, url, deep))
35+
logging.info("{0} add task {1}".format(self.thread_name, url.encode("utf8")))
3436

3537
def get_task(self):
3638
#只使用一个线程,能正常退出
@@ -51,6 +53,8 @@ def start_task(self):
5153
# 开始保存html
5254
SaveToSqlite(self, self.args.dbfile)
5355

56+
logging.debug("task start")
57+
5458
def increase_success(self):
5559
self.success += 1
5660

‎utils/progress.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
import threading
55
import time
6+
import logging
67

78
# 打印进度信息
89
class PrintProgress(threading.Thread):
@@ -25,6 +26,8 @@ def run(self):
2526
print '待保存:', progress_info['save_queue_number']
2627
print '---------------------------------------'
2728

29+
logging.debug("print progress")
30+
2831
time.sleep(10)
2932

3033

‎utils/spider.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,13 @@ def get_html(url, args, flag_get_new_link):
4040
else:
4141
html = response.read()
4242
except urllib2.URLError as e:
43-
logging.error("{0} URLError: {1}".format(url.encode("utf8"), e.reason))
43+
logging.warning("{0} URLError: {1}".format(url.encode("utf8"), e.reason))
4444
except urllib2.HTTPError as e:
45-
logging.error("{0} HTTPError: {1}".format(url.encode("utf8"), e.code))
45+
logging.warning("{0} HTTPError: {1}".format(url.encode("utf8"), e.code))
4646
except Exception as e:
47-
logging.error("{0} Unexpected: {1}".format(url.encode("utf8"), str(e)))
47+
logging.warning("{0} Unexpected: {1}".format(url.encode("utf8"), str(e)))
4848
else:
49-
logging.debug("{0} downloaded {1}".format(thread_name, url.encode("utf8")))
49+
logging.info("{0} downloaded {1}".format(thread_name, url.encode("utf8")))
5050

5151
if args.key == "":
5252
if flag_get_new_link:
@@ -59,10 +59,10 @@ def get_html(url, args, flag_get_new_link):
5959

6060
match = re.search(re.compile(args.key), html.decode(args.encoding, "ignore"))
6161
if match and flag_get_new_link:
62-
logging.debug("{0} {1} match key".format(thread_name, url.encode("utf8")))
62+
logging.info("{0} {1} match key".format(thread_name, url.encode("utf8")))
6363
new_link = get_link(html)
6464
else:
65-
logging.debug("{0} {1} not match key".format(thread_name, url.encode("utf8")))
65+
logging.info("{0} {1} not match key".format(thread_name, url.encode("utf8")))
6666
finally:
6767
return html, new_link
6868

0 commit comments

Comments
 (0)