|
1 | 1 | # coding=utf-8 |
| 2 | +import logging |
2 | 3 | import os |
3 | 4 | import re |
4 | 5 | import time |
5 | | -import logging |
| 6 | + |
| 7 | +try: |
| 8 | + from urllib.parse import urlparse # py3 |
| 9 | +except: |
| 10 | + from urlparse import urlparse # py2 |
| 11 | + |
6 | 12 | import pdfkit |
7 | 13 | import requests |
8 | 14 | from bs4 import BeautifulSoup |
|
21 | 27 | """ |
22 | 28 |
|
23 | 29 |
|
24 | | -def parse_url_to_html(url, name): |
| 30 | +class Crawler(object): |
25 | 31 | """ |
26 | | - 解析URL，返回HTML内容 |
27 | | - :param url:解析的url |
28 | | - :param name: 保存的html文件名 |
29 | | - :return: html |
| 32 | + 爬虫基类，所有爬虫都应该继承此类 |
30 | 33 | """ |
31 | | - try: |
| 34 | + name = None |
| 35 | + |
| 36 | + def __init__(self, name, start_url): |
| 37 | + """ |
| 38 | + 初始化 |
| 39 | + :param name: 保存问的PDF文件名,不需要后缀名 |
| 40 | + :param start_url: 爬虫入口URL |
| 41 | + """ |
| 42 | + self.name = name |
| 43 | + self.start_url = start_url |
| 44 | + self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url)) |
| 45 | + |
| 46 | + def crawl(self, url): |
| 47 | + """ |
| 48 | + pass |
| 49 | + :return: |
| 50 | + """ |
| 51 | + print(url) |
32 | 52 | response = requests.get(url) |
33 | | - soup = BeautifulSoup(response.content, 'html.parser') |
34 | | - # 正文 |
35 | | - body = soup.find_all(class_="x-wiki-content")[0] |
36 | | - # 标题 |
37 | | - title = soup.find('h4').get_text() |
38 | | - |
39 | | - # 标题加入到正文的最前面，居中显示 |
40 | | - center_tag = soup.new_tag("center") |
41 | | - title_tag = soup.new_tag('h1') |
42 | | - title_tag.string = title |
43 | | - center_tag.insert(1, title_tag) |
44 | | - body.insert(1, center_tag) |
45 | | - html = str(body) |
46 | | - # body中的img标签的src相对路径的改成绝对路径 |
47 | | - pattern = "(<img .*?src=\")(.*?)(\")" |
48 | | - |
49 | | - def func(m): |
50 | | - if not m.group(3).startswith("http"): |
51 | | - rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3) |
52 | | - return rtn |
53 | | - else: |
54 | | - return m.group(1)+m.group(2)+m.group(3) |
55 | | - html = re.compile(pattern).sub(func, html) |
56 | | - html = html_template.format(content=html) |
57 | | - html = html.encode("utf-8") |
58 | | - with open(name, 'wb') as f: |
59 | | - f.write(html) |
60 | | - return name |
61 | | - |
62 | | - except Exception as e: |
63 | | - |
64 | | - logging.error("解析错误", exc_info=True) |
65 | | - |
66 | | - |
67 | | -def get_url_list(): |
| 53 | + return response |
| 54 | + |
| 55 | + def parse_menu(self, response): |
| 56 | + """ |
| 57 | + 解析目录结构,获取所有URL目录列表:由子类实现 |
| 58 | + :param response 爬虫返回的response对象 |
| 59 | + :return: url 可迭代对象(iterable) 列表,生成器,元组都可以 |
| 60 | + """ |
| 61 | + raise NotImplementedError |
| 62 | + |
| 63 | + def parse_body(self, response): |
| 64 | + """ |
| 65 | + 解析正文,由子类实现 |
| 66 | + :param response: 爬虫返回的response对象 |
| 67 | + :return: 返回经过处理的html文本 |
| 68 | + """ |
| 69 | + raise NotImplementedError |
| 70 | + |
| 71 | + def run(self): |
| 72 | + start = time.time() |
| 73 | + options = { |
| 74 | + 'page-size': 'Letter', |
| 75 | + 'margin-top': '0.75in', |
| 76 | + 'margin-right': '0.75in', |
| 77 | + 'margin-bottom': '0.75in', |
| 78 | + 'margin-left': '0.75in', |
| 79 | + 'encoding': "UTF-8", |
| 80 | + 'custom-header': [ |
| 81 | + ('Accept-Encoding', 'gzip') |
| 82 | + ], |
| 83 | + 'cookie': [ |
| 84 | + ('cookie-name1', 'cookie-value1'), |
| 85 | + ('cookie-name2', 'cookie-value2'), |
| 86 | + ], |
| 87 | + 'outline-depth': 10, |
| 88 | + } |
| 89 | + htmls = [] |
| 90 | + for index, url in enumerate(self.parse_menu(self.crawl(self.start_url))): |
| 91 | + html = self.parse_body(self.crawl(url)) |
| 92 | + f_name = ".".join([str(index), "html"]) |
| 93 | + with open(f_name, 'wb') as f: |
| 94 | + f.write(html) |
| 95 | + htmls.append(f_name) |
| 96 | + |
| 97 | + pdfkit.from_file(htmls, self.name + ".pdf", options=options) |
| 98 | + for html in htmls: |
| 99 | + os.remove(html) |
| 100 | + total_time = time.time() - start |
| 101 | + print(u"总共耗时：%f 秒" % total_time) |
| 102 | + |
| 103 | + |
| 104 | +class LiaoxuefengPythonCrawler(Crawler): |
68 | 105 | """ |
69 | | - 获取所有URL目录列表 |
70 | | - :return: |
| 106 | + 廖雪峰Python3教程 |
71 | 107 | """ |
72 | | - response = requests.get("http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000") |
73 | | - soup = BeautifulSoup(response.content, "html.parser") |
74 | | - menu_tag = soup.find_all(class_="uk-nav uk-nav-side")[1] |
75 | | - urls = [] |
76 | | - for li in menu_tag.find_all("li"): |
77 | | - url = "http://www.liaoxuefeng.com" + li.a.get('href') |
78 | | - urls.append(url) |
79 | | - return urls |
80 | 108 |
|
81 | | - |
82 | | -def save_pdf(htmls, file_name): |
83 | | - """ |
84 | | - 把所有html文件保存到pdf文件 |
85 | | - :param htmls: html文件列表 |
86 | | - :param file_name: pdf文件名 |
87 | | - :return: |
88 | | - """ |
89 | | - options = { |
90 | | - 'page-size': 'Letter', |
91 | | - 'margin-top': '0.75in', |
92 | | - 'margin-right': '0.75in', |
93 | | - 'margin-bottom': '0.75in', |
94 | | - 'margin-left': '0.75in', |
95 | | - 'encoding': "UTF-8", |
96 | | - 'custom-header': [ |
97 | | - ('Accept-Encoding', 'gzip') |
98 | | - ], |
99 | | - 'cookie': [ |
100 | | - ('cookie-name1', 'cookie-value1'), |
101 | | - ('cookie-name2', 'cookie-value2'), |
102 | | - ], |
103 | | - 'outline-depth': 10, |
104 | | - } |
105 | | - pdfkit.from_file(htmls, file_name, options=options) |
106 | | - |
107 | | - |
108 | | -def main(): |
109 | | - start = time.time() |
110 | | - urls = get_url_list() |
111 | | - file_name = u"liaoxuefeng_Python3_tutorial.pdf" |
112 | | - htmls = [parse_url_to_html(url, str(index) + ".html") for index, url in enumerate(urls)] |
113 | | - save_pdf(htmls, file_name) |
114 | | - |
115 | | - for html in htmls: |
116 | | - os.remove(html) |
117 | | - |
118 | | - total_time = time.time() - start |
119 | | - print(u"总共耗时：%f 秒" % total_time) |
| 109 | + def parse_menu(self, response): |
| 110 | + """ |
| 111 | + 解析目录结构,获取所有URL目录列表 |
| 112 | + :param response 爬虫返回的response对象 |
| 113 | + :return: url生成器 |
| 114 | + """ |
| 115 | + soup = BeautifulSoup(response.content, "html.parser") |
| 116 | + menu_tag = soup.find_all(class_="uk-nav uk-nav-side")[1] |
| 117 | + for li in menu_tag.find_all("li"): |
| 118 | + url = li.a.get("href") |
| 119 | + if not url.startswith("http"): |
| 120 | + url = "".join([self.domain, url]) # 补全为全路径 |
| 121 | + yield url |
| 122 | + |
| 123 | + def parse_body(self, response): |
| 124 | + """ |
| 125 | + 解析正文 |
| 126 | + :param response: 爬虫返回的response对象 |
| 127 | + :return: 返回处理后的html文本 |
| 128 | + """ |
| 129 | + try: |
| 130 | + soup = BeautifulSoup(response.content, 'html.parser') |
| 131 | + body = soup.find_all(class_="x-wiki-content")[0] |
| 132 | + |
| 133 | + # 加入标题, 居中显示 |
| 134 | + title = soup.find('h4').get_text() |
| 135 | + center_tag = soup.new_tag("center") |
| 136 | + title_tag = soup.new_tag('h1') |
| 137 | + title_tag.string = title |
| 138 | + center_tag.insert(1, title_tag) |
| 139 | + body.insert(1, center_tag) |
| 140 | + |
| 141 | + html = str(body) |
| 142 | + # body中的img标签的src相对路径的改成绝对路径 |
| 143 | + pattern = "(<img .*?src=\")(.*?)(\")" |
| 144 | + |
| 145 | + def func(m): |
| 146 | + if not m.group(3).startswith("http"): |
| 147 | + rtn = "".join([m.group(1), self.domain, m.group(2), m.group(3)]) |
| 148 | + return rtn |
| 149 | + else: |
| 150 | + return "".join([m.group(1), m.group(2), m.group(3)]) |
| 151 | + |
| 152 | + html = re.compile(pattern).sub(func, html) |
| 153 | + html = html_template.format(content=html) |
| 154 | + html = html.encode("utf-8") |
| 155 | + return html |
| 156 | + except Exception as e: |
| 157 | + logging.error("解析错误", exc_info=True) |
120 | 158 |
|
121 | 159 |
|
122 | 160 | if __name__ == '__main__': |
123 | | - main() |
| 161 | + start_url = "http://www.liaoxuefeng.com/wiki/0013739516305929606dd18361248578c67b8067c8c017b000" |
| 162 | + crawler = LiaoxuefengPythonCrawler("廖雪峰Git", start_url) |
| 163 | + crawler.run() |
0 commit comments