Skip to content

Commit 18c0683

Browse files
lzjun567lzjun567
committed
refactor htmltopdf
1 parent ac103b2 commit 18c0683

3 files changed

Lines changed: 167 additions & 141 deletions

File tree

‎pdf/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ python crawler.py
3939

4040
3. 生成的PDF文件可以在公众号回复『pdf』下载
4141

42+
### 更新记录
43+
44+
* 2017-2-21: 对代码进行了全面的重构,可扩展, 子类爬虫只需实现 `parse_menu`和`parse_body`方法就可以实现HTML转换PDF的逻辑
45+
4246

4347
### Contact me
4448

‎pdf/crawler.py‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# coding=utf-8
2+
import logging
3+
import os
4+
import re
5+
import time
6+
7+
try:
8+
from urllib.parse import urlparse # py3
9+
except:
10+
from urlparse import urlparse # py2
11+
12+
import pdfkit
13+
import requests
14+
from bs4 import BeautifulSoup
15+
16+
html_template = """
17+
<!DOCTYPE html>
18+
<html lang="en">
19+
<head>
20+
<meta charset="UTF-8">
21+
</head>
22+
<body>
23+
{content}
24+
</body>
25+
</html>
26+
27+
"""
28+
29+
30+
class Crawler(object):
31+
"""
32+
爬虫基类，所有爬虫都应该继承此类
33+
"""
34+
name = None
35+
36+
def __init__(self, name, start_url):
37+
"""
38+
初始化
39+
:param name: 保存问的PDF文件名,不需要后缀名
40+
:param start_url: 爬虫入口URL
41+
"""
42+
self.name = name
43+
self.start_url = start_url
44+
self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url))
45+
46+
def crawl(self, url):
47+
"""
48+
pass
49+
:return:
50+
"""
51+
print(url)
52+
response = requests.get(url)
53+
return response
54+
55+
def parse_menu(self, response):
56+
"""
57+
解析目录结构,获取所有URL目录列表:由子类实现
58+
:param response 爬虫返回的response对象
59+
:return: url 可迭代对象(iterable) 列表,生成器,元组都可以
60+
"""
61+
raise NotImplementedError
62+
63+
def parse_body(self, response):
64+
"""
65+
解析正文,由子类实现
66+
:param response: 爬虫返回的response对象
67+
:return: 返回经过处理的html文本
68+
"""
69+
raise NotImplementedError
70+
71+
def run(self):
72+
start = time.time()
73+
options = {
74+
'page-size': 'Letter',
75+
'margin-top': '0.75in',
76+
'margin-right': '0.75in',
77+
'margin-bottom': '0.75in',
78+
'margin-left': '0.75in',
79+
'encoding': "UTF-8",
80+
'custom-header': [
81+
('Accept-Encoding', 'gzip')
82+
],
83+
'cookie': [
84+
('cookie-name1', 'cookie-value1'),
85+
('cookie-name2', 'cookie-value2'),
86+
],
87+
'outline-depth': 10,
88+
}
89+
htmls = []
90+
for index, url in enumerate(self.parse_menu(self.crawl(self.start_url))):
91+
html = self.parse_body(self.crawl(url))
92+
f_name = ".".join([str(index), "html"])
93+
with open(f_name, 'wb') as f:
94+
f.write(html)
95+
htmls.append(f_name)
96+
97+
pdfkit.from_file(htmls, self.name + ".pdf", options=options)
98+
for html in htmls:
99+
os.remove(html)
100+
total_time = time.time() - start
101+
print(u"总共耗时：%f 秒" % total_time)
102+
103+
104+
class LiaoxuefengPythonCrawler(Crawler):
105+
"""
106+
廖雪峰Python3教程
107+
"""
108+
109+
def parse_menu(self, response):
110+
"""
111+
解析目录结构,获取所有URL目录列表
112+
:param response 爬虫返回的response对象
113+
:return: url生成器
114+
"""
115+
soup = BeautifulSoup(response.content, "html.parser")
116+
menu_tag = soup.find_all(class_="uk-nav uk-nav-side")[1]
117+
for li in menu_tag.find_all("li"):
118+
url = li.a.get("href")
119+
if not url.startswith("http"):
120+
url = "".join([self.domain, url]) # 补全为全路径
121+
yield url
122+
123+
def parse_body(self, response):
124+
"""
125+
解析正文
126+
:param response: 爬虫返回的response对象
127+
:return: 返回处理后的html文本
128+
"""
129+
try:
130+
soup = BeautifulSoup(response.content, 'html.parser')
131+
body = soup.find_all(class_="x-wiki-content")[0]
132+
133+
# 加入标题, 居中显示
134+
title = soup.find('h4').get_text()
135+
center_tag = soup.new_tag("center")
136+
title_tag = soup.new_tag('h1')
137+
title_tag.string = title
138+
center_tag.insert(1, title_tag)
139+
body.insert(1, center_tag)
140+
141+
html = str(body)
142+
# body中的img标签的src相对路径的改成绝对路径
143+
pattern = "(<img .*?src=\")(.*?)(\")"
144+
145+
def func(m):
146+
if not m.group(3).startswith("http"):
147+
rtn = "".join([m.group(1), self.domain, m.group(2), m.group(3)])
148+
return rtn
149+
else:
150+
return "".join([m.group(1), m.group(2), m.group(3)])
151+
152+
html = re.compile(pattern).sub(func, html)
153+
html = html_template.format(content=html)
154+
html = html.encode("utf-8")
155+
return html
156+
except Exception as e:
157+
logging.error("解析错误", exc_info=True)
158+
159+
160+
if __name__ == '__main__':
161+
start_url = "http://www.liaoxuefeng.com/wiki/0013739516305929606dd18361248578c67b8067c8c017b000"
162+
crawler = LiaoxuefengPythonCrawler("廖雪峰Git", start_url)
163+
crawler.run()

‎pdf/crawler1.py‎

Lines changed: 0 additions & 141 deletions
This file was deleted.

0 commit comments

Comments
 (0)