Skip to content

Commit a809dec

Browse files
author
刘志军
committed
add xingqiu crawler
1 parent ec4b815 commit a809dec

6 files changed

Lines changed: 4887 additions & 1 deletion

File tree

‎heart2/heart.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,11 @@ def generate_image():
8181
background_color='white',
8282
mask=mask_img
8383
).generate(data)
84+
plt.title(u"天下有情人终成眷属")
8485
plt.imshow(wordcloud.recolor(color_func=grey_color_func, random_state=3),
8586
interpolation="bilinear")
8687
plt.axis('off')
87-
plt.savefig('./heart2.jpg', dpi=1600)
88+
plt.savefig('./heart3.jpg', dpi=1600)
8889

8990

9091
if __name__ == '__main__':

‎heart2/heart3.jpg‎

2.13 MB
Loading

‎heart2/weibo1.txt‎

Lines changed: 4730 additions & 0 deletions
Large diffs are not rendered by default.

‎lucky/__init__.py‎

Whitespace-only changes.

‎lucky/crawler.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# coding=utf-8
2+
3+
import requests
4+
5+
6+
def main():
7+
url = "https://lucky.nocode.com/public_lottery?page=1&size=5"
8+
headers = {
9+
"Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoxMjc0NjUxLCJuaWNrX25hbWUiOiJsemp1biIsImF2YXRhciI6Imh0dHBzOi8vd3gucWxvZ28uY24vbW1vcGVuL3ZpXzMyL2liRWYwaWFZZFV2R3hMbXVmcVpDQmxtQ1pBMUlGZVdwWUc1aWJQQ0dTOUZNaFRwY2xyaWJRWDFjakEwN2lhcnhNaWNxODVRdzQ2aWNFWDRwTmg0a3k1WXdoRjJhdy8wIiwiaWF0IjoxNTIyOTcwNzc2LCJleHAiOjE1MjM1NzU1NzZ9.9tl6PYffqtUitVIYAYr_TCo9CD_h7Qn-mWsA32KN4Cg"}
10+
res = requests.get(url, headers=headers)
11+
lotteries = res.json().get("data")
12+
join_url = "https://lucky.nocode.com/lottery/{id}/join"
13+
for lottery in lotteries:
14+
res = requests.post(join_url.format(id=lottery.get("id")), headers=headers)
15+
data = res.json()
16+
if res.status_code == 200 and 'errors' not in data:
17+
print("成功参与抽奖：《%s》" % lottery.get("prizes").get("data")[0].get("name"))
18+
19+
20+
if __name__ == '__main__':
21+
main()

‎xingqiu/xingqiu.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
"""
2+
注意：代码基于Python3.6，不兼容python2
3+
有疑问可通过公众号“Python之禅”联系作者
4+
"""
5+
import codecs
6+
import csv
7+
from urllib import parse
8+
9+
import requests
10+
from pymongo import MongoClient
11+
12+
client = MongoClient()
13+
db = client.xingqiu
14+
15+
16+
def str_to_dict(s, join_symbol="\n", split_symbol=":"):
17+
"""
18+
将字符串转换成dict对象
19+
:param s:
20+
:param join_symbol:
21+
:param split_symbol:
22+
:return:
23+
"""
24+
s_list = s.split(join_symbol)
25+
data = dict()
26+
for item in s_list:
27+
item = item.strip()
28+
if item:
29+
k, v = item.split(split_symbol, 1)
30+
data[k] = v.strip()
31+
return data
32+
33+
34+
# 直接从浏览器里面拷贝过来的，请替换成你自己的浏览器中的内容
35+
headers = """
36+
Accept: application/json, text/plain, */*
37+
Accept-Encoding: gzip, deflate, br
38+
Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7
39+
Authorization: 8EAE71AC-081C-FE82-BE8C-954696
40+
Connection: keep-alive
41+
DNT: 1
42+
Host: api.zsxq.com
43+
Origin: https://wx.zsxq.com
44+
Referer: https://wx.zsxq.com/dweb/
45+
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36
46+
X-Request-Id: 02001c28-fb3a-f027-20e7-8a1c458f3eee
47+
X-Version: 1.10.0
48+
"""
49+
50+
init_url = "https://api.zsxq.com/v1.10/groups/518855855524/topics?count=20"
51+
52+
53+
def crawl(url):
54+
res = requests.get(url, headers=str_to_dict(headers))
55+
topics = res.json().get("resp_data").get("topics")
56+
if len(topics) <= 1:
57+
return
58+
for i in topics:
59+
print(i.get("talk").get("text")[:10])
60+
db.topics.insert_one(i)
61+
else:
62+
last_time = i.get("create_time")
63+
crawl(url + "&end_time=" + parse.quote(last_time))
64+
65+
66+
def statics():
67+
# 打卡
68+
talk = db.topics.aggregate(
69+
[
70+
{"$match": {"create_time": {"$gte": "2018-05-28T00:00:14.202+0800"}}},
71+
{
72+
"$group": {
73+
"_id": {
74+
"user_id": "$talk.owner.user_id",
75+
"name": "$talk.owner.name",
76+
},
77+
"count": {"$sum": 1},
78+
}
79+
},
80+
{"$sort": {"count": -1}},
81+
]
82+
)
83+
84+
# 作业
85+
solution = db.topics.aggregate(
86+
[
87+
{"$match": {"create_time": {"$gte": "2018-05-28T00:00:14.202+0800"}}},
88+
{
89+
"$group": {
90+
"_id": {
91+
"user_id": "$solution.owner.user_id",
92+
"name": "$solution.owner.name",
93+
},
94+
"count": {"$sum": 1},
95+
}
96+
},
97+
{"$sort": {"count": -1}},
98+
]
99+
)
100+
101+
data = dict()
102+
103+
for item in talk:
104+
name = item.get("_id").get("name")
105+
if name:
106+
data[name] = {"talk": item.get("count")}
107+
108+
for item in solution:
109+
name = item.get("_id").get("name")
110+
if name:
111+
data.setdefault(name, {}).update({"solution": item.get("count")})
112+
113+
return data
114+
115+
116+
if __name__ == "__main__":
117+
# 爬取数据并存储
118+
crawl(init_url)
119+
# 统计数据
120+
data = statics()
121+
# 统计写入cvs文件
122+
with codecs.open("names.csv", "w", "utf_8_sig") as csvfile:
123+
fieldnames = ["name", "talk", "solution"]
124+
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
125+
writer.writeheader()
126+
127+
for name, value in data.items():
128+
writer.writerow(
129+
{
130+
"name": name,
131+
"talk": value.get("talk"),
132+
"solution": value.get("solution"),
133+
}
134+
)

0 commit comments

Comments
 (0)