add

javasmall · javasmall · commit 6d7c764811f2 · 2019-06-02T10:26:07.000+08:00
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+
+        
+        {
+            "name": "Python: 当前文件",
+            "type": "python",
+            "request": "launch",
+            "pythonPath": "/usr/bin/python3",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,5 @@
+{
+    "editor.fontSize": 16,
+    "debug.console.fontSize": 16,
+    "terminal.integrated.fontSize": 16
+}
diff --git a/爬虫/Include/base/test.py b/爬虫/Include/base/test.py
@@ -0,0 +1 @@
+print('jj')
diff --git a/爬虫/Include/spider/handercookie.py b/爬虫/Include/spider/handercookie.py
@@ -1,5 +1,5 @@
 import urllib.request as urllib2
-import
+
 
 # 构建一个CookieJar对象实例来保存cookie
 cookiejar = cookielib.CookieJar()
diff --git a/爬虫/Include/网易云信/jincheng.py b/爬虫/Include/网易云信/jincheng.py
@@ -0,0 +1,26 @@
+from multiprocessing import Pool 
+from time import sleep,ctime 
+
+def worker(msg):
+    sleep(2)
+    print(msg)
+    return ctime()
+
+#创建进程池
+pool = Pool(processes = 4)
+
+result = []
+for i in range(10):
+    msg = "hello %d"%i
+    #将事件放入进程池队列,等待执行
+    r = pool.apply_async(func = worker,args = (msg,))
+    result.append(r)
+
+#关闭进程池
+pool.close()
+
+#回收
+pool.join()
+
+for i in result:
+    print(i.get())  #获取事件函数的返回值
diff --git a/爬虫/Include/网易云信/wangyi.py b/爬虫/Include/网易云信/wangyi.py
@@ -3,19 +3,16 @@
 import time
 import threading
 from queue import Queue
-def loadip():
+def loadip():##从代理ip中获取ip 一次若干扩充到queue中
     url2 = 'http://piping.mogumiao.com/proxy/api/get_ip_al?appKey=f16367295e284173ae450fc38d9098b3&count=20&expiryDate=0&format=1&newLine=2'
     req = requests.get(url2)
     date = req.json()
     if(date['code'])!='3001':
         ipdate2 = date['msg']
-        global ipdate
-        ipdate.extend(ipdate2)
         for va in ipdate2:
           que.put(va)
-        print(ipdate)
 
-class downspider(threading.Thread):
+class downspider(threading.Thread):##线程类
     def __init__(self, threadname, que):
         threading.Thread.__init__(self)
         self.threadname = threadname
@@ -25,20 +22,17 @@ def run(self):
         print('start thread' + self.threadname)
         while True:
             try:
-                print(self.name,end='')
-                toupiaospider(que,self.threadname)
+                toupiaospider(que,self.threadname)##投票函数
             except Exception as e:
                 print(e,'888')
                 break
-def getproxies():
-    b = ipdate[0]
+def getproxies():#获取ip 拼接成需要的代理格式
     b=que.get()
     d = '%s:%s' % (b['ip'], b['port'])
     global proxies
     proxies['http'] = d
-    global msg
-    msg = b
-    return proxies
+    proxies2={'http':d}
+    return proxies2
 def toupiaospider(que,threadname):
     if (que.qsize() < 15):  # 拓展ip池
         loadip()
@@ -49,16 +43,13 @@ def toupiaospider(que,threadname):
             req = requests.post(url, headers=header, data=formData, proxies=proxies2, timeout=1.5)
             res = req.json()
             if res['res']==2001 or req.status_code!=200:
-                #ipdate.remove(msg)
                  continue
-            print(threadname,res,que.qsize())
+            print(threadname,proxies2['http'],res,que.qsize())
         except Exception as e:
             print('errror',e)
-            # ipdate.remove(msg)
 
 if __name__ == '__main__':
-    ipdate = []
-    msg = {}
+    
     proxies = {'http': ''}
     stadus = 0
     que = Queue()
@@ -76,19 +67,12 @@ def toupiaospider(que,threadname):
                }
     loadip()
     time.sleep(5)
-    threadList = ['thread-1','thread-2','thread-3','thread-4','thread-4']
+    ##线程数组   ->启动 ——>等待join
+    threadList = ['thread-1','thread-2','thread-3','thread-4','thread-4','thread-5']
     for j in threadList:
         thread = downspider(j, que)
         thread.start()
         threads.append(thread)
     for t in threads:
         t.join()
-    # for i in range(100):
-    #     try:
-    #         toupiaospider()
-    #     except Exception as e:
-    #         print('error')
-    #         try:
-    #             ipdate.remove(msg)
-    #         except Exception as e2:
-    #             print("e2",e2)
+
