利用爬虫爬取套图网站美女的所有写真（带多线程）

篡命师

====================================

==两种方式==原作者@zrq648022547

==指定某美女，爬取全部所有的图册                          ==

==指定图集首页地址，爬取单图册图片                      ==

====================================
代码一：爬取指定美女图册
缺点：
1、虽然写入了多线程爬取，但是测试貌似还是比较慢；
2、爬取的图片名称不能按照序号命名
3、未处理反扒机制

1. # 多进程异步并发
   
2. import random
   
3. import requests
   
4. from bs4 import BeautifulSoup
   
5. import os
   
6. import time
   
7. import threading
   
8. from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
   
9. import concurrent.futures
   
10. 
    
11. USER_AGENTS = [
    
12.     "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    
13.     "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    
14.     "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
    
15.     "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
    
16.     "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    
17.     "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
    
18.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    
19.     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
    
20.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    
21.     "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    
22.     "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
    
23.     "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
    
24.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    
25.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
    
26.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
    
27.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    
28.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
    
29.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    
30.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    
31.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
    
32.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    
33.     "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    
34.     "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    
35.     "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    
36.     "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    
37.     "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    
38.     "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    
39.     "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    
40.     "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    
41.     "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    
42.     "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    
43.     "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    
44.     "UCWEB7.0.2.37/28/999",
    
45.     "NOKIA5700/ UCWEB7.0.2.37/28/999",
    
46.     "Openwave/ UCWEB7.0.2.37/28/999",
    
47.     "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
    
48.     # iPhone 6：
    
49.     "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25",
    
50. ]
    
51. 
    
52. headers = {
    
53.     'User-Agent': random.choice(USER_AGENTS),
    
54.     "referer": "https://www.xiurenji.vip/"
    
55. }
    
56. 
    
57. 
    
58. def get_item(item, main_title):
    
59.     title = item['title']
    
60.     threads = []
    
61.     item_links = [start_url + item['href']]
    
62.     # print(f'{title}>>>{item_links}')
    
63.     for item_link in item_links:
    
64.         threads.append(threading.Thread(target=get_images,args=(title, item_link, main_title)))
    
65.     for thread in threads:
    
66.         thread.start()
    
67.     for thread in threads:
    
68.         thread.join()
    
69. 
    
70. 
    
71. def get_images(title, item_link, main_title):
    
72.     try:
    
73.         item_res = requests.get(url=item_link, headers=headers,timeout=30)
    
74.         item_res.encoding = 'gzip'
    
75.         item_soup = BeautifulSoup(item_res.text, 'lxml')
    
76.         img_list = item_soup.select('.content_left p img')
    
77.         # print(img_list)
    
78.         folder = main_folder + '/' + main_title + '/' + title + '/'
    
79.         if not os.path.exists(folder):
    
80.             os.makedirs(folder)
    
81.         else:
    
82.             pass
    
83.         try:
    
84.             for img in img_list:
    
85.                 img_link = start_url + img['src']
    
86.                 # print(img_link)
    
87.                 with open(folder + img_link.split('/')[-1], 'wb') as f:
    
88.                     starttime = time.time()
    
89.                     image = requests.get(url=img_link, headers=headers,timeout=30).content
    
90.                     f.write(image)
    
91.                     time.sleep(1)
    
92.                     endtime = time.time()
    
93.                     print(f'正在保存>>>{title}' + '>>>' + img_link.split('/')[-1] + '>>>用时%.3f'%(endtime - starttime), 'seconds')
    
94.         except IndexError:
    
95.             pass
    
96.         next = item_soup.select_one('.page a:last-of-type')
    
97.         if 'class="current"' in str(next):
    
98.             pass
    
99.         else:
    
100.             item_link = start_url + next['href']
     
101.             with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
     
102.                 pool.submit(get_images,title, item_link, main_title)
     
103.     except IndexError:
     
104.         pass
     
105. 
     
106. 
     
107. if __name__ == '__main__':
     
108.     start = time.time()
     
109.     start_url = 'https://www.xrmn5.cc'  # 网站根地址
     
110.     main_title = '秀人网'
     
111.     main_folder = "单项目2\"  # 主文件夹路径（请填入你自己的文件夹路径）
     
112.     main_url = 'https://www.xrmn5.cc/younisi.html'
     
113.     itemlist_res = requests.get(url=main_url, headers=headers,timeout=30)
     
114.     itemlist_res.encoding = 'gzip'
     
115.     itemlist_soup = BeautifulSoup(itemlist_res.text, 'lxml')
     
116.     itemlist = itemlist_soup.select('.list_n2 a')
     
117.     # print(itemlist)
     
118.     for item in itemlist:
     
119.         get_item(item, main_title)
     
120.     end = time.time()
     
121.     print('总共用时：',end - start,'seconds',end='')

复制代码

代码二：爬取美女单图集
缺点：
1、文件夹名称需要手动指定
2、爬取的图册首页地址需要手动指定
3、爬取的图片和页面显示的图片数量不符合，有的页面图片漏掉了，不知道为啥

1. import requests
   
2. import parsel
   
3. import random
   
4. import os
   
5. import datetime
   
6. import time
   
7. 
   
8. 
   
9. starttime = datetime.datetime.now()
   
10. USER_AGENTS = [
    
11.     "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    
12.     "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    
13.     "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
    
14.     "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
    
15.     "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    
16.     "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
    
17.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    
18.     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
    
19.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    
20.     "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    
21.     "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
    
22.     "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
    
23.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    
24.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
    
25.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
    
26.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    
27.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
    
28.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    
29.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    
30.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
    
31.     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    
32.     "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    
33.     "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    
34.     "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    
35.     "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    
36.     "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    
37.     "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    
38.     "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    
39.     "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    
40.     "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    
41.     "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    
42.     "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    
43.     "UCWEB7.0.2.37/28/999",
    
44.     "NOKIA5700/ UCWEB7.0.2.37/28/999",
    
45.     "Openwave/ UCWEB7.0.2.37/28/999",
    
46.     "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
    
47.     # iPhone 6：
    
48.     "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25",
    
49. ]
    
50. 
    
51. headers = {
    
52.     'User-Agent': random.choice(USER_AGENTS),
    
53.     # 'Connection': 'close'
    
54.     "referer":"https://www.xrmn5.cc"
    
55. }
    
56. 
    
57. # 0.创建文件夹
    
58. directory = '[YouMi尤蜜荟]Vol.809_女神尤妮丝Egg红色轻透上衣配红短裙半脱露红色内衣诱惑写真60P\\'
    
59. if os.path.exists(directory):
    
60.     pass
    
61. else:
    
62.     os.mkdir(directory)
    
63. # 1.确定爬取的网站
    
64. 
    
65. for page in range(0, 30):
    
66.     try:
    
67.         if page >= 1:
    
68.             base_url = 'https://www.xrmn5.com/YouMi/2022/202211014_{}.html'.format(page)
    
69.         else:
    
70.             base_url = 'https://www.xrmn5.com/YouMi/2022/202211014.html'
    
71.         # print('==========正在爬取第{}页数据============='.format(page))
    
72.         # 2.发送请求
    
73.         response = requests.get(url=base_url, headers=headers)
    
74.         response.encoding = 'UTF-8'  # 自动识别响应体的编码
    
75.         # print(response)
    
76.         html_data = response.text
    
77.         # print(html_data)
    
78.         # 解析详情页图片地址
    
79.         response_1 = requests.get(base_url, headers=headers).text
    
80.         html_1 = parsel.Selector(response_1)
    
81.         # print(html_1)
    
82.         # 解析图册中图片地址
    
83.         for i in range(1,4):
    
84.             img_list_1 = html_1.xpath('//*[@class="content_left"]/p/img[{}]/@src'.format(i)).extract_first()
    
85.             img_list = img_list_1.replace('uploadfile','Uploadfile')
    
86.             img_url = 'https://p.xrmn5.com/' + str(img_list)
    
87.             # print(img_list_1)
    
88.             # 请求图片地址
    
89.             img_data = requests.get(img_url, headers=headers).content
    
90.             img_name_1 = str(int(page) + 1)  # 图片文件名称
    
91.             # print(img_name_1)
    
92.             # 4.数据保存
    
93.             with open(directory + img_name_1 + '-' + str(i) + '.jpg', 'wb') as f:
    
94.                 f.write(img_data)
    
95.                 time.sleep(10)
    
96.             print('#####################正在爬取第', page + 1, '页,第', int(i),'张图片#####################')
    
97.     except IndexError:
    
98.         continue
    
99. endtime = datetime.datetime.now()
    
100. print('####################下载完成，共用时', (endtime - starttime).seconds, '秒###################')

复制代码

woc原

666