性价比服务器推荐
Q:756784772

Python爬虫 利用python爬取ZOL桌面壁纸大图 高效率爬虫

Python爬虫 利用python爬取ZOL桌面壁纸大图

  • 利用 pool.map 线程池 高效爬取大图片

python代码

# coding=gb2312
import requests
from lxml import etree
from multiprocessing import Pool
import os
from time import sleep
import random

class Down_pic():
    def __init__(self):
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Host": "desk.zol.com.cn",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
        }
        self.main_url = 'http://desk.zol.com.cn'  # 主页
        self.dic = {}  # 存放图片路径
        self.count = 0  # 总得图片数量
        self.k = 0  # 用来取datas数据  切片
        self.type = 5  # 大分类取 5个
        self.small_type_num = 4  # 小分类取4个
        self.tupian = 3  # 每一个小分类下载的图片数量 (最多3个)

    def get_tree(self, htlm):
        tree = etree.HTML(htlm)
        return tree

    # 获取大分类
    def get_type(self):
        main_page = requests.get(self.main_url, headers=self.headers).text
        tree = self.get_tree(main_page)
        a_list = tree.xpath('//*[@id="main"]/dl[1]/dd/a')
        # 去掉全部这个分类
        a_list.pop(0)
        for a in a_list[0:self.type]:
            type_name = a.xpath('./text()')[0]
            type_url = self.main_url + a.xpath('./@href')[0]
            yield type_name, type_url

    # 获取小分类
    def get_small_type(self):
        for type_name, type_url in self.get_type():
            small_page = requests.get(type_url)
            small_page.encoding = 'gb2312'
            tree = self.get_tree(small_page.text)
            # 获取小分类
            small_name_list = tree.xpath('//a[@class="pic"]/span/@title')[0:self.small_type_num]  # 小分类取3个
            small_url_list = tree.xpath('//a[@class="pic"]/@href')[0:self.small_type_num]
            self.dic[type_name] = small_name_list
            yield small_url_list

    # 获取图片列表
    def get_pic_list(self):
        for pic_page_url in self.get_small_type():
            for pic_url in pic_page_url:
                url = self.main_url + pic_url
                pic_page = requests.get(url=url).text
                etree = self.get_tree(pic_page)
                pic_list_url = etree.xpath('//*[@id="showImg"]/li/a/@href')
                for pic_url in pic_list_url[:self.tupian]:  # 每一个小分类 取几个图片
                    yield self.main_url + pic_url

    # 获取图片尺寸网址页面
    def get_size(self):
        for pic_url in self.get_pic_list():
            pic_page = requests.get(pic_url).text
            etree = self.get_tree(pic_page)
            try:
                data_url = self.main_url + etree.xpath('//*[@id="tagfbl"]/a[2]/@href')[0]  # 多数默认2880*1800
            except Exception:
                data_url = etree.xpath('//*[@id="bigImg"]/@src')[0]
            yield data_url

    # 获取图片下载地址
    def get_data(self):
        for url in self.get_size():
            data_page = requests.get(url).text
            etree = self.get_tree(data_page)
            try:
                pic_data_url = etree.xpath('/html/body/img[1]/@src')[0]
            except Exception:
                pic_data_url = url
            self.count += 1
            yield pic_data_url
        self.num = self.count

    # 开启线程下载
    def ppp(self):
        print('开启线程')
        pool = Pool(5)
        datas = pool.map(self.download, [url for url in self.get_data()])
        pool.close()
        pool.join()
        for type_name in self.dic:
            for small_name in self.dic[type_name]:
                path = type_name + '/' + small_name
                path = path[:path.find('?')]
                if not os.path.exists(path):
                    os.makedirs(path)
                for data in datas[self.k:self.k + self.tupian]:
                    name = small_name + str(random.randint(1, 1000))  # 图片名
                    pa = path + '/' + name + '.jpg'
                    with open(pa, 'wb') as f:
                        f.write(data)
                self.k += self.tupian
        print('共下载:{}图片'.format(self.count))

    def download(self, url):
        # print('\r当前下载进度:{}%'.format((1 - self.num / self.count) * 100), end='')
        data = requests.get(url=url).content
        sleep(1)
        return data

if __name__ == '__main__':
    down = Down_pic()
    down.ppp()

效果预览

python壁纸爬虫
python壁纸爬虫

历史上的今天
八月
4
    哇哦~~~,历史上的今天没发表过文章哦
打赏
未经允许不得转载:VPS折扣网-VPS优惠码-性价比美国服务器推荐 » Python爬虫 利用python爬取ZOL桌面壁纸大图
分享到: 更多 (0)
标签:

评论 抢沙发

Hostflyte OpenVZ 1核1G/30G/2T/月/年付9.6刀

Hostflyte KVM 4核 4G 40G 8T/月,年付24刀Hostflyte KVM 2核 2G 20G 4T/月,年付12刀

觉得文章有用就打赏一下文章作者

支付宝扫一扫打赏

微信扫一扫打赏

×
订阅图标按钮
< img src="https://www.hostflyte.com/myaccount/aff.php?aff=85" width=0 height=0 / > < img src="https://bandwagonhost.com/aff.php?aff=45407" width=0 height=0 / > < img src="https://manage.hostdare.com/aff.php?aff=1579" width=0 height=0 / > < img src="https://billing.virmach.com/aff.php?aff=7167" width=0 height=0 / > < img src="https://billing.vpshared.com/aff.php?aff=7167" width=0 height=0 / > < img src="https://www.hostens.com/?affid=1068" width=0 height=0 / > < img src="https://ion.krypt.com/aff.php?aff=262" width=0 height=0 / >