功能分析

网上爬取豆瓣电影排行很多，但是由于做一个h5画廊需求大量电影素材，而列表与详情页的图片清晰度满足不了要求，所以决定爬取豆瓣原图，在查看是发现需要登陆，然而登陆后原图链接查看并没有任何cookie信息在请求头，遂想直接构造链接爬取，爬取过程出现302，重新分析请求头，发现有referer,嗯，直接加了豆瓣首页作为referer,mdfuck居然没有用，后来发现每个原图需要带着该图前中等缩略图链接作为referer，可以看出豆瓣在服务器端作了比较严格的过滤处理，虽然在nginx上也做过图片防盗链，自认为没有做如此猥琐，别人猥琐，咱也不能谦虚是吧，一个个构造请求头，利用python爬取原图。

所需模块

网页内容抓取用urllib2

页面解析用pyquery(一个可以用jquey方式解析html的模块)

下面是代码部分：（好久没写python,表示有点手生ฅʕ•̫͡•ʔฅ，图片大概200m，页面爬取很快，最后是io下载，所以速度还是取决网速）

#coding:utf-8

import urllib2
import re
import sys
import time
from pyquery import PyQuery as pq
#http://movie.douban.com/top250?start=0&filter=&type= 
class Douban:
    def __init__(self):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        self.start = 0
        self.param = '&filter=&type='
        self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.113 Safari/537.36'}
        self.movieList = []
        self.filePath = './img/'
        self.imgpath = 'https://img3.doubanio.com/view/photo/raw/public/'
        self.refer = 'https://movie.douban.com/photos/photo/'

    def getPage(self):
        print '---getpagestart---'
        try:
            URL = 'http://movie.douban.com/top250?start=' + str(self.start)
            request = urllib2.Request(url = URL, headers=self.headers)
            response = urllib2.urlopen(request)
            page = response.read().decode('utf-8')
            pageNum = (self.start + 25)/25
            print 'scrabing ' + str(pageNum) + 'page...'
            self.start += 25;
            return page
        except urllib2.URLError, e:
            if hasattr(e, 'reason'):
                print 'failed reason', e.reason
    def htmlparse(self):
        print '---getMoviestart---'
        while self.start  < 250:

            page = self.getPage()
            html = pq(page);
            list = html(".grid_view>li")
            info = {};
            for item in list:
                item = pq(item)
                info['name'] = item(".hd>a").text()
                info['des'] = item(".bd p:first").text()
                info['img'] = item(".pic img").attr('src');

                group = re.findall(r'\\/(p(\\d+)\\.jpg)', info['img'])

                info['img'] = self.imgpath + group[0][0]
                info['refer'] = self.refer + group[0][1] + '/'
                #print info

                self.movieList.append([info['name'], info['des'], info['img'],info['refer']])


    def hook(self):
        mfile = open(self.filePath + 'movielist.txt', 'w')
        try:
            for index, movie in enumerate(self.movieList):
                print movie[0].encode('gbk','ignore')
                #print movie
                self.downImg(movie[2], movie[3], self.filePath + 'movie' + str(index+1) + '.jpg')
                mfile.write(str(index+1) + '、' + movie[0] + '\\n'+ movie[1] + '\\n')
            print 'wirte done'
        finally:
            mfile.close()

    def downImg(self, URL, refer, imgpath):
        head = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.113 Safari/537.36'};
        head['Referer'] = refer
        request = urllib2.Request(url = URL, headers = head)
        try:
            f = open(imgpath, 'wb')
            res = urllib2.urlopen(request).read()
            f.write(res)
            f.close()
        except urllib2.URLError, e:
            if hasattr(e, 'reason'):
                print 'failed reason', e.reason
    def main(self):
        print '---mainstart---'
        self.htmlparse()
        print len(self.movieList) 
        self.hook()
DB = Douban()
DB.main()

运行效果

图片发自简书App

图片发自简书App

图片发自简书App

图片发自简书App