从京东网站抓取_006

目标：抓取的内容组织成如下字典形式

[{'url' : 'xxxxxxxxx'
 'price' : 'xxx'
 'id' : 'xxx'
 'img' : 'xxx'
 'title' : 'xxx'
}#宝贝1
{'url' : 'xxxxxxxxx'
 'price' : 'xxx'
 'id' : 'xxx'
 'img' : 'xxx'
 'title' : 'xxx'
}#宝贝2
{'url' : 'xxxxxxxxx'
 'price' : 'xxx'
 'id' : 'xxx'
 'img' : 'xxx'
 'title' : 'xxx'
}#宝贝3
........
]

分析及组织访问网址
字段以键对值的形式表示，其中psort表示产品以某种形式进行排序，page表示当前是第几页，keyword表示搜索产品的关键词，可以通过建立一个函数，包含此三个参数来构建访问网址，以此达到抓取特定产品信息的目的

import urlparse
import urllib
from bs4 import BeautifulSoup
url = "http://search.jd.com/Search?keyword=BOSE&enc=utf-8&qr=%E5%B7%B4%E5%A1%9E&qrst=correct&et=4&rt=1&stop=1&click=&psort=3"
parsed = urlparse.urlparse(url)
d = urlparse.parse_qs(parsed.query,True)
"""
{'qrst': ['correct'], 
 'rt': ['1'], 
 'enc': ['utf-8'], 
 'keyword': ['BOSE'], 
 'stop': ['1'], 
 'psort': ['3'], 
 'qr': ['\xe5\xb7\xb4\xe5\xa1\x9e'], 
 'et': ['4'],
 'page':['1'] 
 'click': ['']}
 """

构建函数访问特定网址

def get_content_from_jd(keyword = 'Bose', page = '', product_sort = ''):
    params = {'qrst': 'correct', 
              'rt': '1', 
              'enc': 'utf-8', 
              'keyword': keyword, 
              'stop': '1', 
              'psort': product_sort, 
              'et': '4',
              'page': page,
              'click': ''} #建立参数，其中keyword，psort，page通过函数参数传入
    html = urllib.urlencode(params)
    url = 'http://search.jd.com/Search?' + html #构建访问网址
    html_content = urllib.urlopen(url) #抓取网址内容
    content = html_content.read()
    html_content.close()
    return content

抓取商品ID

if __name__ == '__main__':
    bose_info = get_content_from_jd('Bose', '1', '3')  #keyword = Bose, page = 1, product_sort = 3 3为按销量排序
    bose_content = BeautifulSoup(bose_info)
    bose_div = bose_content.find_all('div', class_ = 'lh-wrap') #每件商品以lh-wrap关键字区分
    # <strong class="J_1150767"></strong> #id号以strong区分
    for i in bose_div:
        print i.strong['class'][0]

返回内容

J_1150767
J_1225287
J_1225340
J_1150768
J_1150841
J_1237666
J_1150698
J_1237665
J_1150821
J_1150753
J_1237663
J_1182133
J_1150733
J_1253123
J_1253126
J_1150701
J_1291112
J_1253232
J_1150689
J_1150690
J_1150699
J_1150704
J_1255545
J_1237659
J_1253213
J_1253226
J_1150759
J_1150760
J_1150766
J_1291110