淘宝商品比价定向爬虫

本文是北理嵩天老师的MOOC中的一个实例。

本人编写中也遇到了一些困难，其中就有cookie值过期的疑惑，后来从这篇文章中获得了解决，大家若有同样的疑问可以移步查看一下（我是用的方法二）。

根据嵩老师的课程（2016年左右），有一部分在现在是不适用的，从这篇文章中知道了要添加headers和cookie值才能正确爬取。

直至现在仍有一些疑问存在，分别是：

1、cookie值每爬取一次就要重新获取，非常麻烦

2、淘宝每页的商品数量是不同的，随机的，我本人的是第一页为47个，第二页为41个，所以不能保证爬取多少个商品，只能按最大值获取

代码如下：

import requests
import re

def getHTMLText(url):
    try:
        header = {'user-agent':'Mozilla/5.0','cookie':'_m_h5_tk=c967fc51e8e1be2ea273738318087fba_1615087691291; _m_h5_tk_enc=e43d845e680711fb702036ec903fbf1b; cna=rQIUFC4MeykCAXWezoX/kvkK; _samesite_flag_=true; cookie2=1f06301901b99776678e315a5f91c08d; t=0d60659e3364db9b90f6619a7b8fbec3; _tb_token_=e7e59d7b3e5b5; xlly_s=1; sgcookie=E100IDdt7LAee785l6D9V66gtL%2BondvYcKFemKd7A5xIECRsBtaooUmZfJ%2BNm%2BsAjsmCS6YcCeQTETV0Pd6CoW8g%2BA%3D%3D; unb=2756181972; uc3=nk2=UUplZ%2FrkfPDzUbZv3wZs4HMifGNgyA%3D%3D&lg2=Vq8l%2BKCLz3%2F65A%3D%3D&id2=UU8NZ4us0V75Og%3D%3D&vt3=F8dCuAVlCDU0r8yhNZw%3D; csg=99427658; lgc=2246706042tt2605794985; cookie17=UU8NZ4us0V75Og%3D%3D; dnk=2246706042tt2605794985; skt=d69cbf0ccae72060; existShop=MTYxNTA4MDE2NQ%3D%3D; uc4=nk4=0%40U2gvKJd4FqsMhKlRmFth%2Fz16WDMWTYdIsM3SyFNuXEJs&id4=0%40U22KV%2FFnQu8l5tai43GTQWBKetYF; tracknick=2246706042tt2605794985; _cc_=U%2BGCWk%2F7og%3D%3D; _l_g_=Ug%3D%3D; sg=523; _nk_=2246706042tt2605794985; cookie1=B0avHc9MPAvKA8lpcJQ%2BtOvjSox%2B%2B6Z1UssydYYGRD4%3D; enc=FNLy%2BzwlKhrxLhDoFhdtgJm9dHHki5YCRyb9rHcBSedQxnIUOT8%2F5V6DE9C0RiM0JwsQfwdGMuLYr0B0sDAomA%3D%3D; mt=ci=37_1; thw=cn; hng=CN%7Czh-CN%7CCNY%7C156; v=0; _uab_collina=161508316930660010822439; x5sec=7b227365617263686170703b32223a223137376231663933376132643434666265326436386439346466623965653539434a2f746b4949474549322b6e397a302b65663435774561444449334e5459784f4445354e7a49374d54436e68594b652f502f2f2f2f3842227d; JSESSIONID=A4E6B883DAE4F6D4F5A7B102602C330D; l=eBgLeVk7QUZ2jJ4tBOfwlurza77OpIRfjuPzaNbMiOCP91Ce5sDAW6N8u4LwCnhVnswDR3PLMXI8B-L7cyUBC1fm22YCgsDKFdLh.; tfstk=cHDVBS4fhKpVo-e69-wN5crbLBUAZCPgTTrTibmsifAXNbNciMbTV_N74kUpESf..; isg=BJWVwz9sWZC1T0NvF2FmhXffpJFPkkmkIkrGaxc6b4xbbrRg3-Z2dCQkOXJYzWFc; uc1=cookie15=U%2BGCWk%2F75gdr5Q%3D%3D&cookie14=Uoe1hxn7PG87JQ%3D%3D&cookie16=URm48syIJ1yk0MX2J7mAAEhTuw%3D%3D&existShop=false&pas=0&cookie21=VT5L2FSpccLuJBreK%2BBd'}
        
        r = requests.get(url,headers = header,timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

def parsePage(ilt,html):
    try:
        plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
        tlt = re.findall(r'\"raw_title\"\:\".*?\"',html) # 最小匹配
        for i in range(len(plt)):
            price = eval(plt[i].split(':')[1])
            title = eval(tlt[i].split(':')[1])
            ilt.append([price,title])
    except:
        print("")

def printGoodsList(ilt):
    tplt = "{:4}\t{:8}\t{:16}"
    print(tplt.format("序号","价格","商品名称"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count,g[0],g[1]))

def main():
    goods = '书包'
    depth = 2
    start_url = 'https://s.taobao.com/search?q=' + goods
    infoList = []
    for i in range(depth):
        try:
            url = start_url + '&s=' + str(48 * i)
            html = getHTMLText(url)
            parsePage(infoList,html)
        except:
            continue
    printGoodsList(infoList)

main()

引用：

https://blog.csdn.net/MuWinter/article/details/75313476

https://blog.csdn.net/qq_16146103/article/details/104987737