腾讯招聘信息爬取

腾讯招聘网站信息爬取，用到了伪造headers信息，xpath进行内容解析以及存储为json文件等。

开发环境：Windows10
开发语言:Python3.6
开发工具：pycharm
抓包工具：Charles

import requests
import random
import json
import time
from lxml import etree


class TencentSpider(object):
    def __init__(self):
        self.base_url = "http://hr.tencent.com/position.php?&start="
        self.offset = 0
        USER_AGENTS = [
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
            "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
            "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
            "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3192.0 Safari/537.36Name"
        ]
        self.headers = {"User-Agent": random.choice(USER_AGENTS)}

    def start_work(self):
        print("爬虫已经爬到%s页" % (int(self.offset/10 + 1)))
        item = {}
        content = requests.get(url=self.base_url + str(self.offset), headers=self.headers).content.decode()
        html = etree.HTML(content)
        node_list = html.xpath('//*[@class="even"]|//*[@class="odd"]')
        if node_list == []:
            return
        for node in node_list:

            item['name'] = "".join(node.xpath("./td[1]/a/text()"))
            item['detailLink'] = "".join(node.xpath("./td[1]/a/@href"))
            item['positionInfo'] = "".join(node.xpath("./td[2]/text()"))
            item['pepopleNumber'] = "".join(node.xpath("./td[3]/text()"))
            item['workLocation'] = "".join(node.xpath("./td[4]/text()"))
            item['publishTime'] = "".join(node.xpath("./td[5]/text()"))
            self.write_data(item)

        self.offset += 10
        time.sleep(random.random())
        self.start_work()

    def write_data(self, item):
        content = json.dumps(item, ensure_ascii=False) + ",\n"
        with open("Tencent.json", "a", encoding="utf-8") as f:
            f.write(content)


if __name__ == "__main__":
    work = TencentSpider()
    work.start_work()