关于python的段子_Python-好段子爬取

import urllib.request

import urllib.parse

from lxml import etree #导入xpath#

import time

import json

import os

item_list=[]

def handle_request(url,page):

headers={

"User-Agnet":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"

}

#将url和page进行拼接#

url=url%page

#如果下面的用{},可以写成url=url.format(page)

print(url)

request=urllib.request.Request(url=url,headers=headers)

return request

def parse_content(content):

#生成对象, 构造xpath的解析对象#

tree=etree.HTML(content)

#抓取内容#

div_list=tree.xpath('//div/a/p')

#遍历div列表#

print(div_list)

print(len(div_list))

for odiv in div_list:

#获取标题#

title=odiv.xpath('//div[@class="head"]/h2/text()')[0]

print(title)

text_lt=odiv.xpath('.//div[@class="content"]/p/text()')#返回的是一个列表#

text='\n'.join(text_lt)

print(text)

item={

'标题':title,

'内容':text,

}

#将内容添加到列表中#

item_list.append(item)

def main():

start_page=int(input("请输入起始页码："))

end_page=int(input("请输入结束页码："))

url='http://www.haoduanzi.com/category/?1-%s.html'

#这样写也可以：url='http://www.haoduanzi.com/category/?1-{.}html'

for page in range(start_page,end_page+1):

request=handle_request(url,page)

content=urllib.request.urlopen(request).read().decode()

#解析内容#

parse_content(content)

time.sleep(2)

#写入到文件中#

string=json.dumps(item_list,ensure_ascii=False)

with open('/Users/marine/Desktop/python/duanzi.txt','w', encoding='utf8') as fp:

fp.write(string)

if name=='main':

main()