Python爬虫requests基础

数据、headers、代理、cookies怎么用上

import requests

url='http://www.renren.com/PLogin.do'
data = {}
headers = {}
proxies = {
"http":"http://10.10.1.10:222",
"https":"https://10.10.1.10:1431"}
#加密代理
# proxies = {"http":"http://user:pass@ip:port"}
cookies = {} 
files = {'file':open('test.txt','rb')}
#get
res = request.get(url,params=data, headers=headers, proxies=proxies,cookies=cookies)
#post
res =requests.post(url,data=data,headers=headers,
proxies=proxies,cookies=cookies)
#文件
res = requests.post(url,files=files)

二进制网页
res.content

经过默认utf-8解码的网页,乱码就自己修改解码方式
res.text

状态码
res.status_code

网页cookie，用来模拟登录
res.cookies
字典化cookies
res.cookies.get_dict()

禁止重定向
allow_redirects=Ture

超时设置
timeout=2

session

#用法和requests差不多
import requests

s = requests.session()
res = s.get(url)

认证

import requests
from requests.auth import HTTPBasicAuth
res = requests.get(url, auth=HTTPBasicAuth('username','password'))
#res = requests.get(url, auth=('username','password'))

第二种cookie携带方法

最简单的，把内容复制下来

headers = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36",
    "Connection": "keep-alive",
    'Cookie':'自己复制'
}
res = requests.get(url,headers=headers)

秘籍

访问http://httpbin.org/get，可以看到请求信息

{
“args”: {},
“headers”: {
“Accept”: “text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3”,
“Accept-Encoding”: “gzip, deflate”,
“Accept-Language”: “zh-CN,zh;q=0.9”,
“Host”: “httpbin.org”,
“Upgrade-Insecure-Requests”: “1”,
“User-Agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36”
},
“origin”: “124.236.115.242, 124.236.115.242”,
“url”: “https://httpbin.org/get”
}