获取一个post请求

需要使用字节文件来封装，模拟用户密码登陆界面

import urllib.parse #解析器
data = bytes(urllib.parse.urlencode({"user":"password"}),encoding="utf-8")
#把数据包转化成二进制
response = urllib.request.urlopen("http://httpbin.org/post",data=data)
print(response.read().decode("utf-8"))

Get超时处理

try:
    response = urllib.request.urlopen("http://httpbin.org/get",timeout=1)#timeout 超时判断
    print(response.read().decode("utf-8"))
except urllib.error.URLError as e:
    print("time out!")

爬网站头响应信息

response = urllib.request.urlopen("http://www.baidu.com")
print(response.status)
print(response.getheaders())#爬网站头文件信息
print(response.getheader("Bdpagetype"))#爬单个具体的信息

绕网站检测爬虫程序，伪装浏览器封装发送post

要给自己一个User-Agent

url = "https://www.douban.com"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36 Edg/94.0.992.47"
}
req = urllib.request.Request(url=url, headers=headers, method="POST")
response = urllib.request.urlopen(req)
print(response.read().decode("utf-8"))