1 Star 3 Fork 0

Programmer-1048576 / PythonTools

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
crawler.py 1.63 KB
一键复制 编辑 原始数据 按行查看 历史
Programmer-1048576 提交于 2020-06-10 19:04 . update crawler.py.
# Version: Test
# Author:LAI-1048576
# -*- coding:utf-8 -*-
from UA import UA
from urllib import request
from json import loads,dumps
import requests
class Response(object):
def __init__(self,res,coding="utf-8"):
self.res = res if isinstance(res,requests.Request) else None
self.data = res if isinstance(res,str) else res.text
self.bytes = res.encode(coding) if isinstance(res,str) else res.content
try:
self.json = loads(self.data)
except:
self.json = {"ERROR":"JSON DECODE FAILED"}
class Crawler(object):
def log(self,*msg):
if self.debug:
print(*msg)
def __init__(self,user_agent=UA,debug=True):
self.user_agent = user_agent
self.debug = debug
self.session = requests.session()
def GET(self,url:str,session = True):
if session:
res = self.session.get(url)
self.log(res.status_code)
return Response(res)
else:
res = request.urlopen(request.Request(url,headers=self.user_agent))
self.log(res.status)
return Response(res)
def POST(self,url:str,data:dict,session=True):
if session:
res = self.session.post(url,data=data)
self.log(res.status_code)
return Response(res)
else:
res = request.urlopen(request.Request(url,headers=self.user_agent,data=json.dumps(data)))
if __name__ == '__main__':
c = Crawler()
response = c.GET("https://shequ.codemao.cn/community")
print(response.data)
print(response.json)
Python
1
https://gitee.com/index-1048576/PythonTools.git
git@gitee.com:index-1048576/PythonTools.git
index-1048576
PythonTools
PythonTools
master

搜索帮助