2 Star 1 Fork 0

蜗牛山水/colite

Create your Gitee Account
Explore and code with more than 12 million developers,Free private repositories !:)
Sign up
Clone or Download
request.go 3.98 KB
Copy Edit Raw Blame History
蜗牛山水 authored 2020-03-08 14:01 . 初始化代码库
package colite
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync/atomic"
)
// Request 请求类
type Request struct {
// ID 请求标识符
ID uint32
// URL 请求网址
URL *url.URL
// Method 请求方法
Method string
// Headers 请求头
Headers *http.Header
// Body 请求主体(POST/PUT)
Body io.Reader
// Ctx 上下文环境
Ctx *Context
// ResponseCharacterEncoding 响应体字符编码(为空自动检测)
ResponseCharacterEncoding string
// ProxyURL 代理地址
ProxyURL string
collector *Collector
baseURL *url.URL
abort bool // 是否终止请求
}
// New 在原有请求基础上创建新请求实例(共享Collector、Context)
func (r *Request) New(URL, method string, body io.Reader) (*Request, error) {
u, err := url.Parse(URL)
if err != nil {
return nil, err
}
return &Request{
ID: atomic.AddUint32(&r.collector.requestCount, 1),
URL: u,
Method: method,
Headers: &http.Header{},
Body: body,
Ctx: r.Ctx, // 共享
collector: r.collector, // 共享
}, nil
}
// Abort 取消请求
func (r *Request) Abort() {
r.abort = true
}
// AbsoluteURL 解析为绝对网址(不包含Fragment部分)
func (r *Request) AbsoluteURL(u string) string {
if strings.HasPrefix(u, "#") {
return ""
}
var base *url.URL
if r.baseURL != nil {
base = r.baseURL
} else {
base = r.URL
}
absURL, err := base.Parse(u)
if err != nil {
return ""
}
absURL.Fragment = ""
if absURL.Scheme == "//" {
absURL.Scheme = r.URL.Scheme
}
return absURL.String()
}
// Visit 发送GET请求(保留上下文环境)
func (r *Request) Visit(URL string) error {
return r.collector.scrape(r.AbsoluteURL(URL), "GET", nil, r.Ctx, nil, true)
}
// HasVisited 检查指定网址是否已经访问
func (r *Request) HasVisited(URL string) (bool, error) {
return r.collector.HasVisited(URL)
}
// Post 发送POST请求(包含提交数据,保留上下文环境)
func (r *Request) Post(URL string, requestData map[string]string) error {
return r.collector.scrape(r.AbsoluteURL(URL), "POST", createFormReader(requestData), r.Ctx, nil, true)
}
// PostRaw 发送POST请求(包含实体数据,保留上下文环境)
func (r *Request) PostRaw(URL string, requestData []byte) error {
return r.collector.scrape(r.AbsoluteURL(URL), "POST", bytes.NewReader(requestData), r.Ctx, nil, true)
}
// PostMultipart 发送多部分实体POST请求(包含多部分实体数据,保留上下文环境)
func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error {
boundary := randomBoundary()
hdr := http.Header{}
hdr.Set("Content-Type", "multipart/form-data; boundary="+boundary)
hdr.Set("User-Agent", r.collector.UserAgent)
return r.collector.scrape(r.AbsoluteURL(URL), "POST", createMultipartReader(boundary, requestData), r.Ctx, hdr, true)
}
// Do 执行请求
func (r *Request) Do() error {
return r.collector.scrape(r.URL.String(), r.Method, r.Body, r.Ctx, *r.Headers, !r.collector.AllowURLRevisit)
}
// Retry 重新提交请求
func (r *Request) Retry() error {
r.Headers.Del("Cookie")
return r.collector.scrape(r.URL.String(), r.Method, r.Body, r.Ctx, *r.Headers, false)
}
// Marshal 将请求序列化为JSON
func (r *Request) Marshal() ([]byte, error) {
// 转存上下文环境
ctx := make(map[string]interface{})
if r.Ctx != nil {
r.Ctx.ForEach(func(k string, v interface{}) interface{} {
ctx[k] = v
return nil
})
}
// 转存请求主体
var err error
var body []byte
if r.Body != nil {
body, err = ioutil.ReadAll(r.Body)
if err != nil {
return nil, err
}
}
// 创建序列化请求
sr := &serializableRequest{
ID: r.ID,
URL: r.URL.String(),
Method: r.Method,
Body: body,
Ctx: ctx,
}
if r.Headers != nil {
sr.Headers = *r.Headers
}
return json.Marshal(sr)
}
// serializableRequest 序列化请求类
type serializableRequest struct {
ID uint32
URL string
Method string
Headers http.Header
Body []byte
Ctx map[string]interface{}
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Go
1
https://gitee.com/i2worker/colite.git
git@gitee.com:i2worker/colite.git
i2worker
colite
colite
v0.1.2

Search