Ai
1 Star 0 Fork 0

徐大周的春天/colly

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
proxy.go 1.56 KB
一键复制 编辑 原始数据 按行查看 历史
pyjac 提交于 2018-09-17 14:40 +08:00 . Proxy URL for request
// Copyright 2018 Adam Tauber
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package proxy
import (
"context"
"net/http"
"net/url"
"sync/atomic"
"github.com/gocolly/colly"
)
type roundRobinSwitcher struct {
proxyURLs []*url.URL
index uint32
}
func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) {
u := r.proxyURLs[r.index%uint32(len(r.proxyURLs))]
atomic.AddUint32(&r.index, 1)
ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String())
*pr = *pr.WithContext(ctx)
return u, nil
}
// RoundRobinProxySwitcher creates a proxy switcher function which rotates
// ProxyURLs on every request.
// The proxy type is determined by the URL scheme. "http", "https"
// and "socks5" are supported. If the scheme is empty,
// "http" is assumed.
func RoundRobinProxySwitcher(ProxyURLs ...string) (colly.ProxyFunc, error) {
urls := make([]*url.URL, len(ProxyURLs))
for i, u := range ProxyURLs {
parsedU, err := url.Parse(u)
if err != nil {
return nil, err
}
urls[i] = parsedU
}
return (&roundRobinSwitcher{urls, 0}).GetProxy, nil
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/xu_dazhous_spring/colly.git
git@gitee.com:xu_dazhous_spring/colly.git
xu_dazhous_spring
colly
colly
v1.2.0

搜索帮助