1 Star 0 Fork 0

J-star / BaiduPCS-Go

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
download.go 14.92 KB
一键复制 编辑 原始数据 按行查看 历史
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598
package pcscommand
import (
"encoding/hex"
"errors"
"fmt"
"github.com/iikira/BaiduPCS-Go/baidupcs"
"github.com/iikira/BaiduPCS-Go/baidupcs/pcserror"
"github.com/iikira/BaiduPCS-Go/internal/pcsconfig"
"github.com/iikira/BaiduPCS-Go/pcstable"
"github.com/iikira/BaiduPCS-Go/pcsutil/checksum"
"github.com/iikira/BaiduPCS-Go/pcsutil/converter"
"github.com/iikira/BaiduPCS-Go/pcsutil/waitgroup"
"github.com/iikira/BaiduPCS-Go/requester"
"github.com/iikira/BaiduPCS-Go/requester/downloader"
"github.com/oleiade/lane"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"time"
)
const (
//DownloadSuffix 文件下载后缀
DownloadSuffix = ".BaiduPCS-Go-downloading"
//StrDownloadInitError 初始化下载发生错误
StrDownloadInitError = "初始化下载发生错误"
// DefaultDownloadMaxRetry 默认下载失败最大重试次数
DefaultDownloadMaxRetry = 3
)
var (
// ErrNotSupportChecksum 文件不支持校验
ErrNotSupportChecksum = errors.New("该文件不支持校验")
// ErrChecksumFailed 文件校验失败
ErrChecksumFailed = errors.New("该文件校验失败, 文件md5值与服务器记录的不匹配")
)
type (
// dtask 下载任务
dtask struct {
ListTask
path string // 下载的路径
savePath string // 保存的路径
downloadInfo *baidupcs.FileDirectory // 文件或目录详情
}
//DownloadOptions 下载可选参数
DownloadOptions struct {
IsTest bool
IsPrintStatus bool
IsExecutedPermission bool
IsOverwrite bool
IsShareDownload bool
IsLocateDownload bool
IsStreaming bool
SaveTo string
Parallel int
Load int
MaxRetry int
NoCheck bool
Out io.Writer
}
)
func downloadPrintFormat(load int) string {
if load <= 1 {
return "\r[%d] ↓ %s/%s %s/s in %s, left %s ............"
}
return "[%d] ↓ %s/%s %s/s in %s, left %s ...\n"
}
func download(id int, downloadURL, savePath string, loadBalansers []string, client *requester.HTTPClient, newCfg downloader.Config, downloadOptions *DownloadOptions) error {
var (
file *os.File
writerAt io.WriterAt
err error
exitChan chan struct{}
)
if !newCfg.IsTest {
newCfg.InstanceStatePath = savePath + DownloadSuffix
// 创建下载的目录
dir := filepath.Dir(savePath)
fileInfo, err := os.Stat(dir)
if err != nil {
err = os.MkdirAll(dir, 0777)
if err != nil {
return err
}
} else if !fileInfo.IsDir() {
return fmt.Errorf("%s, path %s: not a directory", StrDownloadInitError, dir)
}
file, err = os.OpenFile(savePath, os.O_CREATE|os.O_WRONLY, 0666)
if file != nil {
defer file.Close()
}
if err != nil {
return fmt.Errorf("%s, %s", StrDownloadInitError, err)
}
// 空指针和空接口不等价
if file != nil {
writerAt = file
}
}
download := downloader.NewDownloader(downloadURL, writerAt, &newCfg)
download.SetClient(client)
download.TryHTTP(!pcsconfig.Config.EnableHTTPS())
download.AddLoadBalanceServer(loadBalansers...)
exitChan = make(chan struct{})
download.OnExecute(func() {
if downloadOptions.IsPrintStatus {
go func() {
for {
time.Sleep(1 * time.Second)
select {
case <-exitChan:
return
default:
download.PrintAllWorkers()
}
}
}()
}
if newCfg.IsTest {
fmt.Fprintf(downloadOptions.Out, "[%d] 测试下载开始\n\n", id)
}
var (
ds = download.GetDownloadStatusChan()
format = downloadPrintFormat(downloadOptions.Load)
downloaded, totalSize, speeds int64
leftStr string
)
for {
select {
case <-exitChan:
return
case v, ok := <-ds:
if !ok { // channel 已经关闭
return
}
downloaded, totalSize, speeds = v.Downloaded(), v.TotalSize(), v.SpeedsPerSecond()
if speeds <= 0 {
leftStr = "-"
} else {
leftStr = (time.Duration((totalSize-downloaded)/(speeds)) * time.Second).String()
}
fmt.Fprintf(downloadOptions.Out, format, id,
converter.ConvertFileSize(v.Downloaded(), 2),
converter.ConvertFileSize(v.TotalSize(), 2),
converter.ConvertFileSize(v.SpeedsPerSecond(), 2),
v.TimeElapsed()/1e7*1e7, leftStr,
)
}
}
})
err = download.Execute()
close(exitChan)
fmt.Fprintf(downloadOptions.Out, "\n")
if err != nil {
// 下载失败, 删去空文件
if info, infoErr := file.Stat(); infoErr == nil {
if info.Size() == 0 {
pcsCommandVerbose.Infof("[%d] remove empty file: %s\n", id, savePath)
os.Remove(savePath)
}
}
return err
}
if downloadOptions.IsExecutedPermission {
err = file.Chmod(0766)
if err != nil {
fmt.Fprintf(downloadOptions.Out, "[%d] 警告, 加执行权限错误: %s\n", id, err)
}
}
if !newCfg.IsTest {
fmt.Fprintf(downloadOptions.Out, "[%d] 下载完成, 保存位置: %s\n", id, savePath)
} else {
fmt.Fprintf(downloadOptions.Out, "[%d] 测试下载结束\n", id)
}
return nil
}
// checkFileValid 检测文件有效性
func checkFileValid(filePath string, fileInfo *baidupcs.FileDirectory) error {
if len(fileInfo.BlockList) != 1 {
return ErrNotSupportChecksum
}
f := checksum.NewLocalFileInfo(filePath, int(256*converter.KB))
err := f.OpenPath()
if err != nil {
return err
}
defer f.Close()
f.Md5Sum()
if strings.Compare(hex.EncodeToString(f.MD5), fileInfo.MD5) != 0 {
return ErrChecksumFailed
}
return nil
}
// RunDownload 执行下载网盘内文件
func RunDownload(paths []string, options *DownloadOptions) {
if options == nil {
options = &DownloadOptions{}
}
if options.Out == nil {
options.Out = os.Stdout
}
if options.Load <= 0 {
options.Load = pcsconfig.Config.MaxDownloadLoad()
}
if options.MaxRetry < 0 {
options.MaxRetry = DefaultDownloadMaxRetry
}
// 设置下载配置
cfg := &downloader.Config{
IsTest: options.IsTest,
CacheSize: pcsconfig.Config.CacheSize(),
}
// 设置下载最大并发量
if options.Parallel < 1 {
options.Parallel = pcsconfig.Config.MaxParallel()
}
paths, err := getAllAbsPaths(paths...)
if err != nil {
fmt.Println(err)
return
}
fmt.Fprintf(options.Out, "\n")
fmt.Fprintf(options.Out, "[0] 提示: 当前下载最大并发量为: %d, 下载缓存为: %d\n", options.Parallel, cfg.CacheSize)
var (
pcs = GetBaiduPCS()
dlist = lane.NewDeque()
lastID = 0
loadCount = 0
)
// 预测要下载的文件数量
// TODO: pcscache
for k := range paths {
pcs.FilesDirectoriesRecurseList(paths[k], baidupcs.DefaultOrderOptions, func(depth int, _ string, fd *baidupcs.FileDirectory, pcsError pcserror.Error) bool {
if pcsError != nil {
pcsCommandVerbose.Warnf("%s\n", pcsError)
return true
}
if !fd.Isdir {
loadCount++
if loadCount >= options.Load {
return false
}
}
return true
})
if loadCount >= options.Load {
break
}
}
// 修改Load, 设置MaxParallel
if loadCount > 0 {
options.Load = loadCount
// 取平均值
cfg.MaxParallel = pcsconfig.AverageParallel(options.Parallel, loadCount)
} else {
cfg.MaxParallel = options.Parallel
}
// 处理队列
for k := range paths {
lastID++
ptask := &dtask{
ListTask: ListTask{
ID: lastID,
MaxRetry: options.MaxRetry,
},
path: paths[k],
}
if options.SaveTo != "" {
ptask.savePath = filepath.Join(options.SaveTo, filepath.Base(paths[k]))
} else {
ptask.savePath = GetActiveUser().GetSavePath(paths[k])
}
dlist.Append(ptask)
fmt.Fprintf(options.Out, "[%d] 加入下载队列: %s\n", lastID, paths[k])
}
var (
totalSize int64
failedList []string
handleTaskErr = func(task *dtask, errManifest string, err error) {
if task == nil {
panic("task is nil")
}
if err == nil {
return
}
// 不重试的情况
switch {
case err == ErrNotSupportChecksum:
fallthrough
case strings.Compare(errManifest, "下载文件错误") == 0 && strings.Contains(err.Error(), StrDownloadInitError):
fmt.Fprintf(options.Out, "[%d] %s, %s\n", task.ID, errManifest, err)
return
}
// 未达到失败重试最大次数, 将任务推送到队列末尾
if task.retry < task.MaxRetry {
task.retry++
fmt.Fprintf(options.Out, "[%d] %s, %s, 重试 %d/%d\n", task.ID, errManifest, err, task.retry, task.MaxRetry)
dlist.Append(task)
time.Sleep(3 * time.Duration(task.retry) * time.Second)
} else {
failedList = append(failedList, task.path)
}
switch err {
case ErrChecksumFailed:
// 删去旧的文件, 重新下载
rerr := os.Remove(task.savePath)
if rerr != nil {
fmt.Fprintf(options.Out, "[%d] 删除校验失败的文件出错, %s\n", task.ID, rerr)
failedList = append(failedList, task.path)
return
}
fmt.Fprintf(options.Out, "[%d] 已删除校验失败的文件\n", task.ID)
}
}
startTime = time.Now()
wg = waitgroup.NewWaitGroup(options.Load)
)
for {
e := dlist.Shift()
if e == nil { // 任务为空
if wg.Parallel() == 0 { // 结束
break
} else {
time.Sleep(1e9)
continue
}
}
task := e.(*dtask)
wg.AddDelta()
go func() {
defer wg.Done()
if task.downloadInfo == nil {
task.downloadInfo, err = pcs.FilesDirectoriesMeta(task.path)
if err != nil {
// 不重试
fmt.Printf("[%d] 获取路径信息错误, %s\n", task.ID, err)
return
}
}
fmt.Fprintf(options.Out, "\n")
fmt.Fprintf(options.Out, "[%d] ----\n%s\n", task.ID, task.downloadInfo.String())
// 如果是一个目录, 将子文件和子目录加入队列
if task.downloadInfo.Isdir {
if !options.IsTest { // 测试下载, 不建立空目录
os.MkdirAll(task.savePath, 0777) // 首先在本地创建目录, 保证空目录也能被保存
}
fileList, err := pcs.FilesDirectoriesList(task.path, baidupcs.DefaultOrderOptions)
if err != nil {
// 不重试
fmt.Fprintf(options.Out, "[%d] 获取目录信息错误, %s\n", task.ID, err)
return
}
for k := range fileList {
lastID++
subTask := &dtask{
ListTask: ListTask{
ID: lastID,
MaxRetry: options.MaxRetry,
},
path: fileList[k].Path,
downloadInfo: fileList[k],
}
if options.SaveTo != "" {
subTask.savePath = filepath.Join(task.savePath, fileList[k].Filename)
} else {
subTask.savePath = GetActiveUser().GetSavePath(subTask.path)
}
dlist.Append(subTask)
fmt.Fprintf(options.Out, "[%d] 加入下载队列: %s\n", lastID, fileList[k].Path)
}
return
}
fmt.Fprintf(options.Out, "[%d] 准备下载: %s\n", task.ID, task.path)
if !options.IsTest && !options.IsOverwrite && fileExist(task.savePath) {
fmt.Fprintf(options.Out, "[%d] 文件已经存在: %s, 跳过...\n", task.ID, task.savePath)
return
}
if !options.IsTest {
fmt.Fprintf(options.Out, "[%d] 将会下载到路径: %s\n\n", task.ID, task.savePath)
}
// 获取直链, 或者以分享文件的方式获取下载链接来下载
var (
dlink string
dlinks []string
)
if options.IsLocateDownload {
// 提取直链下载
rawDlinks := getDownloadLinks(task.path)
if len(rawDlinks) > 0 {
dlink = rawDlinks[0].String()
dlinks = make([]string, 0, len(rawDlinks)-1)
for _, rawDlink := range rawDlinks[1:len(rawDlinks)] {
if rawDlink == nil {
continue
}
dlinks = append(dlinks, rawDlink.String())
}
}
} else if options.IsShareDownload {
// 分享下载
dlink = getShareDLink(task.path)
}
if dlink != "" {
pcsCommandVerbose.Infof("[%d] 获取到下载链接: %s\n", task.ID, dlink)
client := pcsconfig.Config.HTTPClient()
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
// 去掉 Referer
if !pcsconfig.Config.EnableHTTPS() {
req.Header.Del("Referer")
}
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
return nil
}
client.SetTimeout(20 * time.Minute)
client.SetKeepAlive(true)
err = download(task.ID, dlink, task.savePath, dlinks, client, *cfg, options)
} else {
dfunc := func(downloadURL string, jar http.CookieJar) error {
h := pcsconfig.Config.HTTPClient()
h.SetCookiejar(jar)
h.SetKeepAlive(true)
h.SetTimeout(10 * time.Minute)
err := download(task.ID, downloadURL, task.savePath, dlinks, h, *cfg, options)
if err != nil {
return err
}
return nil
}
if options.IsStreaming {
err = pcs.DownloadStreamFile(task.path, dfunc)
} else {
err = pcs.DownloadFile(task.path, dfunc)
}
}
if err != nil {
handleTaskErr(task, "下载文件错误", err)
return
}
if !cfg.IsTest && !options.NoCheck {
if task.downloadInfo.Size >= 128*converter.MB {
fmt.Fprintf(options.Out, "[%d] 开始检验文件有效性, 稍后...\n", task.ID)
}
err = checkFileValid(task.savePath, task.downloadInfo)
if err != nil {
handleTaskErr(task, "检验文件有效性出错", err)
return
}
fmt.Fprintf(options.Out, "[%d] 检验文件有效性成功\n", task.ID)
}
atomic.AddInt64(&totalSize, task.downloadInfo.Size)
}()
}
wg.Wait()
fmt.Fprintf(options.Out, "\n任务结束, 时间: %s, 数据总量: %s\n", time.Since(startTime)/1e6*1e6, converter.ConvertFileSize(totalSize))
if len(failedList) != 0 {
fmt.Printf("以下文件下载失败: \n")
tb := pcstable.NewTable(os.Stdout)
for k := range failedList {
tb.Append([]string{strconv.Itoa(k), failedList[k]})
}
tb.Render()
}
}
// RunLocateDownload 执行获取直链
func RunLocateDownload(pcspaths ...string) {
absPaths, err := getAllAbsPaths(pcspaths...)
if err != nil {
fmt.Println(err)
return
}
pcs := GetBaiduPCS()
for i, pcspath := range absPaths {
info, err := pcs.LocateDownload(pcspath)
if err != nil {
fmt.Printf("[%d] %s, 路径: %s\n", i, err, pcspath)
continue
}
fmt.Printf("[%d] %s: \n", i, pcspath)
tb := pcstable.NewTable(os.Stdout)
tb.SetHeader([]string{"#", "链接"})
for k, u := range info.URLStrings(pcsconfig.Config.EnableHTTPS()) {
tb.Append([]string{strconv.Itoa(k), u.String()})
}
tb.Render()
fmt.Println()
}
fmt.Printf("提示: 访问下载链接, 需将下载器的 User-Agent 设置为: %s\n", pcsconfig.Config.UserAgent())
}
func getDownloadLinks(pcspath string) (dlinks []*url.URL) {
pcs := GetBaiduPCS()
dInfo, pcsError := pcs.LocateDownload(pcspath)
if pcsError != nil {
pcsCommandVerbose.Warn(pcsError.Error())
return
}
us := dInfo.URLStrings(pcsconfig.Config.EnableHTTPS())
if len(us) == 0 {
pcsCommandVerbose.Warn("no any url")
return
}
return us
}
// fileExist 检查文件是否存在,
// 只有当文件存在, 文件大小不为0或断点续传文件不存在时, 才判断为存在
func fileExist(path string) bool {
if info, err := os.Stat(path); err == nil {
if info.Size() == 0 {
return false
}
if _, err = os.Stat(path + DownloadSuffix); err != nil {
return true
}
}
return false
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/j-star/BaiduPCS-Go.git
git@gitee.com:j-star/BaiduPCS-Go.git
j-star
BaiduPCS-Go
BaiduPCS-Go
v3.5.6

搜索帮助

344bd9b3 5694891 D2dac590 5694891