1 Star 0 Fork 0

ishmaelwanglin/download

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
download.go 6.70 KB
一键复制 编辑 原始数据 按行查看 历史
ishmaelwanglin 提交于 2023-10-25 15:44 . release: v0.0.4
package download
import (
"bufio"
"crypto/md5"
"fmt"
"io"
"math"
"net/http"
"os"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
)
// 线程数
var Processer int = runtime.NumCPU()
/*
分片
*/
type Trip struct {
Url string `json:"-"`
FileName string `json:"filename"` // 切片文件,全路径
Index int `json:"-"`
From int64 `json:"from"`
To int64 `json:"to"`
Size int64 `json:"size"`
Down bool `json:"over"` // 下载是否完成
Merged bool `json:"merged"` // 是否已合并当前分片
}
func (t *Trip) down(wg *sync.WaitGroup) error {
defer wg.Done()
/*
\ 如果文件存在,则断点续传
/ 如果不存在就正常下载
*/
var isExist bool
var reqFrom int64 = t.From
fileInfo, err := os.Stat(t.FileName)
// 文件如果没有下载完: 文件存在
if err == nil {
isExist = true
size := fileInfo.Size()
if size >= t.Size {
return nil
}
reqFrom += size
}
req, err := http.NewRequest(http.MethodGet, t.Url, nil)
if err != nil {
return err
}
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", reqFrom, t.To))
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode > 299 {
return fmt.Errorf("response code error")
}
// 调试: 打印一下response的header
// fmt.Println(resp.Header)
var f *os.File
if isExist {
f, err = os.OpenFile(t.FileName, os.O_RDWR, os.ModePerm)
if err != nil {
return err
}
_, err := f.Seek(0, io.SeekEnd)
if err != nil {
return err
}
} else {
f, err = os.Create(t.FileName)
if err != nil {
return err
}
}
defer f.Close()
// _, err = io.Copy(f, resp.Body)
// if err != nil {
// return err
// }
b := make([]byte, 1024*1024)
w := bufio.NewWriter(f)
for {
n, err := resp.Body.Read(b)
if err == io.EOF {
break
}
if err != nil {
return err
}
_, err = w.Write(b[:n])
if err != nil {
return err
}
}
// 下载分片完成
t.Down = true
return nil
}
type DownLoader struct {
Url string `json:"-"`
FileName string `json:"filename"`
Dir string `json:"-"` // 下载文件存放的目录
Length int64 `json:"-"` // 目标的content-length
WG *sync.WaitGroup `json:"-"`
Threads int `json:"threads"`
Meta string `json:"-"` // 元数据文件,全路径
Trips []*Trip `json:"trips"` // 分片的列表
}
/*
创建一个下载器,用于下载.
<url>: 目标的URL;
[dir]: 存放的目录.
*/
func New(url string, dir ...string) *DownLoader {
d := DownLoader{
Url: url,
FileName: path.Base(url),
Threads: Processer,
WG: new(sync.WaitGroup),
}
var destDir string
if len(dir) == 0 {
destDir, _ = os.Getwd()
} else {
destDir = dir[0]
}
d.SetDir(destDir)
return &d
}
func (d *DownLoader) GetPath() string {
return d.Dir + "/" + d.FileName
}
func (d *DownLoader) SetThread(t int) {
if t == 0 {
t = Processer
}
d.Threads = t
}
func (d *DownLoader) SetDir(dir string) {
// 空目录
if dir == "" {
dir, _ = os.Getwd()
goto RETURN
}
// 去掉后边的"/"
if strings.HasSuffix(dir, "/") {
dir = strings.TrimRight(dir, "/")
}
// 不是绝对路径
if !filepath.IsAbs(dir) {
// 相对路径
if strings.HasPrefix(dir, "./") || strings.HasPrefix(dir, "../") {
dir, _ = filepath.Abs(dir)
} else {
// 只有目录名
wd, _ := os.Getwd()
dir = wd + "/" + dir
}
}
RETURN:
d.Dir = dir
}
func (d *DownLoader) Down() error {
accept, err := d.AcceptRages()
if err != nil {
return err
}
if accept {
if err := d.multiTreadDown(); err != nil {
return err
}
} else {
if err := d.sigleDown(); err != nil {
return err
}
}
return nil
}
/*
是否支持分段断点续传,
如果支持断点则获取长度
*/
func (d *DownLoader) AcceptRages() (accept bool, err error) {
req, err := http.NewRequest(http.MethodHead, d.Url, nil)
if err != nil {
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return
}
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("response code: %d", resp.StatusCode)
return
}
defer resp.Body.Close()
if resp.Header.Get("Accept-Ranges") != "bytes" {
return
}
accept = true
length, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
if err != nil {
return
}
d.Length = length
return
}
// 多线程下载方法
func (d *DownLoader) multiTreadDown() error {
_, err := os.Stat(d.Dir)
if os.IsNotExist(err) {
err = os.MkdirAll(d.Dir, os.ModeDir|os.ModePerm)
}
if err != nil {
return err
}
// 计算分片的SIZE
size := int64(math.Ceil(float64(d.Length) / float64(d.Threads)))
var start, end int64
// 分片
for i := 0; i < d.Threads; i++ {
if i == d.Threads-1 {
end = d.Length - 1
} else {
end = start + size - 1
}
trip := &Trip{
FileName: fmt.Sprintf("%s/.%s-%s.%d", d.Dir, d.FileName, "temp", i),
Index: i,
From: start,
To: end,
Url: d.Url,
Size: end - start + 1,
}
d.Trips = append(d.Trips, trip)
start += size
}
d.Meta = fmt.Sprintf("%s/.%s-%s.%d.json", d.Dir, d.FileName, "meta", d.Threads)
for _, v := range d.Trips {
d.WG.Add(1)
go v.down(d.WG)
}
d.WG.Wait()
// 合并失败后,返回报错
if err := d.merge(); err != nil {
return err
}
return nil
}
// 合并分片成一个文件
func (d *DownLoader) merge() error {
fp := d.Dir + "/" + d.FileName
dest, err := os.OpenFile(fp, os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
return err
}
for _, v := range d.Trips {
f, err := os.Open(v.FileName)
if err != nil {
return err
}
defer os.Remove(v.FileName)
defer f.Close()
dest.Seek(v.From, io.SeekStart)
r := bufio.NewReader(f)
var sizeof int
buff := make([]byte, 1024*1024)
for {
rc, err := r.Read(buff)
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("read %s failure", f.Name())
}
wc, err := dest.Write(buff[:rc])
if err != nil {
return err
}
sizeof += wc
}
if int64(sizeof) != v.Size {
return fmt.Errorf("sizeof %d not eq trip size %d", sizeof, v.Size)
}
// 当前分片merge成功
v.Merged = true
}
return nil
}
func (d *DownLoader) sigleDown() error {
resp, err := http.Get(d.Url)
if err != nil {
return err
}
defer resp.Body.Close()
f, err := os.Create(d.Dir + "/" + d.FileName)
if err != nil {
return err
}
r := bufio.NewReader(resp.Body)
w := bufio.NewWriter(f)
_, err = io.Copy(w, r)
if err != nil {
return err
}
return nil
}
func Md5sum(fp string) (md5sum string, err error) {
f, err := os.Open(fp)
if err != nil {
return
}
reader := bufio.NewReader(f)
h := md5.New()
_, err = io.Copy(h, reader)
if err != nil {
return
}
md5sum = fmt.Sprintf("%x", h.Sum(nil))
return
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Go
1
https://gitee.com/ishmaelwanglin/download.git
git@gitee.com:ishmaelwanglin/download.git
ishmaelwanglin
download
download
v0.0.4

搜索帮助