From 800321bf8da55a554b526dd680bcb6337b748da1 Mon Sep 17 00:00:00 2001 From: qjfoidnh Date: Sat, 9 Jan 2021 14:15:24 +0800 Subject: [PATCH] Update v3.7.2 --- README.md | 49 +++++++++++---- baidupcs/download.go | 4 +- baidupcs/netdisksign/devuid.go | 6 +- baidupcs/netdisksign/locatedownloadsign.go | 2 +- baidupcs/prepare.go | 15 ++++- baidupcs/transfer.go | 11 +++- build.sh | 20 +++--- go.mod | 2 +- go.sum | 6 +- internal/pcscommand/download.go | 31 +++++----- internal/pcscommand/export.go | 34 ++++++---- internal/pcscommand/login.go | 46 ++++++++++---- internal/pcscommand/transfer.go | 1 + .../pcsdownload/download_task_unit.go | 62 +++++++++---------- .../pcsfunctions/pcsdownload/pcsdownload.go | 2 +- main.go | 7 ++- requester/downloader/config.go | 5 +- requester/downloader/downloader.go | 33 +++++++--- requester/downloader/monitor.go | 7 ++- requester/transfer/rangelist.go | 10 +++ versioninfo.json | 8 +-- 21 files changed, 240 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index fea3bd4..26224a1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ iikira/BaiduPCS-Go was largely inspired by [GangZhuo/BaiduPCS](https://github.co ## 注意 -此版本基于iikira原版BaiduPCS-Go最新版本(v3.6.2)修改, 添加了转存功能, 替换了文档中一些失效的链接和页面并更新少量使用说明. +此版本基于iikira原版BaiduPCS-Go v3.6.2继续开发, 并添加了转存功能. + +本软件不提供超出官方客户端的下载提速, 普通用户和SVIP的配置建议参见 [显示和修改程序配置项](#显示和修改程序配置项) ## 目录 @@ -96,16 +98,33 @@ iikira/BaiduPCS-Go was largely inspired by [GangZhuo/BaiduPCS](https://github.co [离线下载](#离线下载), 支持http/https/ftp/电驴/磁力链协议. # 版本更新 -**2020.12.19** -v3.7.0: + +**2021.1.9** v3.7.2: + +- 基本修复了登录验证失效问题(#15) +- 优化下载模块的实现策略, 保证稳定性同时进一步提升下载速度 (需按[显示和修改程序配置项](#显示和修改程序配置项)中建议修改) +- update 功能恢复, 以后可以在线升级了 +- 支持导出秒传链接不写文件, 直接输出到控制台; 支持通用秒传格式导出, 具体参见export --help +- 其他bug修正 + +**2021.1.2** v3.7.1: + +- 支持了多文件并发上传，文件并发数和单文件分片数可在配置中指定 +- 修复了最大同时下载文件数配置不生效的问题 +- 修正了部分显示和帮助的错误 + +**2020.12.19** v3.7.0: + * 替换了iikira版本的失效仓库 * 转存功能支持旧的短链接 * 默认关闭下载文件校验，配置文件可设置开启 * 修复了关闭校验时会误报下载失败的问题 * 转存功能除了cookies方式登录，现已支持用户名密码登录和bduss登录；bduss登录需同时指定stoken -**2020.11.08** -v3.6.3: 修复转存失败, 修复分享文件失败 +**2020.11.08** v3.6.3: + +* 修复转存失败 +* 修复分享文件失败 # 编译/交叉编译 说明 @@ -413,17 +432,17 @@ BaiduPCS-Go d <网盘文件或目录的路径1> <文件或目录2> <文件或目 通过 `BaiduPCS-Go config set -savedir `, 自定义保存的目录. 支持多个文件或目录下载. - + 支持下载完成后自动校验文件, 但并不是所有的文件都支持校验! - + 自动跳过下载重名的文件! #### 下载模式说明 -* pcs: 通过百度网盘的 PCS API 下载(已废弃) +* pcs: 通过百度网盘的 PCS API 下载(不建议使用) -* stream: 通过百度网盘的 PCS API, 以流式文件的方式下载, 效果同 pcs(已废弃) +* stream: 通过百度网盘的 PCS API, 以流式文件的方式下载, 效果同 pcs(不建议使用) * locate: 默认的下载模式。从百度网盘 Android 客户端, 获取下载链接的方式来下载 @@ -835,7 +854,11 @@ Windows: `%APPDATA%\BaiduPCS-Go` `cache_size` 的值支持可选设置单位了, 单位不区分大小写, `b` 和 `B` 均表示字节的意思, 如 `64KB`, `1MB`, `32kb`, `65536b`, `65536`. -`max_upload_parallel`, `max_download_load` 的值支持可选设置单位了, 单位为每秒的传输速率, 后缀`/s` 可省略, 如 `2MB/s`, `2MB`, `2m`, `2mb` 均为一个意思. +`max_download_rate`, `max_upload_rate` 的值支持可选设置单位了, 单位为每秒的传输速率, 后缀`/s` 可省略, 如 `2MB/s`, `2MB`, `2m`, `2mb` 均为一个意思. + +普通用户请将`max_parallel`和`max_download_load`都设置为1, 调大线程数只会在短时间内提升下载速度, 且极易很快触发限速, 导致几小时至几天内账号在各客户端都接近0速. 本软件不支持普通用户提速. + +SVIP用户建议`max_parallel`设置为9 - 13, `max_download_load`设置为1 - 2, 可以维持长时间的稳定高速下载. #### 例子 ``` @@ -965,13 +988,13 @@ cli交互模式下, 运行命令 `config set -max_parallel 2` 将下载最大并 * 分片上传文件时, 当文件分片数大于1, 网盘端最终计算所得的md5值和本地的不一致, 这可能是百度网盘的bug, 测试把上传的文件下载到本地后，对比md5值是匹配的. 可通过秒传的原理来修复md5值. * 开启MD5校验下载时可能有 check MD5 不通过, 但文件其实并未出错的情况, 使用--no-check下载或配置中启用no_check即可(3.7版本默认已启用). -* 下载线程, 即max_parallel参数设置过大时可能导致账号进入黑名单状态, 建议普通用户设置为1, 超级会员可尝试调大, 但不建议超过5. -* update功能已失效, 短期无修复计划. +* 用户名登录时图片验证码至少要输入两次, 第一次的输入无效 +* 登录出现手机/邮箱验证时要输入至少4次图片验证码 # TODO - +* 转存文件数量绕过单次限制 # 交流反馈 diff --git a/baidupcs/download.go b/baidupcs/download.go index 65106ae..d1f242e 100644 --- a/baidupcs/download.go +++ b/baidupcs/download.go @@ -11,8 +11,8 @@ import ( var ( // ErrLocateDownloadURLNotFound 未找到下载链接 ErrLocateDownloadURLNotFound = errors.New("locatedownload url not found") - // MaxDownloadRangeSize 文件片段最大值 - MaxDownloadRangeSize = 55 * converter.MB + // InitRangeSize 初次请求的片段值 + InitRangeSize = 32 * converter.KB ) type ( diff --git a/baidupcs/netdisksign/devuid.go b/baidupcs/netdisksign/devuid.go index 575fbf7..a7f44d0 100644 --- a/baidupcs/netdisksign/devuid.go +++ b/baidupcs/netdisksign/devuid.go @@ -13,8 +13,8 @@ func DevUID(feature string) string { m.Write(converter.ToBytes(feature)) res := m.Sum(nil) resHex := cachepool.RawMallocByteSlice(34) - hex.Encode(resHex[2:], res) - resHex[0] = 'O' - resHex[1] = '|' + hex.Encode(resHex[:32], res) + resHex[32] = '|' + resHex[33] = '0' return converter.ToString(bytes.ToUpper(resHex)) } diff --git a/baidupcs/netdisksign/locatedownloadsign.go b/baidupcs/netdisksign/locatedownloadsign.go index d8ae5cb..bb0772a 100644 --- a/baidupcs/netdisksign/locatedownloadsign.go +++ b/baidupcs/netdisksign/locatedownloadsign.go @@ -48,5 +48,5 @@ func (s *LocateDownloadSign) Sign(uid uint64, bduss string) { } func (s *LocateDownloadSign) URLParam() string { - return "time=" + strconv.FormatInt(s.Time, 10) + "&rand=" + s.Rand + "&devuid=" + s.DevUID + return "time=" + strconv.FormatInt(s.Time, 10) + "&rand=" + s.Rand + "&devuid=" + s.DevUID + "&cuid=" + s.DevUID } diff --git a/baidupcs/prepare.go b/baidupcs/prepare.go index fd72a77..abc195d 100644 --- a/baidupcs/prepare.go +++ b/baidupcs/prepare.go @@ -317,10 +317,23 @@ func (pcs *BaiduPCS) PrepareLocateDownload(pcspath string) (dataReadCloser io.Re Host: PCSBaiduCom, Path: "/rest/2.0/pcs/file", RawQuery: (url.Values{ + "check_blue": []string{"1"}, + "es": []string{"1"}, + "esl": []string{"1"}, "app_id": []string{PanAppID}, "method": []string{"locatedownload"}, "path": []string{pcspath}, - "ver": []string{"2"}, + "ver": []string{"4.0"}, + //"vip": []string{"2"}, + "clienttype": []string{"17"}, + "version": []string{"2.271.76"}, + "channel": []string{"0"}, + "version_app": []string{"10.1.72"}, + "apn_id": []string{"1_0"}, + "freeisp": []string{"0"}, + "queryfree": []string{"0"}, + "use": []string{"0"}, + }).Encode() + "&" + ns.URLParam(), } baiduPCSVerbose.Infof("%s URL: %s\n", OperationLocateDownload, pcsURL) diff --git a/baidupcs/transfer.go b/baidupcs/transfer.go index 0bad2b9..f3115d7 100644 --- a/baidupcs/transfer.go +++ b/baidupcs/transfer.go @@ -172,7 +172,16 @@ func (pcs *BaiduPCS) GenerateRequestQuery(mode string, params map[string]string) if mode == "POST" && errno == 12 { path := gjson.Get(string(body), `info.0.path`).String() _, file := filepath.Split(path) - res["ErrMsg"] = fmt.Sprintf("当前目录下已有%s同名文件/文件夹", file) + _errno := gjson.Get(string(body), `info.0.errno`).Int() + if _errno == -33 { + filenum := gjson.Get(string(body), `target_file_nums`).Int() + userlimit := gjson.Get(string(body), `target_file_nums_limit`).Int() + res["ErrMsg"] = fmt.Sprintf("转存文件数%d超过当前用户上限, 当前用户单次最大转存数%d", filenum, userlimit) + } else if _errno == -30 { + res["ErrMsg"] = fmt.Sprintf("当前目录下已有%s同名文件/文件夹", file) + } else { + res["ErrMsg"] = fmt.Sprintf("未知错误, 错误代码%d", _errno) + } } else { res["ErrMsg"] = fmt.Sprintf("未知错误, 错误代码%d", errno) } diff --git a/build.sh b/build.sh index 2391904..f5a89af 100755 --- a/build.sh +++ b/build.sh @@ -4,13 +4,13 @@ name="BaiduPCS-Go" version=$1 if [ "$1" = "" ]; then - version=v3.7.0 + version=v3.7.1 fi output="out/" old_golang() { - GOROOT=/usr/local/go1.10.8 + GOROOT=/usr/local/go go=$GOROOT/bin/go } @@ -91,16 +91,16 @@ RicePack() { touch ./vendor/golang.org/x/sys/windows/windows.s # Android -export NDK_INSTALL=$ANDROID_NDK_ROOT/bin +#export NDK_INSTALL=$ANDROID_NDK_ROOT/bin # CC=$NDK_INSTALL/arm-linux-androideabi-4.9/bin/arm-linux-androideabi-gcc AndroidBuild $name-$version"-android-16-armv5" android arm 5 # CC=$NDK_INSTALL/arm-linux-androideabi-4.9/bin/arm-linux-androideabi-gcc AndroidBuild $name-$version"-android-16-armv6" android arm 6 -CC=$NDK_INSTALL/arm-linux-androideabi-4.9/bin/arm-linux-androideabi-gcc AndroidBuild $name-$version"-android-16-armv7" android arm 7 -CC=$NDK_INSTALL/aarch64-linux-android-4.9/bin/aarch64-linux-android-gcc AndroidBuild $name-$version"-android-21-arm64" android arm64 7 -CC=$NDK_INSTALL/i686-linux-android-4.9/bin/i686-linux-android-gcc AndroidBuild $name-$version"-android-16-386" android 386 7 -CC=$NDK_INSTALL/x86_64-linux-android-4.9/bin/x86_64-linux-android-gcc AndroidBuild $name-$version"-android-21-amd64" android amd64 7 +#CC=$NDK_INSTALL/arm-linux-androideabi-4.9/bin/arm-linux-androideabi-gcc AndroidBuild $name-$version"-android-16-armv7" android arm 7 +#CC=$NDK_INSTALL/aarch64-linux-android-4.9/bin/aarch64-linux-android-gcc AndroidBuild $name-$version"-android-21-arm64" android arm64 7 +#CC=$NDK_INSTALL/i686-linux-android-4.9/bin/i686-linux-android-gcc AndroidBuild $name-$version"-android-16-386" android 386 7 +#CC=$NDK_INSTALL/x86_64-linux-android-4.9/bin/x86_64-linux-android-gcc AndroidBuild $name-$version"-android-21-amd64" android amd64 7 # iOS -IOSBuild $name-$version"-darwin-ios-arm" +#IOSBuild $name-$version"-darwin-ios-arm" # OS X / macOS Build $name-$version"-darwin-osx-amd64" darwin amd64 @@ -113,8 +113,8 @@ Build $name-$version"-windows-x64" windows amd64 # Linux Build $name-$version"-linux-386" linux 386 Build $name-$version"-linux-amd64" linux amd64 -Build $name-$version"-linux-armv5" linux arm 5 -Build $name-$version"-linux-armv7" linux arm 7 +#Build $name-$version"-linux-armv5" linux arm 5 +Build $name-$version"-linux-arm" linux arm Build $name-$version"-linux-arm64" linux arm64 GOMIPS=softfloat Build $name-$version"-linux-mips" linux mips Build $name-$version"-linux-mips64" linux mips64 diff --git a/go.mod b/go.mod index 3b560fa..5187fef 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/oleiade/lane v1.0.1 github.com/olekukonko/tablewriter v0.0.4 github.com/peterh/liner v1.2.1 - github.com/qjfoidnh/Baidu-Login v1.3.9 + github.com/qjfoidnh/Baidu-Login v1.4.0 github.com/qjfoidnh/baidu-tools v0.0.0-20201218182636-dfa5778abeed github.com/tidwall/gjson v1.6.4 github.com/urfave/cli v1.22.5 diff --git a/go.sum b/go.sum index 5601a4a..f14c64b 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -154,13 +155,14 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/qjfoidnh/Baidu-Login v1.3.8 h1:XQXeHkUu3GLrKvCxQ1f8eaZXAWExxe0P39NmzAalHkQ= -github.com/qjfoidnh/Baidu-Login v1.3.8/go.mod h1:wNfM+DJJ7X6OD5vDWfgF5Ye5zshoYYNocw21oSBFg08= github.com/qjfoidnh/Baidu-Login v1.3.9 h1:pwccQOAEpAGd1taAJI7XYUOwqBS6QUjJ9DSjJYVyM/w= github.com/qjfoidnh/Baidu-Login v1.3.9/go.mod h1:oRFCmVYQka0KYwvbf2zS6UeMupgv0w1CSq4Jovhz6qg= +github.com/qjfoidnh/Baidu-Login v1.4.0 h1:47WKkWgAlDSn5CBbeRfh8FwzFeE6qVnaw7+y2km5h7g= +github.com/qjfoidnh/Baidu-Login v1.4.0/go.mod h1:oRFCmVYQka0KYwvbf2zS6UeMupgv0w1CSq4Jovhz6qg= github.com/qjfoidnh/BaiduPCS-Go v0.0.0-20201218134534-d55d9918bd1b/go.mod h1:00iH1dQEStMeT3t+oeVrIucWcu3fFEaFYyygNxfOEv4= github.com/qjfoidnh/baidu-tools v0.0.0-20201218182636-dfa5778abeed h1:Hv47YId8ZGCvLI7d32USeopK5t5o7Y1PaPwKmVdYi2w= github.com/qjfoidnh/baidu-tools v0.0.0-20201218182636-dfa5778abeed/go.mod h1:TzIKHinLPcQbWxAROpqoSvYxM/kDeswfXJaQ2E1p4zs= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/internal/pcscommand/download.go b/internal/pcscommand/download.go index ccd4e31..38c82c5 100644 --- a/internal/pcscommand/download.go +++ b/internal/pcscommand/download.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "runtime" + "sort" ) type ( @@ -71,7 +72,7 @@ func RunDownload(paths []string, options *DownloadOptions) { cfg := &downloader.Config{ Mode: transfer.RangeGenMode_BlockSize, CacheSize: pcsconfig.Config.CacheSize, - BlockSize: baidupcs.MaxDownloadRangeSize, + BlockSize: baidupcs.InitRangeSize, MaxRate: pcsconfig.Config.MaxDownloadRate, InstanceStateStorageFormat: downloader.InstanceStateStorageFormatProto3, IsTest: options.IsTest, @@ -98,29 +99,24 @@ func RunDownload(paths []string, options *DownloadOptions) { ) // 预测要下载的文件数量 - // TODO: pcscache + file_dir_list := make([]*baidupcs.FileDirectory,0,10) for k := range paths { pcs.FilesDirectoriesRecurseList(paths[k], baidupcs.DefaultOrderOptions, func(depth int, _ string, fd *baidupcs.FileDirectory, pcsError pcserror.Error) bool { if pcsError != nil { pcsCommandVerbose.Warnf("%s\n", pcsError) return true } - + file_dir_list = append(file_dir_list, fd) // 忽略统计文件夹数量 if !fd.Isdir { loadCount++ if loadCount >= options.Load { - return false + loadCount = options.Load } } return true }) - - if loadCount >= options.Load { - break - } } - // 修改Load, 设置MaxParallel if loadCount > 0 { options.Load = loadCount @@ -136,8 +132,12 @@ func RunDownload(paths []string, options *DownloadOptions) { } statistic = &pcsdownload.DownloadStatistic{} ) - // 处理队列 - for k := range paths { + + // 处理队列, 小文件优先下载 + sort.Slice(file_dir_list, func(i, j int) bool { + return file_dir_list[i].Size < file_dir_list[j].Size + }) + for _,v := range file_dir_list { newCfg := *cfg unit := pcsdownload.DownloadTaskUnit{ Cfg: &newCfg, // 复制一份新的cfg @@ -151,19 +151,20 @@ func RunDownload(paths []string, options *DownloadOptions) { IsOverwrite: options.IsOverwrite, NoCheck: options.NoCheck, DownloadMode: options.DownloadMode, - PcsPath: paths[k], + PcsPath: v.Path, + FileInfo: v, } // 设置下载并发数 executor.SetParallel(loadCount) // 设置储存的路径 if options.SaveTo != "" { - unit.SavePath = filepath.Join(options.SaveTo, filepath.Base(paths[k])) + unit.SavePath = filepath.Join(options.SaveTo, filepath.Base(v.Path)) } else { // 使用默认的保存路径 - unit.SavePath = GetActiveUser().GetSavePath(paths[k]) + unit.SavePath = GetActiveUser().GetSavePath(v.Path) } info := executor.Append(&unit, options.MaxRetry) - fmt.Printf("[%s] 加入下载队列: %s\n", info.Id(), paths[k]) + fmt.Printf("[%s] 加入下载队列: %s\n", info.Id(), v.Path) } // 开始计时 diff --git a/internal/pcscommand/export.go b/internal/pcscommand/export.go index 93297b1..0dc4876 100644 --- a/internal/pcscommand/export.go +++ b/internal/pcscommand/export.go @@ -30,6 +30,7 @@ type ( MaxRetry int Recursive bool LinkFormat bool + StdOut bool } ) @@ -89,14 +90,17 @@ func RunExport(pcspaths []string, opt *ExportOptions) { fmt.Println(err) return } - saveFile, err := os.OpenFile(opt.SavePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) if err != nil { // 不可写 - fmt.Printf("%s\n", err) - return + if !opt.StdOut { + fmt.Printf("%s\n", err) + return + } } defer saveFile.Close() - fmt.Printf("导出的信息将保存在: %s\n", opt.SavePath) + if !opt.StdOut { + fmt.Printf("导出的信息将保存在: %s\n", opt.SavePath) + } var ( au = GetActiveUser() @@ -159,7 +163,7 @@ func RunExport(pcspaths []string, opt *ExportOptions) { continue } - if len(fds) == 0 { + if len(fds) == 0 && !opt.StdOut { _, writeErr = saveFile.Write(converter.ToBytes(fmt.Sprintf("BaiduPCS-Go mkdir \"%s\"\n", changeRootPath(task.rootPath, task.path, opt.RootPath)))) if writeErr != nil { fmt.Printf("写入文件失败: %s\n", writeErr) @@ -196,13 +200,21 @@ func RunExport(pcspaths []string, opt *ExportOptions) { if opt.LinkFormat { outTemplate = fmt.Sprintf("%s#%s#%d#%s\n", rinfo.ContentMD5, rinfo.SliceMD5, rinfo.ContentLength, path.Base(task.path)) } - _, writeErr = saveFile.Write(converter.ToBytes(outTemplate)) - if writeErr != nil { - fmt.Printf("写入文件失败: %s\n", writeErr) - return // 直接返回 - } + if opt.StdOut { + fmt.Print(outTemplate) + } else { + _, writeErr = saveFile.Write(converter.ToBytes(outTemplate)) + if writeErr != nil { + fmt.Printf("写入文件失败: %s\n", writeErr) + return // 直接返回 + } - fmt.Printf("[%d] - [%s] 导出成功\n", task.ID, task.path) + fmt.Printf("[%d] - [%s] 导出成功\n", task.ID, task.path) + } + } + if opt.StdOut { + os.Remove(opt.SavePath) + fmt.Println("导出完毕") } if failedList.Len() > 0 { diff --git a/internal/pcscommand/login.go b/internal/pcscommand/login.go index 319f293..8865962 100644 --- a/internal/pcscommand/login.go +++ b/internal/pcscommand/login.go @@ -3,13 +3,13 @@ package pcscommand import ( "bytes" "fmt" - "image/png" - "io/ioutil" - baidulogin "github.com/qjfoidnh/Baidu-Login" "github.com/qjfoidnh/BaiduPCS-Go/internal/pcsfunctions/pcscaptcha" "github.com/qjfoidnh/BaiduPCS-Go/pcsliner" "github.com/qjfoidnh/BaiduPCS-Go/requester" + "image/png" + "io/ioutil" + "strings" ) // handleVerifyImg 处理验证码, 下载到本地 @@ -52,7 +52,7 @@ func RunLogin(username, password string) (bduss, ptoken, stoken string, cookies } } - var vcode, vcodestr string + var vcode_raw, vcode, vcodestr string // 移除验证码文件 defer func() { pcscaptcha.RemoveCaptchaPath() @@ -61,7 +61,8 @@ func RunLogin(username, password string) (bduss, ptoken, stoken string, cookies for_1: for i := 0; i < 10; i++ { - lj := bc.BaiduLogin(username, password, vcode, vcodestr) + BEGIN: + lj := bc.BaiduLogin(username, password, vcode_raw, vcodestr) switch lj.ErrInfo.No { case "0": // 登录成功, 退出循环 return lj.Data.BDUSS, lj.Data.PToken, lj.Data.SToken, lj.Data.CookieString, nil @@ -93,20 +94,39 @@ for_1: err = fmt.Errorf("验证方式不合法") return } - - msg := bc.SendCodeToUser(verifyType, lj.Data.Token) // 发送验证码 + msg := "" + if lj.Data.AuthID != "" { + msg = bc.SendCodeToUser(verifyType, lj.Data.VerifyURL, lj.Data.AuthID) // 发送验证码 + } else { + msg = bc.SendCodeToUser2(verifyType, lj.Data.Token) + } fmt.Printf("消息: %s\n\n", msg) - - for et := 0; et < 5; et++ { + if strings.Contains(msg, "系统出错") { + return + } + for et := 0; et < 3; et++ { vcode, err = line.State.Prompt("请输入接收到的验证码 > ") if err != nil { return } - - nlj := bc.VerifyCode(verifyType, lj.Data.Token, vcode, lj.Data.U) + nlj := &baidulogin.LoginJSON{} + if lj.Data.AuthID != "" { + // 此处 BDUSS 等信息尚未获取到, 仅仅完成了邮箱/电话验证 + nlj = bc.VerifyCode(vcode, verifyType, lj.Data.VerifyURL, lj.Data.AuthID, lj.Data.LoginProxy, lj.Data.AuthSID) + } else { + // 此处 BDUSS 等信息已在请求中返回 + nlj = bc.VerifyCode2(verifyType, lj.Data.Token, vcode, lj.Data.U) + } if nlj.ErrInfo.No != "0" { - fmt.Printf("[%d/5] 错误消息: %s\n\n", et+1, nlj.ErrInfo.Msg) + fmt.Printf("[%d/3] 错误消息: %s\n\n", et+1, nlj.ErrInfo.Msg) + if nlj.ErrInfo.No == "-2" { // 需要重发验证码 + return + } continue + } else { + vcode_raw = "" + vcodestr = "" + goto BEGIN } // 登录成功 return nlj.Data.BDUSS, nlj.Data.PToken, nlj.Data.SToken, nlj.Data.CookieString, nil @@ -136,7 +156,7 @@ for_1: fmt.Printf("或者打开以下的网址, 以查看验证码\n") fmt.Printf("%s\n\n", verifyImgURL) - vcode, err = line.State.Prompt("请输入验证码 > ") + vcode_raw, err = line.State.Prompt("请输入验证码 > ") if err != nil { return } diff --git a/internal/pcscommand/transfer.go b/internal/pcscommand/transfer.go index 42f42a6..a3bdbc0 100644 --- a/internal/pcscommand/transfer.go +++ b/internal/pcscommand/transfer.go @@ -99,6 +99,7 @@ func RunRapidTransfer(link string) { } link = string(decodeBytes) } + link = strings.TrimSpace(link) substrs := strings.Split(link, "#") if len(substrs) == 4 { md5 := strings.ToLower(substrs[0]) diff --git a/internal/pcsfunctions/pcsdownload/download_task_unit.go b/internal/pcsfunctions/pcsdownload/download_task_unit.go index e5a2b4a..4d79e8a 100644 --- a/internal/pcsfunctions/pcsdownload/download_task_unit.go +++ b/internal/pcsfunctions/pcsdownload/download_task_unit.go @@ -50,7 +50,7 @@ type ( PcsPath string // 要下载的网盘文件路径 SavePath string // 保存的路径 - fileInfo *baidupcs.FileDirectory // 文件或目录详情 + FileInfo *baidupcs.FileDirectory // 文件或目录详情 } ) @@ -124,6 +124,7 @@ func (dtu *DownloadTaskUnit) download(downloadURL string, client *requester.HTTP der := downloader.NewDownloader(downloadURL, writer, dtu.Cfg) der.SetClient(client) der.SetDURLCheckFunc(BaiduPCSURLCheckFunc) + //der.SetFileContentLength(dtu.FileInfo.Size) der.SetStatusCodeBodyCheckFunc(func(respBody io.Reader) error { // 返回的错误可能是pcs的json // 解析错误 @@ -336,13 +337,13 @@ func (dtu *DownloadTaskUnit) checkFileValid(result *taskframework.TaskUnitRunRes return true } - if dtu.fileInfo.Size >= 128*converter.MB { + if dtu.FileInfo.Size >= 128*converter.MB { // 大文件, 输出一句提示消息 fmt.Printf("[%s] 开始检验文件有效性, 请稍候...\n", dtu.taskInfo.Id()) } // 就在这里处理校验出错 - err := CheckFileValid(dtu.SavePath, dtu.fileInfo) + err := CheckFileValid(dtu.SavePath, dtu.FileInfo) if err != nil { result.ResultMessage = StrDownloadChecksumFailed result.Err = err @@ -407,11 +408,11 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { result = &taskframework.TaskUnitRunResult{} // 获取文件信息 var err error - if dtu.fileInfo == nil || dtu.taskInfo.Retry() > 0 { + if dtu.FileInfo == nil || dtu.taskInfo.Retry() > 0 { // 没有获取文件信息 // 如果是动态添加的下载任务, 是会写入文件信息的 // 如果该任务重试过, 则应该再获取一次文件信息 - dtu.fileInfo, err = dtu.PCS.FilesDirectoriesMeta(dtu.PcsPath) + dtu.FileInfo, err = dtu.PCS.FilesDirectoriesMeta(dtu.PcsPath) if err != nil { // 如果不是未登录或文件不存在, 则不重试 result.ResultMessage = "获取下载路径信息错误" @@ -423,37 +424,37 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { // 输出文件信息 fmt.Print("\n") - fmt.Printf("[%s] ----\n%s\n", dtu.taskInfo.Id(), dtu.fileInfo.String()) + fmt.Printf("[%s] ----\n%s\n", dtu.taskInfo.Id(), dtu.FileInfo.String()) // 如果是一个目录, 将子文件和子目录加入队列 - if dtu.fileInfo.Isdir { + if dtu.FileInfo.Isdir { if !dtu.Cfg.IsTest { // 测试下载, 不建立空目录 os.MkdirAll(dtu.SavePath, 0777) // 首先在本地创建目录, 保证空目录也能被保存 } // 获取该目录下的文件列表 - fileList, err := dtu.PCS.FilesDirectoriesList(dtu.PcsPath, baidupcs.DefaultOrderOptions) - if err != nil { - result.ResultMessage = "获取目录信息错误" - result.Err = err - result.NeedRetry = true - return - } - - for k := range fileList { - // 添加子任务 - subUnit := *dtu - newCfg := *dtu.Cfg - subUnit.Cfg = &newCfg - subUnit.fileInfo = fileList[k] // 保存文件信息 - subUnit.PcsPath = fileList[k].Path - subUnit.SavePath = filepath.Join(dtu.SavePath, fileList[k].Filename) // 保存位置 - - // 加入父队列 - info := dtu.ParentTaskExecutor.Append(&subUnit, dtu.taskInfo.MaxRetry()) - fmt.Printf("[%s] 加入下载队列: %s\n", info.Id(), fileList[k].Path) - } - + //fileList, err := dtu.PCS.FilesDirectoriesList(dtu.PcsPath, baidupcs.DefaultOrderOptions) + //if err != nil { + // result.ResultMessage = "获取目录信息错误" + // result.Err = err + // result.NeedRetry = true + // return + //} + // + //for k := range fileList { + // // 添加子任务 + // subUnit := *dtu + // newCfg := *dtu.Cfg + // subUnit.Cfg = &newCfg + // subUnit.FileInfo = fileList[k] // 保存文件信息 + // subUnit.PcsPath = fileList[k].Path + // subUnit.SavePath = filepath.Join(dtu.SavePath, fileList[k].Filename) // 保存位置 + // + // // 加入父队列 + // info := dtu.ParentTaskExecutor.Append(&subUnit, dtu.taskInfo.MaxRetry()) + // fmt.Printf("[%s] 加入下载队列: %s\n", info.Id(), fileList[k].Path) + //} + // result.Succeed = true // 执行成功 return } @@ -491,9 +492,8 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { // 校验不成功, 返回结果 return result } - // 统计下载 - dtu.DownloadStatistic.AddTotalSize(dtu.fileInfo.Size) + dtu.DownloadStatistic.AddTotalSize(dtu.FileInfo.Size) // 下载成功 result.Succeed = true return diff --git a/internal/pcsfunctions/pcsdownload/pcsdownload.go b/internal/pcsfunctions/pcsdownload/pcsdownload.go index f16ab9b..abb9f0c 100644 --- a/internal/pcsfunctions/pcsdownload/pcsdownload.go +++ b/internal/pcsfunctions/pcsdownload/pcsdownload.go @@ -21,7 +21,7 @@ func IsSkipMd5Checksum(size int64, md5Str string) bool { // BaiduPCSURLCheckFunc downloader 首次检查下载地址要执行的函数 func BaiduPCSURLCheckFunc(client *requester.HTTPClient, durl string) (contentLength int64, resp *http.Response, err error) { resp, err = client.Req(http.MethodGet, durl, nil, map[string]string{ - "Range": "bytes=0-" + strconv.FormatInt(baidupcs.MaxDownloadRangeSize-1, 10), + "Range": "bytes=0-" + strconv.FormatInt(baidupcs.InitRangeSize-1, 10), }) if err != nil { if resp != nil { diff --git a/main.go b/main.go index 6125c08..6d7dba3 100644 --- a/main.go +++ b/main.go @@ -55,7 +55,7 @@ const ( var ( // Version 版本号 - Version = "v3.7.1-devel" + Version = "v3.7.2-devel" historyFilePath = filepath.Join(pcsconfig.GetConfigDir(), "pcs_command_history.txt") reloadFn = func(c *cli.Context) error { @@ -1489,6 +1489,7 @@ func main() { MaxRetry: c.Int("retry"), Recursive: c.Bool("r"), LinkFormat: c.Bool("link"), + StdOut: c.Bool("stdout"), }) return nil }, @@ -1514,6 +1515,10 @@ func main() { Name: "link", Usage: "以通用秒传链接格式导出(将丢失路径信息)", }, + cli.BoolFlag{ + Name: "stdout", + Usage: "导出信息不存文件, 直接打印至标准输出", + }, }, }, { diff --git a/requester/downloader/config.go b/requester/downloader/config.go index ec5b9b0..78d659c 100644 --- a/requester/downloader/config.go +++ b/requester/downloader/config.go @@ -7,11 +7,12 @@ import ( const ( //CacheSize 默认的下载缓存 CacheSize = 8192 + ParallelSize = 5 ) var ( // MinParallelSize 单个线程最小的数据量 - MinParallelSize int64 = 128 * 1024 // 128kb + MinParallelSize int64 = 256 * 1024 // 256kb ) //Config 下载配置 @@ -30,7 +31,7 @@ type Config struct { //NewConfig 返回默认配置 func NewConfig() *Config { return &Config{ - MaxParallel: 5, + MaxParallel: ParallelSize, CacheSize: CacheSize, IsTest: false, } diff --git a/requester/downloader/downloader.go b/requester/downloader/downloader.go index e9845a9..10f5370 100644 --- a/requester/downloader/downloader.go +++ b/requester/downloader/downloader.go @@ -6,6 +6,7 @@ import ( "errors" "github.com/qjfoidnh/BaiduPCS-Go/pcsutil" "github.com/qjfoidnh/BaiduPCS-Go/pcsutil/cachepool" + "github.com/qjfoidnh/BaiduPCS-Go/pcsutil/converter" "github.com/qjfoidnh/BaiduPCS-Go/pcsutil/prealloc" "github.com/qjfoidnh/BaiduPCS-Go/pcsutil/waitgroup" "github.com/qjfoidnh/BaiduPCS-Go/pcsverbose" @@ -23,6 +24,8 @@ const ( DefaultAcceptRanges = "bytes" ) +var BlockSizeList = [6]int64{128*converter.KB, 256*converter.KB, 1024*converter.KB, 2*converter.MB, 4*converter.MB, 999*converter.GB} + type ( // Downloader 下载 Downloader struct { @@ -83,6 +86,14 @@ func (der *Downloader) SetDURLCheckFunc(f DURLCheckFunc) { der.durlCheckFunc = f } +func (der *Downloader) SetFileContentLength(length int64) { + if der.firstInfo == nil { + der.firstInfo = &DownloadFirstInfo{ + ContentLength: length, + AcceptRanges : DefaultAcceptRanges, + } + } +} // SetLoadBalancerCompareFunc 设置负载均衡检测函数 func (der *Downloader) SetLoadBalancerCompareFunc(f LoadBalancerCompareFunc) { der.loadBalancerCompareFunc = f @@ -146,14 +157,23 @@ func (der *Downloader) SelectBlockSizeAndInitRangeGen(single bool, status *trans gen = transfer.NewRangeListGenDefault(status.TotalSize(), 0, 0, parallel) blockSize = gen.LoadBlockSize() case transfer.RangeGenMode_BlockSize: - b2 := status.TotalSize()/int64(parallel) + 1 - if b2 > der.config.BlockSize { // 选小的BlockSize, 以更高并发 - blockSize = der.config.BlockSize + //b2 := status.TotalSize()/int64(parallel) + 1 + //if b2 > der.config.BlockSize { // 选小的BlockSize, 以更高并发 + // blockSize = der.config.BlockSize + //} else { + // blockSize = b2 + //} + totalSize := status.TotalSize() + if totalSize < 2 * converter.MB { + blockSize = BlockSizeList[1] + } else if totalSize < 10 * converter.MB { + blockSize = BlockSizeList[2] + } else if totalSize < 80 * converter.MB { + blockSize = BlockSizeList[3] } else { - blockSize = b2 + blockSize = BlockSizeList[4] } - - gen = transfer.NewRangeListGenBlockSize(status.TotalSize(), 0, blockSize) + gen = transfer.NewRangeListGenBlockSize(totalSize, 0, blockSize) default: initErr = transfer.ErrUnknownRangeGenMode return @@ -271,7 +291,6 @@ func (der *Downloader) checkLoadBalancers() *LoadBalancerResponseList { //Execute 开始任务 func (der *Downloader) Execute() error { der.lazyInit() - var ( resp *http.Response ) diff --git a/requester/downloader/monitor.go b/requester/downloader/monitor.go index 43af06b..bf08ebe 100644 --- a/requester/downloader/monitor.go +++ b/requester/downloader/monitor.go @@ -356,7 +356,9 @@ func (mt *Monitor) Execute(cancelCtx context.Context) { mt.registerAllCompleted() // 注册completed ticker := time.NewTicker(990 * time.Millisecond) + ticker2 := time.NewTicker(99 * time.Millisecond) defer ticker.Stop() + defer ticker2.Stop() //开始监控 for { @@ -385,8 +387,6 @@ func (mt *Monitor) Execute(cancelCtx context.Context) { }) } - // 加入新range - mt.TryAddNewWork() // 不重载worker if !mt.isReloadWorker { @@ -419,6 +419,9 @@ func (mt *Monitor) Execute(cancelCtx context.Context) { mt.ResetWorker(worker) } } // end if 2 + case <-ticker2.C: + // 加入新range + mt.TryAddNewWork() } //end select } //end for } diff --git a/requester/transfer/rangelist.go b/requester/transfer/rangelist.go index 18516bf..3f8a912 100644 --- a/requester/transfer/rangelist.go +++ b/requester/transfer/rangelist.go @@ -112,6 +112,7 @@ func (gen *RangeListGen) RangeCount() (rangeCount int) { switch gen.rangeGenMode { case RangeGenMode_Default: rangeCount = gen.parallel - gen.count + //rangeCount = int(math.Ceil(float64(gen.total) / float64(gen.blockSize))) case RangeGenMode_BlockSize: rangeCount = int((gen.total - gen.begin) / gen.blockSize) if gen.total%gen.blockSize != 0 { @@ -135,6 +136,9 @@ func (gen *RangeListGen) LoadBlockSize() (blockSize int64) { case RangeGenMode_Default: if gen.blockSize <= 0 { gen.blockSize = (gen.total - gen.begin) / int64(gen.parallel) + if gen.blockSize < 256 * converter.KB { + gen.blockSize = 256 * converter.KB + } } blockSize = gen.blockSize case RangeGenMode_BlockSize: @@ -172,6 +176,12 @@ func (gen *RangeListGen) GenRange() (index int, r *Range) { } else { end = gen.begin + gen.blockSize } + + //end = gen.begin + gen.blockSize + if end >= gen.total { + end = gen.total + } + r = &Range{ Begin: gen.begin, End: end, diff --git a/versioninfo.json b/versioninfo.json index 94d52fe..b978186 100644 --- a/versioninfo.json +++ b/versioninfo.json @@ -3,13 +3,13 @@ "FileVersion": { "Major": 3, "Minor": 7, - "Patch": 1, + "Patch": 2, "Build": 0 }, "ProductVersion": { "Major": 3, "Minor": 7, - "Patch": 1, + "Patch": 2, "Build": 0 }, "FileFlagsMask": "3f", @@ -22,14 +22,14 @@ "Comments": "", "CompanyName": "qjfoidnh", "FileDescription": "百度网盘客户端(加强版)", - "FileVersion": "v3.7.1", + "FileVersion": "v3.7.2", "InternalName": "", "LegalCopyright": "© 2016-2020 iikira.", "LegalTrademarks": "", "OriginalFilename": "", "PrivateBuild": "", "ProductName": "BaiduPCS-Go", - "ProductVersion": "v3.7.1", + "ProductVersion": "v3.7.2", "SpecialBuild": "" }, "VarFileInfo": {