Skip to content

Commit 4f8719b

Browse files
YangSen-qnYangSen-qn
authored
optimize download (#401)
1 parent a2a209c commit 4f8719b

26 files changed

+215
-96
lines changed

‎cmd_test/async_fetch_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ package cmd
44

55
import (
66
"github.com/qiniu/qshell/v2/cmd_test/test"
7+
"github.com/qiniu/qshell/v2/iqshell/common/utils"
8+
79
"strings"
810
"testing"
911
)
@@ -15,7 +17,7 @@ func TestAsyncFetch(t *testing.T) {
1517
fetchKeys = append(fetchKeys, "hello10.json")
1618
content := ""
1719
for _, key := range fetchKeys {
18-
content += "http://" + test.BucketDomain + "/" + key + "\t" + "0" + "\t" + "fetch_" + key + "\n"
20+
content += utils.Endpoint(false, test.BucketDomain) + "/" + key + "\t" + "0" + "\t" + "fetch_" + key + "\n"
1921
}
2022
path, err := test.CreateFileWithContent("async_fetch.txt", content)
2123
if err != nil {

‎cmd_test/download_get_test.go

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
package cmd
44

55
import (
6-
"github.com/qiniu/qshell/v2/cmd_test/test"
76
"path/filepath"
87
"strings"
98
"testing"
9+
10+
"github.com/qiniu/qshell/v2/cmd_test/test"
11+
"github.com/qiniu/qshell/v2/iqshell/common/utils"
1012
)
1113

1214
func TestGetImage(t *testing.T) {
@@ -130,7 +132,7 @@ func TestGetWithDomain(t *testing.T) {
130132
}
131133
path := filepath.Join(resultPath, test.Key)
132134
_, errs := test.RunCmdWithError("get", test.Bucket, test.Key,
133-
"--domain", test.BucketDomain,
135+
"--domain", utils.Endpoint(false, test.BucketDomain),
134136
"-o", path)
135137
defer test.RemoveFile(path)
136138

@@ -142,6 +144,27 @@ func TestGetWithDomain(t *testing.T) {
142144
}
143145
}
144146

147+
func TestGetWithErrorDomain(t *testing.T) {
148+
TestCopy(t)
149+
150+
resultPath, err := test.ResultPath()
151+
if err != nil {
152+
t.Fatal("get result path error:", err)
153+
}
154+
path := filepath.Join(resultPath, test.Key)
155+
_, errs := test.RunCmdWithError("get", test.Bucket, test.Key,
156+
"--domain", "error.qiniu.com",
157+
"-o", path)
158+
defer test.RemoveFile(path)
159+
160+
if len(errs) == 0 {
161+
t.Fail()
162+
}
163+
if test.IsFileHasContent(path) {
164+
t.Fatal("TestGetWithErrorDomain get file content should be empty")
165+
}
166+
}
167+
145168
func TestGetNoExistDomain(t *testing.T) {
146169
resultPath, err := test.ResultPath()
147170
if err != nil {

‎cmd_test/download_test.go

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@ package cmd
55
import (
66
"encoding/json"
77
"errors"
8+
"path/filepath"
9+
"strings"
10+
"testing"
11+
812
"github.com/qiniu/qshell/v2/cmd_test/test"
913
"github.com/qiniu/qshell/v2/iqshell/common/config"
1014
"github.com/qiniu/qshell/v2/iqshell/common/data"
15+
"github.com/qiniu/qshell/v2/iqshell/common/utils"
1116
"github.com/qiniu/qshell/v2/iqshell/storage/object/download/operations"
12-
"path/filepath"
13-
"strings"
14-
"testing"
1517
)
1618

1719
func TestDownloadWithKeyFile(t *testing.T) {
@@ -70,7 +72,7 @@ func TestDownloadFromBucket(t *testing.T) {
7072
Bucket: test.Bucket,
7173
Prefix: "hello3,hello5,hello7",
7274
Suffixes: "",
73-
IoHost: test.BucketDomain,
75+
IoHost: utils.Endpoint(false, test.BucketDomain),
7476
Public: true,
7577
CheckSize: true,
7678
Referer: "",
@@ -111,11 +113,11 @@ func TestDownloadWithDomain(t *testing.T) {
111113
Bucket: test.Bucket,
112114
Prefix: "hello3,hello5,hello7",
113115
Suffixes: "",
114-
IoHost: test.BucketDomain,
116+
IoHost: utils.Endpoint(false, test.BucketDomain),
115117
Public: true,
116118
CheckSize: true,
117119
Referer: "",
118-
Domain: test.BucketDomain,
120+
Domain: utils.Endpoint(false, test.BucketDomain),
119121
RecordRoot: "",
120122
},
121123
}
@@ -151,7 +153,7 @@ func TestDownloadNoBucket(t *testing.T) {
151153
Bucket: "",
152154
Prefix: "hello3,hello5,hello7",
153155
Suffixes: "",
154-
IoHost: test.BucketDomain,
156+
IoHost: utils.Endpoint(false, test.BucketDomain),
155157
Public: true,
156158
CheckHash: true,
157159
Referer: "",
@@ -438,7 +440,7 @@ func TestDownload2PublicWithDomain(t *testing.T) {
438440

439441
test.RunCmdWithError("qdownload2",
440442
"--bucket", test.Bucket,
441-
"--domain", test.BucketDomain,
443+
"--domain", utils.Endpoint(false, test.BucketDomain),
442444
"--dest-dir", destDir,
443445
"--key-file", keysFilePath,
444446
"--log-file", logPath,
@@ -462,6 +464,42 @@ func TestDownload2PublicWithDomain(t *testing.T) {
462464
return
463465
}
464466

467+
func TestDownload2PublicWithErrorDomain(t *testing.T) {
468+
test.RemoveRootPath()
469+
470+
keys := test.KeysString + "\nhello_10.json"
471+
keysFilePath, err := test.CreateFileWithContent("download_keys.txt", keys)
472+
if err != nil {
473+
t.Fatal("create cdn config file error:", err)
474+
}
475+
476+
rootPath, err := test.RootPath()
477+
if err != nil {
478+
t.Fatal("get root path error:", err)
479+
}
480+
481+
destDir := filepath.Join(rootPath, "download2")
482+
483+
errString, _ := test.RunCmdWithError("qdownload2",
484+
"--bucket", test.Bucket,
485+
"--domain", "error.qiniu.com",
486+
"--dest-dir", destDir,
487+
"--key-file", keysFilePath,
488+
"--log-level", "debug",
489+
"--public",
490+
"-c", "4",
491+
"-d")
492+
if len(errString) == 0 {
493+
t.Fatal("should be error")
494+
}
495+
496+
if test.FileCountInDir(destDir) > 0 {
497+
t.Fatal("no file should be download")
498+
}
499+
500+
return
501+
}
502+
465503
func TestDownload2Document(t *testing.T) {
466504
test.TestDocument("qdownload2", t)
467505
}

‎cmd_test/fop_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@
33
package cmd
44

55
import (
6-
"github.com/qiniu/qshell/v2/cmd_test/test"
76
"strings"
87
"testing"
8+
9+
"github.com/qiniu/qshell/v2/cmd_test/test"
910
)
1011

1112
const fopObjectKey = "test_mv.mp4"
@@ -19,7 +20,7 @@ func TestFop(t *testing.T) {
1920

2021
result = strings.ReplaceAll(result, "\n", "")
2122
result, errs = test.RunCmdWithError("prefop", result)
22-
if len(errs) > 0 && !strings.Contains(errs, "not_found") {
23+
if len(errs) > 0 && !strings.Contains(errs, "400 Bad Request") {
2324
t.Fail()
2425
}
2526
}

‎cmd_test/match_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"testing"
1010

1111
"github.com/qiniu/qshell/v2/cmd_test/test"
12+
"github.com/qiniu/qshell/v2/iqshell/common/utils"
1213
)
1314

1415
func TestMatch(t *testing.T) {
@@ -100,7 +101,7 @@ func TestBatchMatch(t *testing.T) {
100101
}
101102

102103
objectPath := filepath.Join(resultDir, test.Key)
103-
_, _ = test.RunCmdWithError("get", test.Bucket, test.Key, "--domain", test.BucketDomain,
104+
_, _ = test.RunCmdWithError("get", test.Bucket, test.Key, "--domain", utils.Endpoint(false, test.BucketDomain),
104105
"-o", objectPath, "")
105106
defer test.RemoveFile(objectPath)
106107

‎cmd_test/test/config.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ http://domain/hello7_test.json
6666

6767
Key = "hello1_test.json"
6868
ImageKey = "image.png"
69-
KeyNotExist = "hello_mock.json"
69+
KeyNotExist = "hello_mock_mock.json"
7070
OriginKeys = []string{"hello1.json", "hello2.json", "hello3.json", "hello4.json", "hello5.json", "hello6.json", "hello7.json"}
7171
Keys = []string{"hello1_test.json", "hello2_test.json", "hello3_test.json", "hello4_test.json", "hello5_test.json", "hello6_test.json", "hello7_test.json"}
7272
KeysString = `hello1_test.json

‎docs/get.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ $ qshell get --doc
2525

2626
# 选项
2727
- -o/--outfile：保存在本地的文件路径；不指定，保存在当前文件夹，文件名使用存储空间中的名字【可选】
28-
- --domain：设置下载请求的域名，默认为空表示从存储源站下载，qshell 下载使用域名的优先级：1.配置选项中的下载请求域名(此参数) 2.bucket 绑定的域名(qshell 内部查询，无需配置) 3. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【可选】
28+
- --domain：指定下载请求的域名，当指定了下载域名则仅使用此下载域名进行下载；默认为空，此时 qshell 下载使用域名的优先级：1.bucket 绑定的域名(qshell 内部查询，无需配置) 2. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【可选】
2929
- --get-file-api: 当存储服务端支持 getfile 接口时才有效。【可选】
3030
- --public：空间是否为公开空间；为 `true` 时为公有空间，公有空间下载时不会对下载 URL 进行签名，可以提升 CDN 域名性能，默认为 `false`（私有空间）【可选】
3131
- --check-size: 下载后检测本地文件和服务端文件 size 的一致性。【可选】

‎docs/qdownload.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ $ qshell qdownload --doc
6969
- save_path_handler：指定一个回调函数；在构建文件的保存路径时，优先使用此选项进行构建，如果不配置则使用 $dest_dir + $文件分割符 + $Key 方式进行构建。文档下面有常用场景实例。此函数通过 Go 语言的模板实现，函数验证使用 func 命令，具体语法可参考 func 命令说明，handler 使用方式下方有示例可供参考 【可选】
7070
- check_size：下载后检测本地文件和服务端文件 size 的一致性，默认为 `false`。【可选】
7171
- check_hash：是否验证 hash，如果开启可能会耗费较长时间，默认为 `false` 【可选】
72-
- domain：设置下载请求的域名，默认为空表示从存储源站下载，qshell 下载使用域名的优先级：1.配置文件中的下载请求域名(此参数) 2.bucket 绑定的域名(qshell 内部查询，无需配置) 3. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【该功能默认需要计费，如果希望享受 10G 的免费流量，请自行配置此参数为 CDN 域名，如不配置，需支付源站流量费用，无法减免！！！】 【可选】
72+
- domain：指定下载请求的域名，当指定了下载域名则仅使用此下载域名进行下载；默认为空，此时 qshell 下载使用域名的优先级：1.bucket 绑定的域名(qshell 内部查询，无需配置) 2. 源站域名(qshell 内部查询，无需配置)，当优先级高的域名下载失败后会尝试使用优先级低的域名进行下载。【该功能默认需要计费，如果希望享受 10G 的免费流量，请自行配置此参数为 CDN 域名，如不配置，需支付源站流量费用，无法减免！！！】 【可选】
7373
- referer：如果下载请求域名配置了域名白名单防盗链，需要指定一个允许访问的 referer 地址；默认为空 【可选】
7474
- public：空间是否为公开空间；为 `true` 时为公有空间，公有空间下载时不会对下载 URL 进行签名，可以提升 CDN 域名性能，默认为 `false`（私有空间）【可选】
7575
- enable_slice: 是否开启切片下载，需要注意 `slice_file_size_threshold` 切片阈值选项的配置，只有开启切片下载，并且下载的文件大小大于切片阈值方会启动切片下载。默认不开启。【可选】

‎iqshell/cdn/cdn.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@ package cdn
33
import (
44
"github.com/qiniu/go-sdk/v7/auth/qbox"
55
"github.com/qiniu/go-sdk/v7/cdn"
6-
"github.com/qiniu/qshell/v2/iqshell/common/account"
6+
77
"github.com/qiniu/qshell/v2/iqshell/common/data"
88
"github.com/qiniu/qshell/v2/iqshell/common/log"
9+
"github.com/qiniu/qshell/v2/iqshell/common/workspace"
910
)
1011

1112
const (
@@ -16,7 +17,7 @@ const (
1617

1718
// GetCdnManager 获取CdnManager
1819
func getCdnManager() (cdnManager *cdn.CdnManager, err *data.CodeError) {
19-
acc, gErr := account.GetAccount()
20+
acc, gErr := workspace.GetAccount()
2021
if gErr != nil {
2122
err = data.NewEmptyError().AppendDescF("GetCdnManager error: %v\n", gErr)
2223
return

‎iqshell/common/client/client.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package client
2+
3+
import (
4+
"net"
5+
"net/http"
6+
"time"
7+
8+
"github.com/qiniu/go-sdk/v7/storage"
9+
)
10+
11+
var defaultClient = storage.Client{
12+
Client: &http.Client{
13+
Transport: &http.Transport{
14+
Proxy: http.ProxyFromEnvironment,
15+
DialContext: (&net.Dialer{
16+
Timeout: 10 * time.Second,
17+
KeepAlive: 10 * time.Second,
18+
}).DialContext,
19+
ForceAttemptHTTP2: true,
20+
MaxIdleConns: 4000,
21+
MaxIdleConnsPerHost: 1000,
22+
ResponseHeaderTimeout: 60 * time.Second,
23+
IdleConnTimeout: 10 * time.Second,
24+
TLSHandshakeTimeout: 10 * time.Second,
25+
ExpectContinueTimeout: 1 * time.Second,
26+
},
27+
},
28+
}
29+
30+
func DefaultStorageClient() storage.Client {
31+
return defaultClient
32+
}

0 commit comments

Comments
 (0)