Merge pull request #198 from tonycai653/master

add docs for get and pfop command

Author: longbai

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 2.3.1
+1. batchdelete, batchchgm, batchchtype, batchmove, batchrename命令运行可导出失败，成功文件列表
+2. rput, fput, qupload上传支持设置回调用 
+3. 修复qdownload, qupload 配置文件Windows下UTF-8 BOM解码出错问题
+
 # 2.3.0
 1. 重构qshell代码，方便后续合并qrsctl, qfetch，和添加新的功能
 2. 加入了user命令， 多用户管理，可以在多个用户之间切换

README.md

@@ -99,7 +99,7 @@ $ echo "source <(qshell completion bash)" >> ~/.bashrc
 ```
 
 **Mac上，使用zsh**
-把如下代码加入zsh的启动文件中~/.zshrc
+把如下代码加入zsh的启动文件中~/.zshrc, 然后source ~/.zshrc
 
 ```
 if [ $commands[qshell] ]; then
@@ -127,21 +127,29 @@ fi
 
 默认官方的列举空间的文件使用的是rs.qiniu.com域名，如果因为某种原因，比如私有存储，需要替换使用rs-test.qiniu.com这个域名的话，那么只需要
 在家目录下创建文件名字为.qshell.json的配置文件，文件内容为
+
+```json
 {
     "hosts": {
         "rs_host": "rs-test.qiniu.com"
     }
 }
+```
 
 如果想要更改io host为io-test.qiniu.com的话，只需要继续在上面的hosts中添加，如下：
+
+```json
 {
     "hosts": {
         "rs_host": "rs-test.qiniu.com",
         "io_host": "io-test.qiniu.com"
     }
 }
+```
 
 同理如果全部修改的话
+
+```json
 {
     "hosts": {
         "rs_host": "rs-test.qiniu.com",
@@ -150,6 +158,7 @@ fi
         "rsf_host": ""
     }
 }
+```
 
 
 ## 命令列表
@@ -206,6 +215,8 @@ fi
 |ip|工具|根据淘宝的公开API查询ip地址的地理位置|[文档](docs/ip.md)|
 |unzip|工具|解压zip文件，支持UTF-8编码和GBK编码|[文档](docs/unzip.md)|
 |alilistbucket|第三方|列举阿里OSS空间里面的所有文件|[文档](docs/alilistbucket.md)|
+|get|存储|下载存储空间中的文件|[文档](docs/get.md)|
+|pfop|dora异步处理|提交异步音视频处理请求|[文档](docs/pfop.md)
 
 ## 问题反馈
 

cmd/putfile.go

@@ -19,11 +19,13 @@ var upSettings = storage.Settings{
 }
 
 var (
-	pOverwrite  bool
-	mimeType    string
-	fileType    int
-	workerCount int
-	upHost      string
+	pOverwrite   bool
+	mimeType     string
+	fileType     int
+	workerCount  int
+	upHost       string
+	callbackUrls string
+	callbackHost string
 )
 
 var formPutCmd = &cobra.Command{
@@ -46,12 +48,16 @@ func init() {
 	formPutCmd.Flags().IntVarP(&fileType, "storage", "s", 0, "storage type")
 	formPutCmd.Flags().IntVarP(&workerCount, "worker", "c", 16, "worker count")
 	formPutCmd.Flags().StringVarP(&upHost, "up-host", "u", "", "uphost")
+	formPutCmd.Flags().StringVarP(&callbackUrls, "callback-urls", "l", "", "upload callback urls, separated by comma")
+	formPutCmd.Flags().StringVarP(&callbackHost, "callback-host", "T", "", "upload callback host")
 
 	RePutCmd.Flags().BoolVarP(&pOverwrite, "overwrite", "w", false, "overwrite mode")
 	RePutCmd.Flags().StringVarP(&mimeType, "mimetype", "t", "", "file mime type")
 	RePutCmd.Flags().IntVarP(&fileType, "storage", "s", 0, "storage type")
 	RePutCmd.Flags().IntVarP(&workerCount, "worker", "c", 16, "worker count")
 	RePutCmd.Flags().StringVarP(&upHost, "up-host", "u", "", "uphost")
+	RePutCmd.Flags().StringVarP(&callbackUrls, "callback-urls", "l", "", "upload callback urls, separated by comma")
+	RePutCmd.Flags().StringVarP(&callbackHost, "callback-host", "T", "", "upload callback host")
 
 	viper.BindPFlag("hosts.up_host", formPutCmd.Flags().Lookup("up-host"))
 	viper.BindPFlag("hosts.up_host", RePutCmd.Flags().Lookup("up-host"))
@@ -86,6 +92,17 @@ func FormPut(cmd *cobra.Command, params []string) {
 	policy.FileType = fileType
 	policy.Expires = 7 * 24 * 3600
 	policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`
+	if (callbackUrls == "" && callbackHost != "") || (callbackUrls != "" && callbackHost == "") {
+		fmt.Fprintf(os.Stderr, "callbackUrls and callback must exist at the same time\n")
+		os.Exit(1)
+	}
+	if callbackHost != "" && callbackUrls != "" {
+		callbackUrls = strings.Replace(callbackUrls, ",", ";", -1)
+		policy.CallbackHost = callbackHost
+		policy.CallbackURL = callbackUrls
+		policy.CallbackBody = "key=$(key)&hash=$(etag)"
+		policy.CallbackBodyType = "application/x-www-form-urlencoded"
+	}
 
 	var putExtra storage.PutExtra
 
@@ -193,6 +210,17 @@ func ResumablePut(cmd *cobra.Command, params []string) {
 	policy.FileType = fileType
 	policy.Expires = 7 * 24 * 3600
 	policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`
+	if (callbackUrls == "" && callbackHost != "") || (callbackUrls != "" && callbackHost == "") {
+		fmt.Fprintf(os.Stderr, "callbackUrls and callback must exist at the same time\n")
+		os.Exit(1)
+	}
+	if callbackHost != "" && callbackUrls != "" {
+		callbackUrls = strings.Replace(callbackUrls, ",", ";", -1)
+		policy.CallbackHost = callbackHost
+		policy.CallbackURL = callbackUrls
+		policy.CallbackBody = "key=$(key)&hash=$(etag)"
+		policy.CallbackBodyType = "application/x-www-form-urlencoded"
+	}
 
 	var putExtra storage.RputExtra
 	if upHost != "" {

cmd/qdownload.go

@@ -1,11 +1,13 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"github.com/astaxie/beego/logs"
 	"github.com/qiniu/qshell/iqshell"
 	"github.com/spf13/cobra"
+	"io/ioutil"
 	"os"
 )
 
@@ -38,9 +40,18 @@ func QiniuDownload(cmd *cobra.Command, params []string) {
 		fmt.Fprintf(os.Stderr, "open file: %s: %v\n", configFile, oErr)
 		os.Exit(1)
 	}
-	dErr := json.NewDecoder(cfh).Decode(&downloadConfig)
-	if dErr != nil {
-		fmt.Fprintf(os.Stderr, "decode configFile content: %v\n", dErr)
+	content, rErr := ioutil.ReadAll(cfh)
+	if rErr != nil {
+		fmt.Fprintf(os.Stderr, "read configFile content: %v\n", rErr)
+		os.Exit(1)
+	}
+
+	// remove windows utf-8 BOM
+	content = bytes.TrimPrefix(content, []byte("\xef\xbb\xbf"))
+	uErr := json.Unmarshal(content, &downloadConfig)
+
+	if uErr != nil {
+		fmt.Fprintf(os.Stderr, "decode configFile content: %v\n", uErr)
 		os.Exit(1)
 	}
 

cmd/qupload.go

@@ -1,13 +1,16 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"github.com/astaxie/beego/logs"
+	"github.com/qiniu/api.v7/storage"
 	"github.com/qiniu/qshell/iqshell"
 	"github.com/spf13/cobra"
 	"io/ioutil"
 	"os"
+	"strings"
 )
 
 var qUploadCmd = &cobra.Command{
@@ -30,6 +33,8 @@ func init() {
 	qUploadCmd.Flags().StringVarP(&failureFname, "failure-list", "f", "", "upload failure file list")
 	qUploadCmd.Flags().StringVarP(&overwriteFname, "overwrite-list", "w", "", "upload success (overwrite) file list")
 	qUploadCmd.Flags().Int64VarP(&upthreadCount, "worker", "c", 1, "worker count")
+	qUploadCmd.Flags().StringVarP(&callbackUrls, "callback-urls", "l", "", "upload callback urls, separated by comma")
+	qUploadCmd.Flags().StringVarP(&callbackHost, "callback-host", "T", "", "upload callback host")
 	RootCmd.AddCommand(qUploadCmd)
 }
 
@@ -51,6 +56,8 @@ func parseUploadConfigFile(uploadConfigFile string, uploadConfig *iqshell.Upload
 		err = fmt.Errorf("Read upload config file `%s`: %v\n", uploadConfigFile, rErr)
 		return
 	}
+	//remove UTF-8 BOM
+	configData = bytes.TrimPrefix(configData, []byte("\xef\xbb\xbf"))
 	uErr := json.Unmarshal(configData, uploadConfig)
 	if uErr != nil {
 		err = fmt.Errorf("Parse upload config file `%s`: %v\n", uploadConfigFile, uErr)
@@ -84,6 +91,24 @@ func QiniuUpload(cmd *cobra.Command, params []string) {
 		logs.Error("Upload src dir should be a directory")
 		os.Exit(iqshell.STATUS_HALT)
 	}
+	policy := storage.PutPolicy{}
+
+	if (callbackUrls == "" && callbackHost != "") || (callbackUrls != "" && callbackHost == "") {
+		fmt.Fprintf(os.Stderr, "callbackUrls and callback must exist at the same time\n")
+		os.Exit(1)
+	}
+	if (uploadConfig.CallbackUrls == "" && uploadConfig.CallbackHost != "") || (uploadConfig.CallbackUrls != "" && uploadConfig.CallbackHost == "") {
+		fmt.Fprintf(os.Stderr, "callbackUrls and callback must exist at the same time\n")
+		os.Exit(1)
+	}
+	if (callbackHost != "" && callbackUrls != "") || (uploadConfig.CallbackHost != "" && uploadConfig.CallbackUrls != "") {
+		callbackUrls = strings.Replace(callbackUrls, ",", ";", -1)
+		policy.CallbackHost = callbackHost
+		policy.CallbackURL = callbackUrls
+		policy.CallbackBody = "key=$(key)&hash=$(etag)"
+		policy.CallbackBodyType = "application/x-www-form-urlencoded"
+	}
+	uploadConfig.PutPolicy = policy
 
 	//upload
 	if upthreadCount < iqshell.MIN_UPLOAD_THREAD_COUNT || upthreadCount > iqshell.MAX_UPLOAD_THREAD_COUNT {

cmd/qupload2.go

@@ -3,9 +3,11 @@ package cmd
 import (
 	"fmt"
 	"github.com/astaxie/beego/logs"
+	"github.com/qiniu/api.v7/storage"
 	"github.com/qiniu/qshell/iqshell"
 	"github.com/spf13/cobra"
 	"os"
+	"strings"
 )
 
 var qUpload2Cmd = &cobra.Command{
@@ -48,6 +50,8 @@ func init() {
 	qUpload2Cmd.Flags().StringVar(&successFname, "success-list", "", "upload success file list")
 	qUpload2Cmd.Flags().StringVar(&failureFname, "failure-list", "", "upload failure file list")
 	qUpload2Cmd.Flags().StringVar(&overwriteFname, "overwrite-list", "", "upload success (overwrite) file list")
+	qUpload2Cmd.Flags().StringVarP(&callbackUrls, "callback-urls", "l", "", "upload callback urls, separated by comma")
+	qUpload2Cmd.Flags().StringVarP(&callbackHost, "callback-host", "T", "", "upload callback host")
 
 	RootCmd.AddCommand(qUpload2Cmd)
 }
@@ -69,6 +73,20 @@ func QiniuUpload2(cmd *cobra.Command, params []string) {
 		logs.Error("Upload config `SrcDir` not exist error,", err)
 		os.Exit(iqshell.STATUS_HALT)
 	}
+	policy := storage.PutPolicy{}
+
+	if (callbackUrls == "" && callbackHost != "") || (callbackUrls != "" && callbackHost == "") {
+		fmt.Fprintf(os.Stderr, "callbackUrls and callback must exist at the same time\n")
+		os.Exit(1)
+	}
+	if callbackHost != "" && callbackUrls != "" {
+		callbackUrls = strings.Replace(callbackUrls, ",", ";", -1)
+		policy.CallbackHost = callbackHost
+		policy.CallbackURL = callbackUrls
+		policy.CallbackBody = "key=$(key)&hash=$(etag)"
+		policy.CallbackBodyType = "application/x-www-form-urlencoded"
+	}
+	uploadConfig.PutPolicy = policy
 
 	if up2threadCount < iqshell.MIN_UPLOAD_THREAD_COUNT || up2threadCount > iqshell.MAX_UPLOAD_THREAD_COUNT {
 		fmt.Printf("Tip: you can set <ThreadCount> value between %d and %d to improve speed\n",

cmd/rs.go

@@ -429,8 +429,13 @@ func M3u8Delete(cmd *cobra.Command, params []string) {
 		fmt.Fprintln(os.Stderr, "no m3u8 slices found")
 		os.Exit(iqshell.STATUS_ERROR)
 	}
+	fileExporter, nErr := iqshell.NewFileExporter("", "", "")
+	if nErr != nil {
+		fmt.Fprintf(os.Stderr, "create FileExporter: %v\n", nErr)
+		os.Exit(1)
+	}
 	if entryCnt <= BATCH_ALLOW_MAX {
-		batchDelete(m3u8FileList, bm)
+		batchDelete(m3u8FileList, bm, fileExporter)
 	} else {
 		batchCnt := entryCnt / BATCH_ALLOW_MAX
 		for i := 0; i < batchCnt; i++ {
@@ -439,7 +444,7 @@ func M3u8Delete(cmd *cobra.Command, params []string) {
 				end = entryCnt
 			}
 			entriesToDelete := m3u8FileList[i*BATCH_ALLOW_MAX : end]
-			batchDelete(entriesToDelete, bm)
+			batchDelete(entriesToDelete, bm, fileExporter)
 		}
 	}
 }