fix(stream): http chucked upload issue #1152

TwoOnefour · 2025-08-25T15:54:23Z

流式上传 /api/fs/put 添加X-File-Size请求头与Content-Length 一样用于传递文件大小，如果两个请求头同时存在优先采用Content-Length，X-File-Size可用于无法传递Content-Length请求头时使用，或者仅使用X-File-Size（推荐）
close #1100

TwoOnefour · 2025-08-25T15:54:48Z

问题出在分块上传的时候，有些sdk的实现不会传入Content-Length, 导致f.GetSize()返回0

OpenList/internal/stream/stream.go

Line 209 in b0dbbeb

buf = make([]byte, bufSize)

下面的bufSize也就直接变成0了，而这个commit之前是所有cache都直接存入硬盘中，即utils.CreateTempFile，然后file会变成tmpF，之后GetSize()在 http chunk和非http chunk都会获取到正确的文件大小

internal/stream/stream.go

TwoOnefour · 2025-08-25T16:03:55Z

原来代码有些地方看着有点跟不上@j2rong4cn的思路，不太会改😢

虽然看起来像爆改，但我已经尽量保留原来的思路了

j2rong4cn · 2025-08-25T16:07:51Z

我明天有空再改

j2rong4cn · 2025-08-27T06:45:55Z

问题出在分块上传的时候，有些sdk的实现不会传入Content-Length, 导致f.GetSize()返回0

有个疑问，有些云盘在上传前要上报文件大小的，如果是流式上传用不到缓存，这种情况以前不也是会直接失败吗？

TwoOnefour · 2025-08-27T06:51:18Z

问题出在分块上传的时候，有些sdk的实现不会传入Content-Length, 导致f.GetSize()返回0

有个疑问，有些云盘在上传前要上报文件大小的，如果是流式上传用不到缓存，这种情况以前不也是会直接失败吗？

有点没理解，是指的是原issue复现代码的情况吗

TwoOnefour · 2025-08-27T07:47:46Z

问题出在分块上传的时候，有些sdk的实现不会传入Content-Length, 导致f.GetSize()返回0

有个疑问，有些云盘在上传前要上报文件大小的，如果是流式上传用不到缓存，这种情况以前不也是会直接失败吗？

在b0dbbeb 之前流式上传也是会默认缓存到硬盘上的

你可以git revert 8c244a9

然后断点在

OpenList/server/handles/fsup.go

Line 61 in 8c244a9

sizeStr = "0"

然后再git pull，再走一次上传流程，再断点在

OpenList/internal/stream/stream.go

Line 141 in 4f2de93

return f.cache(f.GetSize())

跟着往下走就大概理解怎么回事了

复现代码可以用这个

点击展开/折叠代码

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

func encodeFilePath(p string) string {
	p = strings.ReplaceAll(p, "\\", "/")
	parts := strings.Split(p, "/")
	for i, s := range parts {
		if s == "" {
			continue
		}
		parts[i] = url.PathEscape(s)
	}
	return strings.Join(parts, "/")
}

func normalizeDestPath(destPath, localFile string) string {
	destPath = strings.ReplaceAll(destPath, "\\", "/")
	if strings.HasSuffix(destPath, "/") {
		return path.Join(destPath, filepath.Base(localFile))
	}
	return destPath
}

func uploadFileChunked(filePath, putURL, token, destPath string, blockSize int, useChunked bool) error {
	f, err := os.Open(filePath)
	if err != nil {
		return printJSON(map[string]any{
			"ok":    false,
			"error": fmt.Sprintf("open file: %v", err),
		})
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return printJSON(map[string]any{
			"ok":    false,
			"error": fmt.Sprintf("stat file: %v", err),
		})
	}
	size := fi.Size()

	destFull := normalizeDestPath(destPath, filePath)
	filePathHeader := encodeFilePath(destFull)

	pr, pw := io.Pipe()
	// 分块写入：边读文件边写到请求体
	go func() {
		defer pw.Close()
		buf := make([]byte, blockSize)
		for {
			n, rerr := f.Read(buf)
			if n > 0 {
				if _, werr := pw.Write(buf[:n]); werr != nil {
					_ = pw.CloseWithError(werr)
					return
				}
			}
			if rerr == io.EOF {
				return
			}
			if rerr != nil {
				_ = pw.CloseWithError(rerr)
				return
			}
		}
	}()

	req, err := http.NewRequest(http.MethodPut, putURL, pr)
	if err != nil {
		return printJSON(map[string]any{
			"ok":    false,
			"error": fmt.Sprintf("new request: %v", err),
		})
	}

	// 头部
	req.Header.Set("Authorization", token)
	req.Header.Set("As-Task", "true")
	req.Header.Set("File-Path", filePathHeader)
	req.Header.Set("Content-Type", "application/octet-stream")

	// 是否使用 HTTP chunked 传输：
	// - useChunked=false（默认）：设置 ContentLength，避免启用 Transfer-Encoding: chunked
	// - useChunked=true：不设置 ContentLength，Go 会自动使用 chunked

	// 模拟不带content-length的情况
	if !useChunked {
		req.ContentLength = size
		req.Header.Set("Content-Length", fmt.Sprint(size))
	}

	//req.ContentLength = size
	//req.Header.Set("Content-Length", fmt.Sprint(size))

	client := &http.Client{Timeout: 2 * time.Minute}
	resp, err := client.Do(req)
	if err != nil {
		return printJSON(map[string]any{
			"ok":    false,
			"error": fmt.Sprintf("request failed: %v", err),
		})
	}
	defer resp.Body.Close()

	body, _ := io.ReadAll(resp.Body)
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		// 成功：JSON 输出
		out := map[string]any{
			"ok":         true,
			"status":     resp.StatusCode,
			"statusText": resp.Status,
		}
		if len(body) > 0 {
			if json.Valid(body) {
				var v any
				_ = json.Unmarshal(body, &v)
				out["body"] = v
			} else {
				out["body"] = string(body)
			}
		}
		return printJSON(out)
	}

	// 失败：也输出 JSON
	out := map[string]any{
		"ok":         false,
		"status":     resp.StatusCode,
		"statusText": resp.Status,
		"body":       string(body),
	}
	return printJSON(out)
}

func printJSON(v any) error {
	enc := json.NewEncoder(os.Stdout)
	enc.SetEscapeHTML(false)
	return enc.Encode(v)
}

func main() {
	filePath := "openlist.exe"
	putURL := "http://127.0.0.1:5244/api/fs/put"
	token := "xxx" // 填入token
	destPath := "/qaq/openlist.exe"

	const blockSize = 50 * 1024 * 1024 // 50MB
	useChunked := true                 // 需要 HTTP chunked 时改为 true

	_ = uploadFileChunked(filePath, putURL, token, destPath, blockSize, useChunked)
}

j2rong4cn · 2025-08-27T09:52:29Z

~~你试一下以前的没问题的版本，分块流式上传到本地存储应该也是0kb的~~
搞错了

TwoOnefour · 2025-08-27T09:53:43Z

你试一下以前的没问题的版本，分块流式上传到本地存储应该也是0kb的

4.1.0吗，我试试

TwoOnefour · 2025-08-27T10:04:29Z

你试一下以前的没问题的版本，分块流式上传到本地存储应该也是0kb的

直接网页端传吗，还是用代码，我复现不了

j2rong4cn · 2025-08-28T04:25:23Z

治标不治本，例如在chunk驱动中请求头As-Task不是true时无法分片

feat(drivers): add a driver that divides large files into multiple chunks #1153

我觉得应该添加 File-Size 请求头

OpenList/server/handles/fsup.go

Lines 59 to 67 in f350ccd

    
           sizeStr := c.GetHeader("Content-Length") 
        
           if sizeStr == "" { 
        
           	sizeStr = "0" 
        
           } 
        
           size, err := strconv.ParseInt(sizeStr, 10, 64) 
        
           if err != nil { 
        
           	common.ErrorResp(c, err, 400) 
        
           	return 
        
           }

TwoOnefour · 2025-08-28T07:49:37Z

治标不治本，例如在chunk驱动中还是0kb

feat(drivers): add a driver that divides large files into multiple chunks #1153

我觉得应该添加 File-Size 请求头

OpenList/server/handles/fsup.go

Lines 59 to 67 in f350ccd

sizeStr := c.GetHeader("Content-Length")

if sizeStr == "" {

sizeStr = "0"

}

size, err := strconv.ParseInt(sizeStr, 10, 64)

if err != nil {

common.ErrorResp(c, err, 400)

return

}

我试了一下chunk分支merge过来流式上传是没问题

j2rong4cn · 2025-08-28T07:56:18Z

我试了一下chunk分支merge过来流式上传是没问题

搞错了，As-Task请求头为false，是不能分片

xrgzs · 2025-08-30T01:54:12Z

#1100 的代码，合并这个 PR 后能上传

~~就是服务端内存占用有点大~~ 正常

完整测试代码：

src\main\java\com\example\FileUploader.java

package com.example;

import cn.hutool.core.io.resource.ResourceUtil;
import cn.hutool.http.Header;
import cn.hutool.http.HttpConfig;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;

import java.io.File;

public class FileUploader {
    
    public static void main(String[] args) {
        try {
            // 配置HTTP参数
            HttpConfig httpConfig = new HttpConfig()
                    // 分块模式，每块50MB
                    .setBlockSize(1024 * 1024 * 50);
            
            // 要上传的文件路径（请根据实际情况修改）
            String filePath = "e:\\xxx\\2GB.bin";
            
            // 检查文件是否存在
            File file = new File(filePath);
            if (!file.exists()) {
                System.err.println("错误：文件不存在 - " + filePath);
                return;
            }
            
            System.out.println("开始上传文件: " + filePath);
            System.out.println("文件大小: " + file.length() + " 字节");
            
            // 发送HTTP PUT请求上传文件
            HttpResponse response = HttpRequest.put("http://10.0.1.111:5244/api/fs/put")
                    .timeout(1000 * 60 * 2)  // 超时时间2分钟
                    .setConfig(httpConfig)
                    .header(Header.AUTHORIZATION, "eyJxxx.xxx.xxx")  // 请替换为实际的授权token
                    .header("As-Task", "true")
                    .header("File-Path", "/123/Temp")  // 服务器上的目标路径
                    .header(Header.CONTENT_LENGTH, String.valueOf(file.length()))
                    .contentType("application/octet-stream")
                    .body(ResourceUtil.getResourceObj(filePath))
                    .execute();
                    
            // 检查响应结果
            if (response.isOk()) {
                System.out.println("文件上传成功！");
                System.out.println("响应状态码: " + response.getStatus());
                System.out.println("响应内容: " + response.body());
            } else {
                System.err.println("文件上传失败！");
                System.err.println("响应状态码: " + response.getStatus());
                System.err.println("错误信息: " + response.body());
            }
            
        } catch (Exception e) {
            System.err.println("上传过程中发生错误: " + e.getMessage());
            e.printStackTrace();
        }
    }
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
         http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    
    <groupId>com.example</groupId>
    <artifactId>file-upload</artifactId>
    <version>1.0.0</version>
    <packaging>jar</packaging>
    
    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
    
    <dependencies>
        <!-- Hutool HTTP工具 -->
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-http</artifactId>
            <version>5.8.22</version>
        </dependency>
        
        <!-- Hutool 核心工具 -->
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-core</artifactId>
            <version>5.8.22</version>
        </dependency>
    </dependencies>
    
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
            
            <!-- 创建可执行JAR -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.2.4</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass>com.example.FileUploader</mainClass>
                                </transformer>
                            </transformers>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

测试命令：

# 生成随机文件
dd if=/dev/urandom of=2GB.bin bs=1G count=2

# Maven 打包
mvn clean package

# 运行
java -jar target/file-upload-1.0.0.jar

j2rong4cn · 2025-08-30T02:31:13Z

试试请求头As-Task设置为false，

xrgzs · 2025-08-30T02:58:45Z

试试请求头As-Task设置为false，

As-Task: false

合入这个 PR

内存占用还是一样，应该正常？

INFO[2025-08-30 10:53:46] max buffer limit: 1629MB
INFO[2025-08-30 10:53:46] mmap threshold: 4MB

data/temp 底下

如果不合，文件大小为 0，123 服务器直接秒传：

确保文件不会秒传，日志如下：

DEBU[2025-08-30 10:50:47]D:/temp/OpenList/server/middlewares/auth.go:73 github.com/OpenListTeam/OpenList/v4/server.Init.Auth.func10() use login token: &{ID:1 Username:admin PwdHash:xxxx PwdTS:0 Salt:xxxx Password: BasePath:/ Role:2 Disabled:false Permission:29183 OtpSecret: SsoID: Authn:[]}
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/path.go:27 github.com/OpenListTeam/OpenList/v4/internal/op.GetStorageAndActualPath() use storage:  /123
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/fs.go:179 github.com/OpenListTeam/OpenList/v4/internal/op.Get() op.Get /Temp
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/fs.go:122 github.com/OpenListTeam/OpenList/v4/internal/op.List() op.List /
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/fs.go:126 github.com/OpenListTeam/OpenList/v4/internal/op.List() use cache when list /
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/fs.go:179 github.com/OpenListTeam/OpenList/v4/internal/op.Get() op.Get /
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/fs.go:179 github.com/OpenListTeam/OpenList/v4/internal/op.Get() op.Get /
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/drivers/123/driver.go:206 github.com/OpenListTeam/OpenList/v4/drivers/123.(*Pan123).Put() upload request res:  {"code":0,"message":"ok","data":{"AccessKeyId":null,"SecretAccessKey":null,"SessionToken":null,"Expiration":null,"Key":"xxx/xxx-0/xxx","Bucket":"","FileId":0,"Reuse":true,"Info":{"FileId":xxx,"FileName":"Temp","Type":0,"Size":0,"ContentType":"0","S3KeyFlag":"xxx-0","CreateAt":"2025-08-30T10:50:47.694252458+08:00","UpdateAt":"2025-08-30T10:50:47.69425258+08:00","Hidden":false,"Etag":"xxx","Status":2,"ParentFileId":0,"Category":0,"PunishFlag":0,"ParentName":"","DownloadUrl":"","AbnormalAlert":1,"Trashed":false,"TrashedExpire":"","TrashedAt":"","StorageNode":"m0","DirectLink":0,"AbsPath":"","PinYin":"temp","PreviewType":0,"BusinessType":0,"Thumbnail":"","Operable":false,"StarredStatus":0,"HighLight":"","LiveSize":0},"UploadId":"","DownloadUrl":"","StorageNode":"m0","EndPoint":"","UploadFileStatus":0,"SliceSize":"16777216","liveUploadInfo":null}}
DEBU[2025-08-30 10:50:47]D:/temp/OpenList/internal/op/fs.go:646 github.com/OpenListTeam/OpenList/v4/internal/op.Put() put file [Temp] done
[GIN] 2025/08/30 - 10:50:50 | 200 |    2.4614985s |      10.0.1.111 | PUT      "/api/fs/put"

j2rong4cn · 2025-08-30T03:10:11Z

那是因为这个PR只是修复了 As-Task为true时无法缓存的问题，，缓存之后是可以获取文件大小，
As-Task不是true时不缓存的话，部分驱动无法正常上传，这是另一个问题
#1152 (comment)

xrgzs · 2025-08-30T03:25:42Z

那是因为这个PR只是修复了 As-Task为true时无法缓存的问题，，缓存之后是可以获取文件大小， As-Task不是true时不缓存的话，部分驱动无法正常上次，这是另一个问题 #1152 (comment)

不合这个 PR，直接改成这样

	sizeStr := c.GetHeader("File-Size")
	if sizeStr == "" {
		sizeStr := c.GetHeader("Content-Length")
		if sizeStr == "" {
			sizeStr = "0"
		}
	}

然后 JAVA 客户端加上

.header("As-Task", "false")
.header("File-Size", String.valueOf(file.length()))

.header("As-Task", "true")
.header("File-Size", String.valueOf(file.length()))

也可以

内存占用还小

…ize management

pkg/buffer/bytes.go

j2rong4cn · 2025-09-01T04:44:59Z

要不要在客户端没有文件大小的请求头时报错呢？

… retrieval

xrgzs

LGTM，记得更新一下文档和 Apifox

* fix(stream): http chucked upload issue * fix(stream): use MmapThreshold * fix(stream): improve caching mechanism and handle size=0 case * fix bug * fix(buffer): optimize ReadAt method for improved performance * fix(upload): handle Content-Length and File-Size headers for better size management * fix(189pc): 移除重复限速 * fix(upload): handle negative file size during streaming uploads * fix(upload): update header key from File-Size to X-File-Size for size retrieval --------- Co-authored-by: j2rong4cn <[email protected]>

fix(stream): http chucked upload issue

440dd0b

xrgzs requested a review from j2rong4cn August 25, 2025 15:54

xrgzs reviewed Aug 25, 2025

View reviewed changes

internal/stream/stream.go Outdated Show resolved Hide resolved

fix(stream): use MmapThreshold

d6dd66c

j2rong4cn added 3 commits August 27, 2025 20:16

fix(stream): improve caching mechanism and handle size=0 case

70b44fa

fix bug

1eec47f

fix(buffer): optimize ReadAt method for improved performance

4ae1464

j2rong4cn force-pushed the main branch from 3ecf682 to 4ae1464 Compare August 28, 2025 03:49

xrgzs added bug go labels Aug 28, 2025

fix(upload): handle Content-Length and File-Size headers for better s…

4ddb579

…ize management

ILoveScratch2 reviewed Aug 30, 2025

View reviewed changes

pkg/buffer/bytes.go Show resolved Hide resolved

fix(189pc): 移除重复限速

3a82d01

j2rong4cn force-pushed the main branch from ab94f97 to 3a82d01 Compare September 1, 2025 04:49

j2rong4cn added 2 commits September 15, 2025 17:20

Merge remote-tracking branch 'upstream/main' into pr/TwoOnefour/1152

a9f5cef

fix(upload): handle negative file size during streaming uploads

0fea41b

j2rong4cn previously approved these changes Sep 15, 2025

View reviewed changes

fix(upload): update header key from File-Size to X-File-Size for size…

cf4aec2

… retrieval

j2rong4cn dismissed their stale review via cf4aec2 September 15, 2025 09:58

xrgzs approved these changes Sep 15, 2025

View reviewed changes

j2rong4cn merged commit cbbb5ad into OpenListTeam:main Sep 15, 2025
12 checks passed

jenken827 mentioned this pull request Sep 29, 2025

fix(webdav): make slice len out of range #1384

Closed

Uh oh!

fix(stream): http chucked upload issue #1152

fix(stream): http chucked upload issue #1152

Uh oh!

Conversation

TwoOnefour commented Aug 25, 2025 • edited by j2rong4cn Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

TwoOnefour commented Aug 25, 2025

Uh oh!

Uh oh!

TwoOnefour commented Aug 25, 2025

Uh oh!

j2rong4cn commented Aug 25, 2025

Uh oh!

j2rong4cn commented Aug 27, 2025

Uh oh!

TwoOnefour commented Aug 27, 2025

Uh oh!

TwoOnefour commented Aug 27, 2025

Uh oh!

j2rong4cn commented Aug 27, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

TwoOnefour commented Aug 27, 2025

Uh oh!

TwoOnefour commented Aug 27, 2025

Uh oh!

j2rong4cn commented Aug 28, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

TwoOnefour commented Aug 28, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

j2rong4cn commented Aug 28, 2025

Uh oh!

xrgzs commented Aug 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

j2rong4cn commented Aug 30, 2025

Uh oh!

xrgzs commented Aug 30, 2025

Uh oh!

j2rong4cn commented Aug 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

xrgzs commented Aug 30, 2025

Uh oh!

Uh oh!

j2rong4cn commented Sep 1, 2025

Uh oh!

xrgzs left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

TwoOnefour commented Aug 25, 2025 •

edited by j2rong4cn

Loading

j2rong4cn commented Aug 27, 2025 •

edited

Loading

j2rong4cn commented Aug 28, 2025 •

edited

Loading

TwoOnefour commented Aug 28, 2025 •

edited

Loading

xrgzs commented Aug 30, 2025 •

edited

Loading

j2rong4cn commented Aug 30, 2025 •

edited

Loading