Stream file in UploadImage to reduce memory usage #1469
Description
Current cloudflare-go version
latest
Description
Just wondering if the idea of streaming multipart file for uploading images has been explored or discussed before.
As far as my understanding goes, currently the UploadImage function expects a *multipart.File and reads it into memory which might become an issue for applications with large amount of traffic.
Instead of loading the file into memory, it can be streamed by streaming the file io.Reader to a io.Writer which can be piped to the request body
Please refer to my code below and sorry for my messy error handling.
Use cases
- Save precious memory by streaming the file.
- Better scalability
Potential cloudflare-go usage
func (c *CloudflareService) UploadImageStreamed(ctx context.Context, file io.Reader, filename string) (*cloudflare.Image, error) {
url := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/images/v1", config.GetCloudflareAccountIdentifier().Identifier)
// Prepare the pipe and multipart writer
pr, pw := io.Pipe()
writer := multipart.NewWriter(pw)
req, err := http.NewRequestWithContext(ctx, "POST", url, pr)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", config.GetCloudflareToken()))
req.Header.Set("Content-Type", writer.FormDataContentType())
// Write to the pipe in a goroutine
errChan := make(chan error, 1)
go func() {
defer pw.Close()
part, err := writer.CreateFormFile("file", filename)
if err != nil {
errChan <- err
return
}
if _, err = io.Copy(part, file); err != nil {
errChan <- err
return
}
errChan <- writer.Close()
}()
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := <-errChan; err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to upload to Cloudflare: %s", resp.Status)
}
var cfRes cloudflare.ImageDetailsResponse
if err = json.NewDecoder(resp.Body).Decode(&cfRes); err != nil {
return nil, err
}
return &cfRes.Result, nil
}References
No response