Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add onUploadProgress option #632

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open
31 changes: 31 additions & 0 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,37 @@ const response = await ky('https://example.com', {
});
```

##### onUploadProgress

Type: `Function`

Upload progress event handler.

The function receives a `progress` object containing the following properties:
- `percent`: A number between 0 and 1 representing the progress percentage.
- `transferredBytes`: The number of bytes transferred so far.
- `totalBytes`: The total number of bytes to be transferred. This is an estimate and may be 0 if the total size cannot be determined.

```js
import ky from 'ky';

const response = await ky.post('https://example.com/api/upload', {
body: largeFile,
onUploadProgress: progress => {
console.log(`${progress.percent * 100}% - ${progress.transferredBytes} of ${progress.totalBytes} bytes`);
}
});
```

Notes:
- This option requires environment support for `ReadableStream`. If streams are not supported, an error will be thrown.
- The `body` of the request must be set for this option to have an effect.
- The total size calculation is an approximation for certain types of bodies (e.g., `FormData`).
- For `FormData` bodies, the size calculation includes an estimation for multipart boundaries and field names.
- If the total size cannot be determined, `totalBytes` will be 0, and `percent` will remain at 0 throughout the upload and will be 1 once upload is finished.

This feature is useful for tracking the progress of large file uploads or when sending substantial amounts of data to a server.
Comment on lines +446 to +453
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The TS doc comments and readme should be in sync.


##### parseJson

Type: `Function`\
Expand Down
117 changes: 115 additions & 2 deletions source/core/Ky.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export class Ky {
throw new Error('Streams are not supported in your environment. `ReadableStream` is missing.');
}

return ky._stream(response.clone(), ky._options.onDownloadProgress);
return ky._streamResponse(response.clone(), ky._options.onDownloadProgress);
}

return response;
Expand Down Expand Up @@ -203,6 +203,24 @@ export class Ky {
// The spread of `this.request` is required as otherwise it misses the `duplex` option for some reason and throws.
this.request = new globalThis.Request(new globalThis.Request(url, {...this.request}), this._options as RequestInit);
}

// Add onUploadProgress handling
if (this._options.onUploadProgress && typeof this._options.onUploadProgress === 'function') {
if (!supportsRequestStreams) {
throw new Error('Request streams are not supported in your environment. The `duplex` option for `Request` is not available.');
}

const originalBody = this.request.body;
if (originalBody) {
const totalBytes = this._getTotalBytes(this._options.body);
this.request
= new globalThis.Request(this._input, {
...this._options,
body: this._streamRequest(
originalBody, totalBytes, this._options.onUploadProgress),
});
}
}
}

protected _calculateRetryDelay(error: unknown) {
Expand Down Expand Up @@ -310,7 +328,7 @@ export class Ky {
}

/* istanbul ignore next */
protected _stream(response: Response, onDownloadProgress: Options['onDownloadProgress']) {
protected _streamResponse(response: Response, onDownloadProgress: Options['onDownloadProgress']) {
const totalBytes = Number(response.headers.get('content-length')) || 0;
let transferredBytes = 0;

Expand Down Expand Up @@ -365,4 +383,99 @@ export class Ky {
},
);
}

// eslint-disable-next-line @typescript-eslint/ban-types
protected _getTotalBytes(body?: globalThis.BodyInit | null): number {
if (!body) {
return 0;
}

if (body instanceof globalThis.FormData) {
// This is an approximation, as FormData size calculation is not straightforward
let size = 0;
// eslint-disable-next-line unicorn/no-array-for-each -- FormData uses forEach method
body.forEach((value: globalThis.FormDataEntryValue, key: string) => {
if (typeof value === 'string') {
size += new globalThis.TextEncoder().encode(value).length;
} else if (typeof value === 'object' && value !== null && 'size' in value) {
// This catches File objects as well, as File extends Blob
size += (value as Blob).size;
}

// Add some bytes for field name and multipart boundaries
size += new TextEncoder().encode(key).length + 40; // 40 is an approximation for multipart overhead
});

return size;
}

if (body instanceof globalThis.Blob) {
return body.size;
}

if (body instanceof globalThis.ArrayBuffer) {
return body.byteLength;
}

if (typeof body === 'string') {
return new globalThis.TextEncoder().encode(body).length;
}

if (body instanceof URLSearchParams) {
return new globalThis.TextEncoder().encode(body.toString()).length;
}

if ('byteLength' in body) {
return (body).byteLength;
}

if (typeof body === 'object' && body !== null) {
try {
const jsonString = JSON.stringify(body);
return new TextEncoder().encode(jsonString).length;
} catch (error) {
console.warn('Unable to stringify object:', error);
return 0;
}
}

return 0; // Default case, unable to determine size
}

protected _streamRequest(
body: BodyInit,
totalBytes: number,
onUploadProgress: (progress: {percent: number; transferredBytes: number; totalBytes: number}) => void,
): globalThis.ReadableStream<Uint8Array> {
let transferredBytes = 0;

return new globalThis.ReadableStream({
async start(controller) {
const reader = body instanceof globalThis.ReadableStream ? body.getReader() : new globalThis.Response(body).body!.getReader();

async function read() {
const {done, value} = await reader.read();
if (done) {
// Ensure 100% progress is reported when the upload is complete
onUploadProgress({percent: 1, transferredBytes, totalBytes: Math.max(totalBytes, transferredBytes)});
controller.close();
return;
}

transferredBytes += value.byteLength as number;
let percent = totalBytes === 0 ? 0 : transferredBytes / totalBytes;
if (totalBytes < transferredBytes || percent === 1) {
percent = 0.99;
}

onUploadProgress({percent: Number(percent.toFixed(2)), transferredBytes, totalBytes});

controller.enqueue(value);
await read();
}

await read();
},
});
}
}
1 change: 1 addition & 0 deletions source/core/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ export const kyOptionKeys: KyOptionsRegistry = {
hooks: true,
throwHttpErrors: true,
onDownloadProgress: true,
onUploadProgress: true,
fetch: true,
};

Expand Down
2 changes: 1 addition & 1 deletion source/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ export type {
NormalizedOptions,
RetryOptions,
SearchParamsOption,
DownloadProgress,
Progress,
} from './types/options.js';

export type {
Expand Down
24 changes: 22 additions & 2 deletions source/types/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export type HttpMethod = 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete';

export type Input = string | URL | Request;

export type DownloadProgress = {
export type Progress = {
percent: number;
transferredBytes: number;

Expand Down Expand Up @@ -186,7 +186,26 @@ export type KyOptions = {
});
```
*/
onDownloadProgress?: (progress: DownloadProgress, chunk: Uint8Array) => void;
onDownloadProgress?: (progress: Progress, chunk: Uint8Array) => void;

/**
Upload progress event handler.

@param progress - Object containing upload progress information.

@example
```
import ky from 'ky';

const response = await ky.post('https://example.com/upload', {
body: new FormData(),
onUploadProgress: progress => {
console.log(`${progress.percent * 100}% - ${progress.transferredBytes} of ${progress.totalBytes} bytes`);
}
});
```
*/
onUploadProgress?: (progress: Progress) => void;

/**
User-defined `fetch` function.
Expand Down Expand Up @@ -287,6 +306,7 @@ export interface NormalizedOptions extends RequestInit { // eslint-disable-line
retry: RetryOptions;
prefixUrl: string;
onDownloadProgress: Options['onDownloadProgress'];
onUploadProgress: Options['onUploadProgress'];
}

export type {RetryOptions} from './retry.js';
4 changes: 2 additions & 2 deletions test/browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import busboy from 'busboy';
import express from 'express';
import {chromium, webkit, type Page} from 'playwright';
import type ky from '../source/index.js'; // eslint-disable-line import/no-duplicates
import type {DownloadProgress} from '../source/index.js'; // eslint-disable-line import/no-duplicates
import type {Progress} from '../source/index.js'; // eslint-disable-line import/no-duplicates
import {createHttpTestServer, type ExtendedHttpTestServer, type HttpServerOptions} from './helpers/create-http-test-server.js';
import {parseRawBody} from './helpers/parse-body.js';
import {browserTest, defaultBrowsersTest} from './helpers/with-page.js';
Expand Down Expand Up @@ -265,7 +265,7 @@ browserTest('onDownloadProgress works', [chromium, webkit], async (t: ExecutionC
await addKyScriptToPage(page);

const result = await page.evaluate(async (url: string) => {
const data: Array<Array<(DownloadProgress | string)>> = [];
const data: Array<Array<(Progress | string)>> = [];
const text = await window
.ky(url, {
onDownloadProgress(progress, chunk) {
Expand Down
7 changes: 7 additions & 0 deletions test/helpers/create-large-file.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Helper function to create a large Blob
export function createLargeBlob(sizeInMB: number): Blob {
const chunkSize = 1024 * 1024; // 1MB
// eslint-disable-next-line unicorn/no-new-array
const chunks = new Array(sizeInMB).fill('x'.repeat(chunkSize));
return new Blob(chunks, {type: 'application/octet-stream'});
}
1 change: 1 addition & 0 deletions test/helpers/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
export * from './create-http-test-server.js';
export * from './parse-body.js';
export * from './with-page.js';
export * from './create-large-file.js';
Loading