Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit dfc43d4

Browse files
authored
fix: replace slice with subarray for increased performance (#4210)
In several places we call `.slice` as a way to transform `BufferList`s to `Uint8Array`s. Due to refactors in some places we are now calling `.slice` on `Uint8Array`s which is a memory-copy operation. In other places `Uint8ArrayList`s are now returned instead of `BufferList`s on which `.slice` is also a memory-copy operation. Swap `.slice` for `.subarray` which is no-copy for `Uint8Array`s and can be no-copy for `Uint8ArrayList`s too, where there is only a single backing buffer. In places where we need to transform multiple `Uint8ArrayList`s to multiple `Uint8Array`s, yield the iterators of the `Uint8ArrayList`s as this is also a no-copy operation.
1 parent acbc1c6 commit dfc43d4

File tree

11 files changed

+43
-36
lines changed

11 files changed

+43
-36
lines changed

packages/ipfs-cli/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@
8585
"ipfs-http-client": "^58.0.1",
8686
"ipfs-utils": "^9.0.6",
8787
"it-concat": "^2.0.0",
88-
"it-map": "^1.0.6",
8988
"it-merge": "^1.0.3",
9089
"it-pipe": "^2.0.3",
9190
"it-split": "^1.0.0",
@@ -110,6 +109,7 @@
110109
"ipfs-repo": "^15.0.3",
111110
"it-all": "^1.0.4",
112111
"it-first": "^1.0.4",
112+
"it-map": "^1.0.6",
113113
"it-to-buffer": "^2.0.0",
114114
"nanoid": "^4.0.0",
115115
"ncp": "^2.0.0",

packages/ipfs-cli/src/commands/cat.js

+8-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import parseDuration from 'parse-duration'
66
* @property {string} Argv.ipfsPath
77
* @property {number} Argv.offset
88
* @property {number} Argv.length
9+
* @property {boolean} Argv.preload
910
* @property {number} Argv.timeout
1011
*/
1112

@@ -26,14 +27,19 @@ const command = {
2627
number: true,
2728
describe: 'Maximum number of bytes to read'
2829
},
30+
preload: {
31+
boolean: true,
32+
default: true,
33+
describe: 'Preload this object when adding'
34+
},
2935
timeout: {
3036
string: true,
3137
coerce: parseDuration
3238
}
3339
},
3440

35-
async handler ({ ctx: { ipfs, print }, ipfsPath, offset, length, timeout }) {
36-
for await (const buf of ipfs.cat(ipfsPath, { offset, length, timeout })) {
41+
async handler ({ ctx: { ipfs, print }, ipfsPath, offset, length, preload, timeout }) {
42+
for await (const buf of ipfs.cat(ipfsPath, { offset, length, preload, timeout })) {
3743
print.write(buf)
3844
}
3945
}

packages/ipfs-cli/src/commands/get.js

-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import {
88
stripControlCharacters
99
} from '../utils.js'
1010
import { extract } from 'it-tar'
11-
import map from 'it-map'
1211

1312
/**
1413
* @typedef {object} Argv
@@ -110,7 +109,6 @@ const command = {
110109
await fs.promises.mkdir(path.dirname(outputPath), { recursive: true })
111110
await pipe(
112111
body,
113-
(source) => map(source, buf => buf.slice()),
114112
toIterable.sink(fs.createWriteStream(outputPath))
115113
)
116114
} else if (header.type === 'directory') {

packages/ipfs-cli/test/cat.spec.js

+15-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
99
const defaultOptions = {
1010
offset: undefined,
1111
length: undefined,
12-
timeout: undefined
12+
timeout: undefined,
13+
preload: true
1314
}
1415

1516
describe('cat', () => {
@@ -81,4 +82,17 @@ describe('cat', () => {
8182
const out = await cli(`cat ${cid} --timeout=1s`, { ipfs, raw: true })
8283
expect(out).to.deep.equal(buf)
8384
})
85+
86+
it('should cat a file without preloading', async () => {
87+
const cid = CID.parse('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB')
88+
const buf = uint8ArrayFromString('hello world')
89+
90+
ipfs.cat.withArgs(cid.toString(), {
91+
...defaultOptions,
92+
preload: false
93+
}).returns([buf])
94+
95+
const out = await cli(`cat ${cid} --preload=false`, { ipfs, raw: true })
96+
expect(out).to.deep.equal(buf)
97+
})
8498
})

packages/ipfs-cli/test/get.spec.js

+1-6
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import sinon from 'sinon'
99
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
1010
import { pack } from 'it-tar'
1111
import { pipe } from 'it-pipe'
12-
import map from 'it-map'
1312
import toBuffer from 'it-to-buffer'
1413
import { clean } from './utils/clean.js'
1514
import Pako from 'pako'
@@ -27,11 +26,7 @@ const defaultOptions = {
2726
async function * tarballed (files) {
2827
yield * pipe(
2928
files,
30-
pack(),
31-
/**
32-
* @param {AsyncIterable<Uint8Array>} source
33-
*/
34-
(source) => map(source, buf => buf.slice())
29+
pack()
3530
)
3631
}
3732

packages/ipfs-core/src/components/files/utils/hamt-constants.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export async function hamtHashFn (buf) {
1111
// Murmur3 outputs 128 bit but, accidentally, IPFS Go's
1212
// implementation only uses the first 64, so we must do the same
1313
// for parity..
14-
.slice(0, 8)
14+
.subarray(0, 8)
1515
// Invert buffer because that's how Go impl does it
1616
.reverse()
1717
}

packages/ipfs-core/src/components/files/write.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ const limitAsyncStreamBytes = (stream, limit) => {
334334
emitted += buf.length
335335

336336
if (emitted > limit) {
337-
yield buf.slice(0, limit - emitted)
337+
yield buf.subarray(0, limit - emitted)
338338

339339
return
340340
}
@@ -353,7 +353,7 @@ const asyncZeroes = (count, chunkSize = MFS_MAX_CHUNK_SIZE) => {
353353

354354
async function * _asyncZeroes () {
355355
while (true) {
356-
yield buf.slice()
356+
yield buf
357357
}
358358
}
359359

packages/ipfs-core/src/components/get.js

+2-11
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { CID } from 'multiformats/cid'
66
import { pack } from 'it-tar'
77
import { pipe } from 'it-pipe'
88
import Pako from 'pako'
9-
import map from 'it-map'
109
import toBuffer from 'it-to-buffer'
1110

1211
// https://www.gnu.org/software/gzip/manual/gzip.html
@@ -57,11 +56,7 @@ export function createGet ({ repo, preload }) {
5756
},
5857
body: file.content()
5958
}],
60-
pack(),
61-
/**
62-
* @param {AsyncIterable<Uint8Array>} source
63-
*/
64-
(source) => map(source, buf => buf.slice())
59+
pack()
6560
)
6661
} else {
6762
args.push(
@@ -126,11 +121,7 @@ export function createGet ({ repo, preload }) {
126121
yield output
127122
}
128123
},
129-
pack(),
130-
/**
131-
* @param {AsyncIterable<Uint8Array>} source
132-
*/
133-
(source) => map(source, buf => buf.slice())
124+
pack()
134125
]
135126

136127
if (options.compress) {

packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -77,15 +77,15 @@ export class WebSocketMessageChannel {
7777
return
7878
}
7979

80-
const header = buf.slice(offset, HEADER_SIZE + offset)
80+
const header = buf.subarray(offset, HEADER_SIZE + offset)
8181
const length = header.readUInt32BE(1)
8282
offset += HEADER_SIZE
8383

8484
if (buf.length < (length + offset)) {
8585
return
8686
}
8787

88-
const message = buf.slice(offset, offset + length)
88+
const message = buf.subarray(offset, offset + length)
8989
const deserialized = this.handler.deserialize(message)
9090
this.source.push(deserialized)
9191
})

packages/ipfs-http-gateway/src/resources/gateway.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -132,11 +132,7 @@ export const Gateway = {
132132
}
133133

134134
const { source, contentType } = await detectContentType(ipfsPath, ipfs.cat(data.cid, catOptions))
135-
const responseStream = toStream.readable((async function * () {
136-
for await (const chunk of source) {
137-
yield chunk.slice() // Convert BufferList to Buffer
138-
}
139-
})())
135+
const responseStream = toStream.readable(source)
140136

141137
const res = h.response(responseStream).code(rangeResponse ? 206 : 200)
142138

packages/ipfs-http-response/src/utils/content-type.js

+10-3
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ export const detectContentType = async (path, source) => {
2828

2929
if (done) {
3030
return {
31-
source: map(stream, (buf) => buf.slice())
31+
source: map(stream, (buf) => buf.subarray())
3232
}
3333
}
3434

35-
fileSignature = await fileTypeFromBuffer(value.slice())
35+
fileSignature = await fileTypeFromBuffer(value.subarray())
3636

3737
output = (async function * () { // eslint-disable-line require-await
3838
yield value
@@ -62,7 +62,14 @@ export const detectContentType = async (path, source) => {
6262
}
6363

6464
if (output != null) {
65-
return { source: map(output, (buf) => buf.slice()), contentType }
65+
return {
66+
source: (async function * () {
67+
for await (const list of output) {
68+
yield * list
69+
}
70+
}()),
71+
contentType
72+
}
6673
}
6774

6875
return { source, contentType }

0 commit comments

Comments
 (0)