First cod3e

Author: sbalko

.gitignore

@@ -1 +1,2 @@
 node_modules/
+dist/

package.json

@@ -7,6 +7,22 @@
   "license": "UNLICENSED",
   "private": true,
   "devDependencies": {
-    "typescript": "^5.4.5"
+    "@types/core-js": "^2.5.8",
+    "@types/jquery": "^3.5.30",
+    "clean-webpack-plugin": "^4.0.0",
+    "copy-webpack-plugin": "^12.0.2",
+    "ts-loader": "^9.5.1",
+    "typescript": "^5.4.5",
+    "webpack": "^5.91.0",
+    "webpack-cli": "^5.1.4"
+  },
+  "scripts": {
+    "clean": "rm -rf dist",
+    "build": "yarn run clean; NODE_ENV=dev webpack",
+    "build:prod": "yarn run clean; NODE_ENV=prod webpack",
+    "watch": "NODE_ENV=dev webpack --watch"
+  },
+  "dependencies": {
+    "nal-extractor": "^1.0.1"
   }
 }

src/decoder/array-utils.ts

@@ -0,0 +1,54 @@
+function buildPattern(search: ArrayLike<unknown>): number[] {
+    const table = new Array<number>(search.length).fill(0);
+    let maxPrefix = 0;
+
+    for (let patternIndex = 1; patternIndex < search.length; ++patternIndex) {
+        while (maxPrefix > 0 && search[patternIndex] !== search[maxPrefix]) {
+            maxPrefix = table[maxPrefix - 1];
+        }
+
+        if (search[maxPrefix] === search[patternIndex]) {
+            ++maxPrefix;
+        }
+
+        table[patternIndex] = maxPrefix;
+    }
+
+    return table;
+}
+
+/**
+ * Generic Knuth-Morris-Pratt algorithm for finding a sequence (string, subarray) in a larger sequence.
+ *
+ * @param text - the larger sequence of elements of type T in which to find the search sequence.
+ * @param search - the shorter search sequence of elements of type T.
+ * @returns an array of numbers, representing the start offsets of all occurrences of `search`
+ * inside `text`.
+ */
+export function searchPattern<T>(text: ArrayLike<T>, search: ArrayLike<T>): number[] {
+    const pattern = buildPattern(search);
+    const matches: number[] = [];
+
+    let textIndex = 0;
+    let patternIndex = 0;
+
+    while (textIndex < text.length) {
+        if (text[textIndex] === search[patternIndex]) {
+            ++textIndex;
+            ++patternIndex;
+        }
+
+        if (patternIndex === search.length) {
+            matches.push(textIndex - patternIndex);
+            patternIndex = pattern[patternIndex - 1];
+        } else if (text[textIndex] !== search[patternIndex]) {
+            if (patternIndex === 0) {
+                ++textIndex;
+            } else {
+                patternIndex = pattern[patternIndex - 1];
+            }
+        }
+    }
+
+    return matches;
+}

src/decoder/bitstream.ts

@@ -0,0 +1,59 @@
+import { EncodedVideoChunkTransformer } from "./chunk-transformer";
+import { AvcNalu, NalUnitType, isTypedNalu } from "./interfaces";
+import { AvcNaluTransformer } from "./nalu-transformer";
+
+import { parseSPS } from 'nal-extractor'
+
+export class AvcBitstream {
+    public readonly config: Promise<VideoDecoderConfig>;
+    public readonly packets: ReadableStreamReader<EncodedVideoChunk>;
+
+    constructor(
+        h264File: Blob,
+        hardwareAcceleration: HardwareAcceleration,
+        frameRate: number
+    ) {
+        const [headerNALUs, remainingNALUs] = h264File.stream().pipeThrough<AvcNalu>(new TransformStream<Uint8Array, AvcNalu>(new AvcNaluTransformer())).tee();
+
+        this.config = AvcBitstream.parseVideoDecoderConfig(headerNALUs, hardwareAcceleration);
+        this.packets = remainingNALUs.pipeThrough<EncodedVideoChunk>(new TransformStream<AvcNalu, EncodedVideoChunk>(new EncodedVideoChunkTransformer(frameRate))).getReader();
+    }
+
+    /**
+     * We read the first SPS NALU and construct the `VideoDecoderConfig` object from it.
+     * 
+     * SPS NALU @see https://www.cardinalpeak.com/blog/the-h-264-sequence-parameter-set
+     * `codec` parameter @see https://www.rfc-editor.org/rfc/rfc6381#section-3
+     * `codecWidth` and `codedHeight` parameters https://stackoverflow.com/questions/12018535/get-the-width-height-of-the-video-from-h-264-nalu
+     */
+    private static async parseVideoDecoderConfig(headerNALUs: ReadableStream<AvcNalu>, hardwareAcceleration: HardwareAcceleration): Promise<VideoDecoderConfig> {
+        const reader = headerNALUs.getReader();
+
+        for (let { value, done } = await reader.read(); value !== undefined && !done; { value, done } = await reader.read()) {
+            if (isTypedNalu(value, NalUnitType.SEQUENCE_PARAMETER_SET)) {
+
+                const { naluBody } = value;
+
+                // The RBSP of a NALU begins after the first byte of its body
+                const { 
+                    profile_idc,
+                    level_idc,
+                    pic_width_in_mbs_minus1,
+                    frame_mbs_only_flag,
+                    pic_height_in_map_units_minus1,
+                    frame_cropping,
+                } = parseSPS(naluBody.subarray(1));
+
+                return {
+                    codec: `avc1.${profile_idc.toString(16).padStart(2, '0')}00${level_idc.toString(16).padStart(2, '0')}}`,
+                    codedWidth: ((pic_width_in_mbs_minus1 + 1) * 16) - (frame_cropping === false ? 0 : (frame_cropping.right_offset * 2 + frame_cropping.left_offset * 2)),
+                    codedHeight:  (2 - (frame_mbs_only_flag ? 1 : 0)) * ((pic_height_in_map_units_minus1 + 1) * 16) - (frame_cropping === false ? 0 : (frame_cropping.top_offset + frame_cropping.bottom_offset) * 2),
+                    hardwareAcceleration,
+                };
+            }
+        }
+
+        throw new Error('Premature end of AVC bitstream before an SPS NALU was found');
+    }
+
+}

src/decoder/chunk-transformer.ts

@@ -0,0 +1,33 @@
+import { AvcNalu, NalUnitType } from "./interfaces";
+
+const ONE_SECOND_IN_MICROS = 1e6;
+
+export class EncodedVideoChunkTransformer implements Transformer<AvcNalu, EncodedVideoChunk> {
+    private seqno = 0;
+    private readonly duration: number;
+
+    /**
+     * 
+     * @param frameRate - the decoding frame rate of this stream in frames per second.
+     */
+    constructor(
+        private readonly frameRate: number
+    ) {
+        this.duration = Math.floor(ONE_SECOND_IN_MICROS / frameRate);
+    }
+
+    private calculateFrameTimestamp(): number {
+        return Math.floor(ONE_SECOND_IN_MICROS * (this.seqno ++) / this.frameRate);
+    }
+
+    transform({ naluType, naluBody }: AvcNalu, controller: TransformStreamDefaultController<EncodedVideoChunk>): void {
+        if (naluType === NalUnitType.CODED_SLICE_IDR || naluType === NalUnitType.CODED_SLICE_NON_IDR) {
+            controller.enqueue(new EncodedVideoChunk({
+                data: naluBody.subarray(1),
+                type: naluType === NalUnitType.CODED_SLICE_IDR ? 'key' : 'delta',
+                timestamp: this.calculateFrameTimestamp(),
+                duration: this.duration,
+            }));
+        }
+    }
+}

src/decoder/interfaces.ts

@@ -0,0 +1,46 @@
+export enum NalUnitType {
+    UNSPECIFIED_1 = 0,
+    CODED_SLICE_NON_IDR = 1,
+    CODED_SLICE_DATA_PARTITION_A = 2,
+    CODED_SLICE_DATA_PARTITION_B = 3,
+    CODED_SLICE_DATA_PARTITION_C = 4,
+    CODED_SLICE_IDR = 5,
+    SUPPLEMENTAL_ENHANCEMENT_INFORMATION = 6,
+    SEQUENCE_PARAMETER_SET = 7,
+    PICTURE_PARAMETER_SET = 8,
+    ACCESS_UNIT_DELIMITER = 9,
+    END_OF_SEQUENCE = 10,
+    END_OF_STREAM = 11,
+    FILLER_DATA = 12,
+    SEQUENCE_PARAMETER_SET_EXTENSION = 13,
+    PREFIX_NAL_UNIT = 14,
+    SUBSET_SEQUENCE_PARAMETER_SET = 15,
+    RESERVED_1 = 16,
+    RESERVED_2 = 17,
+    RESERVED_3 = 18,
+    CODED_SLICE_OF_AUXILIARY_CODED_PICTURE_WITHOUT_PARTITIONING = 19,
+    CODED_SLICE_EXTENSION = 20,
+    CODED_SLICE_EXTENSION_FOR_DEPTH_VIEW_COMPONENTS = 21,
+    RESERVED_4 = 22,
+    RESERVED_5 = 23,
+    UNSPECIFIED_2 = 24,
+    UNSPECIFIED_3 = 25,
+    UNSPECIFIED_4 = 26,
+    UNSPECIFIED_5 = 27,
+    UNSPECIFIED_6 = 28,
+    UNSPECIFIED_7 = 29,
+    UNSPECIFIED_8 = 30,
+    UNSPECIFIED_9 = 31,
+}
+
+export const NALU_PREFIX = [0x00, 0x00, 0x00, 0x01];
+
+export interface AvcNalu<T extends NalUnitType = NalUnitType> {
+    readonly naluType: T;
+    readonly refIdc: number;
+    readonly naluBody: Uint8Array;
+}
+
+export function isTypedNalu<T extends NalUnitType>(nalu: AvcNalu, type: T): nalu is AvcNalu<T> {
+    return nalu.naluType === type;
+}

src/decoder/nalu-transformer.ts

@@ -0,0 +1,66 @@
+import { searchPattern } from "./array-utils";
+import { AvcNalu, NalUnitType, NALU_PREFIX } from "./interfaces";
+
+export class AvcNaluTransformer implements Transformer<Uint8Array, AvcNalu> {
+    private readonly buffered: Uint8Array[] = [];
+
+    private static parseNalu(payload: Uint8Array): {
+        naluType: NalUnitType;
+        refIdc: number;
+    } {
+        return {
+            naluType: payload[0] & 0x1f,
+            refIdc: (payload[0] >> 5) & 0x03,
+        };
+    }
+
+    private async extractNalus(
+        chunk: Uint8Array,
+    ): Promise<Uint8Array[]> {
+        // Find start positions of NALUs in chunk
+        const positions = searchPattern<number>(chunk, NALU_PREFIX);
+
+        // No NALU start markers found in chunk. We append it to the (unfinished) buffered NALU
+        if (positions.length === 0) {
+            this.buffered.push(chunk);
+            return [];
+        }
+
+        // We found at least one NALU start marker. This forms the end of the buffered NALU. The chunk may further contain more complete NALUs, which we
+        // extract.
+        const nalus = [
+            new Uint8Array(await new Blob([...this.buffered, chunk.subarray(0, positions[0])]).arrayBuffer()),
+            ...positions.slice(0, positions.length - 1).map((position, index) => chunk.subarray(position, positions[index + 1])),
+        ];
+
+        // The last NALU start marker demarcates the start of an assumed unfinished NALU.
+        this.buffered.splice(0, this.buffered.length, chunk.subarray(positions[positions.length - 1]));
+
+        return nalus;
+    }
+
+    start(): void {}
+
+    async transform(chunk: Uint8Array, controller: TransformStreamDefaultController<AvcNalu>): Promise<void> {
+        const nalus = await this.extractNalus(chunk);
+
+        nalus.forEach(naluPayload => {
+            const naluBody = naluPayload.subarray(NALU_PREFIX.length);
+            const { naluType, refIdc } = AvcNaluTransformer.parseNalu(naluBody);
+
+            controller.enqueue({ naluType, refIdc, naluBody });
+        });
+    }
+
+    async flush(controller: TransformStreamDefaultController<AvcNalu>): Promise<void> {
+        const naluPayload = new Uint8Array(await new Blob(this.buffered).arrayBuffer());
+
+        const naluBody = naluPayload.subarray(NALU_PREFIX.length);
+        const { naluType, refIdc } = AvcNaluTransformer.parseNalu(naluBody);
+
+        this.buffered.splice(0, this.buffered.length);
+
+        controller.enqueue({ naluType, refIdc, naluBody });
+        controller.terminate();
+    }
+}

src/index.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+
+</head>
+
+<body>
+    <script src="index.js"></script>
+</body>
+</html>

src/index.ts

@@ -0,0 +1,7 @@
+import { AvcBitstream } from "./decoder/bitstream";
+
+console.log('Hello world');
+
+const DEFAULT_FRAMERATE = 30;
+
+new AvcBitstream(new File(['abc'], 'hooha.h264'), 'prefer-hardware', DEFAULT_FRAMERATE);