From 85a8775c1bb06fa94d9eaac248331b346cccb9a9 Mon Sep 17 00:00:00 2001 From: moi15moi Date: Wed, 12 Mar 2025 16:01:37 -0400 Subject: [PATCH 1/6] Send the surface size to any text/subtitle renderer --- .../main/java/androidx/media3/exoplayer/ExoPlayerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index 19ff65ceb1d..e9a6c71a542 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -19,6 +19,7 @@ import static androidx.media3.common.C.TRACK_TYPE_AUDIO; import static androidx.media3.common.C.TRACK_TYPE_CAMERA_MOTION; import static androidx.media3.common.C.TRACK_TYPE_IMAGE; +import static androidx.media3.common.C.TRACK_TYPE_TEXT; import static androidx.media3.common.C.TRACK_TYPE_VIDEO; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; @@ -2812,8 +2813,11 @@ private void maybeNotifySurfaceSizeChanged(int width, int height) { surfaceSize = new Size(width, height); listeners.sendEvent( EVENT_SURFACE_SIZE_CHANGED, listener -> listener.onSurfaceSizeChanged(width, height)); + Size size = new Size(width, height); sendRendererMessage( - TRACK_TYPE_VIDEO, MSG_SET_VIDEO_OUTPUT_RESOLUTION, new Size(width, height)); + TRACK_TYPE_VIDEO, MSG_SET_VIDEO_OUTPUT_RESOLUTION, size); + sendRendererMessage( + TRACK_TYPE_TEXT, MSG_SET_VIDEO_OUTPUT_RESOLUTION, size); } } From 0ccec64a4eb02352dc6dcde441c34a4c0ccc45ca Mon Sep 17 00:00:00 2001 From: moi15moi Date: Wed, 12 Mar 2025 16:13:07 -0400 Subject: [PATCH 2/6] Send the video size to any text/subtitle renderer --- .../java/androidx/media3/exoplayer/ExoPlayerImpl.java | 3 +++ .../main/java/androidx/media3/exoplayer/Renderer.java | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index e9a6c71a542..ea10fc07648 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -25,6 +25,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.castNonNull; +import static androidx.media3.exoplayer.Renderer.MSG_EVENT_VIDEO_SIZE_CHANGED; import static androidx.media3.exoplayer.Renderer.MSG_SET_AUDIO_ATTRIBUTES; import static androidx.media3.exoplayer.Renderer.MSG_SET_AUDIO_SESSION_ID; import static androidx.media3.exoplayer.Renderer.MSG_SET_AUX_EFFECT_INFO; @@ -3089,6 +3090,8 @@ public void onVideoSizeChanged(VideoSize newVideoSize) { videoSize = newVideoSize; listeners.sendEvent( EVENT_VIDEO_SIZE_CHANGED, listener -> listener.onVideoSizeChanged(newVideoSize)); + sendRendererMessage( + TRACK_TYPE_TEXT, MSG_EVENT_VIDEO_SIZE_CHANGED, newVideoSize); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java index 11af79e0b7d..6cb865f832d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java @@ -28,6 +28,7 @@ import androidx.media3.common.Format; import androidx.media3.common.Player; import androidx.media3.common.Timeline; +import androidx.media3.common.VideoSize; import androidx.media3.common.util.Clock; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; @@ -210,7 +211,8 @@ interface WakeupListener { MSG_SET_VIDEO_OUTPUT_RESOLUTION, MSG_SET_IMAGE_OUTPUT, MSG_SET_PRIORITY, - MSG_TRANSFER_RESOURCES + MSG_TRANSFER_RESOURCES, + MSG_EVENT_VIDEO_SIZE_CHANGED }) public @interface MessageType {} @@ -355,6 +357,12 @@ interface WakeupListener { */ int MSG_TRANSFER_RESOURCES = 17; + /** + * The type of a message that can be passed to a renderer to tell it that the video resolution + * changed. The message payload should be a {@link VideoSize}. + */ + int MSG_EVENT_VIDEO_SIZE_CHANGED = 18; + /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * renderers. These custom constants must be greater than or equal to this value. From 1be3fc14a1e64d4c7b44bb03bc06816abb6f732f Mon Sep 17 00:00:00 2001 From: moi15moi Date: Wed, 12 Mar 2025 16:24:54 -0400 Subject: [PATCH 3/6] Send the video colorspace to any text/subtitle renderer --- .../java/androidx/media3/exoplayer/ExoPlayerImpl.java | 3 +++ .../main/java/androidx/media3/exoplayer/Renderer.java | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index ea10fc07648..8f7d9ed3790 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -25,6 +25,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.castNonNull; +import static androidx.media3.exoplayer.Renderer.MSG_EVENT_VIDEO_FORMAT_CHANGED; import static androidx.media3.exoplayer.Renderer.MSG_EVENT_VIDEO_SIZE_CHANGED; import static androidx.media3.exoplayer.Renderer.MSG_SET_AUDIO_ATTRIBUTES; import static androidx.media3.exoplayer.Renderer.MSG_SET_AUDIO_SESSION_ID; @@ -3078,6 +3079,8 @@ public void onVideoInputFormatChanged( Format format, @Nullable DecoderReuseEvaluation decoderReuseEvaluation) { videoFormat = format; analyticsCollector.onVideoInputFormatChanged(format, decoderReuseEvaluation); + sendRendererMessage( + TRACK_TYPE_TEXT, MSG_EVENT_VIDEO_FORMAT_CHANGED, format); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java index 6cb865f832d..feaf28fcdb6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java @@ -212,7 +212,8 @@ interface WakeupListener { MSG_SET_IMAGE_OUTPUT, MSG_SET_PRIORITY, MSG_TRANSFER_RESOURCES, - MSG_EVENT_VIDEO_SIZE_CHANGED + MSG_EVENT_VIDEO_SIZE_CHANGED, + MSG_EVENT_VIDEO_FORMAT_CHANGED }) public @interface MessageType {} @@ -363,6 +364,12 @@ interface WakeupListener { */ int MSG_EVENT_VIDEO_SIZE_CHANGED = 18; + /** + * The type of a message that can be passed to a renderer to tell it that the video format + * changed. The message payload should be a {@link Format}. + */ + int MSG_EVENT_VIDEO_FORMAT_CHANGED = 19; + /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * renderers. These custom constants must be greater than or equal to this value. From 2beb268b9b88369f0402e8554868057478470b15 Mon Sep 17 00:00:00 2001 From: moi15moi <80980684+moi15moi@users.noreply.github.com> Date: Wed, 9 Apr 2025 16:39:36 -0400 Subject: [PATCH 4/6] Parse mkv font attachment The fonts are sent to the renderer via the format metadata. --- .../extractor/mkv/FontMetadataEntry.java | 30 ++++++++ .../extractor/mkv/MatroskaExtractor.java | 76 ++++++++++++++++++- .../text/SubtitleTranscodingTrackOutput.java | 40 +++++++--- 3 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 libraries/extractor/src/main/java/androidx/media3/extractor/mkv/FontMetadataEntry.java diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/FontMetadataEntry.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/FontMetadataEntry.java new file mode 100644 index 00000000000..19f5c4aeb77 --- /dev/null +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/FontMetadataEntry.java @@ -0,0 +1,30 @@ +package androidx.media3.extractor.mkv; + +import android.os.Parcel; +import android.os.Parcelable; +import androidx.media3.common.Metadata; + +/** + * Represents a font attachment in the MKV file. + */ +public class FontMetadataEntry implements Metadata.Entry { + private final String fileName; + private final byte[] fontData; + private final long uid; + + public FontMetadataEntry(String fileName, byte[] fontData, long uid) { + this.fileName = fileName; + this.fontData = fontData; + this.uid = uid; + } + + public String getFileName() { + return fileName; + } + + public byte[] getFontData() { + return fontData; + } + + public long getUid() { return uid; } +} diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/MatroskaExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/MatroskaExtractor.java index d88d1abe922..71a9714d096 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/MatroskaExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mkv/MatroskaExtractor.java @@ -58,6 +58,7 @@ import androidx.media3.extractor.TrueHdSampleRechunker; import androidx.media3.extractor.text.SubtitleParser; import androidx.media3.extractor.text.SubtitleTranscodingExtractorOutput; +import androidx.media3.extractor.text.SubtitleTranscodingTrackOutput; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.lang.annotation.Documented; @@ -70,10 +71,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.UUID; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -272,6 +275,12 @@ public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParser private static final int ID_WHITE_POINT_CHROMATICITY_Y = 0x55D8; private static final int ID_LUMNINANCE_MAX = 0x55D9; private static final int ID_LUMNINANCE_MIN = 0x55DA; + private static final int ID_ATTACHMENTS = 0x1941A469; + private static final int ID_ATTACHED_FILE = 0x61A7; + private static final int ID_FILE_NAME = 0x466E; + private static final int ID_FILE_MEDIA_TYPE = 0x4660; + private static final int ID_FILE_DATA = 0x465C; + private static final int ID_FILE_UID = 0x46AE; /** * BlockAddID value for ITU T.35 metadata in a VP9 track. See also @@ -491,6 +500,13 @@ public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParser private byte sampleSignalByte; private boolean sampleInitializationVectorRead; + // Attachment-related variables + private String attachmentFileName; + private String attachmentMediaType; + private byte[] attachmentFileData; + private long attachmentFileUid; + private List fontAttachments = new ArrayList<>(); + // Extractor outputs. private @MonotonicNonNull ExtractorOutput extractorOutput; @@ -641,6 +657,8 @@ public final int read(ExtractorInput input, PositionHolder seekPosition) throws case ID_PROJECTION: case ID_COLOUR: case ID_MASTERING_METADATA: + case ID_ATTACHMENTS: + case ID_ATTACHED_FILE: return EbmlProcessor.ELEMENT_TYPE_MASTER; case ID_EBML_READ_VERSION: case ID_DOC_TYPE_READ_VERSION: @@ -682,11 +700,14 @@ public final int read(ExtractorInput input, PositionHolder seekPosition) throws case ID_MAX_FALL: case ID_PROJECTION_TYPE: case ID_BLOCK_ADD_ID: + case ID_FILE_UID: return EbmlProcessor.ELEMENT_TYPE_UNSIGNED_INT; case ID_DOC_TYPE: case ID_NAME: case ID_CODEC_ID: case ID_LANGUAGE: + case ID_FILE_NAME: + case ID_FILE_MEDIA_TYPE: return EbmlProcessor.ELEMENT_TYPE_STRING; case ID_SEEK_ID: case ID_BLOCK_ADD_ID_EXTRA_DATA: @@ -697,6 +718,7 @@ public final int read(ExtractorInput input, PositionHolder seekPosition) throws case ID_CODEC_PRIVATE: case ID_PROJECTION_PRIVATE: case ID_BLOCK_ADDITIONAL: + case ID_FILE_DATA: return EbmlProcessor.ELEMENT_TYPE_BINARY; case ID_DURATION: case ID_SAMPLING_FREQUENCY: @@ -726,7 +748,7 @@ public final int read(ExtractorInput input, PositionHolder seekPosition) throws */ @CallSuper protected boolean isLevel1Element(int id) { - return id == ID_SEGMENT_INFO || id == ID_CLUSTER || id == ID_CUES || id == ID_TRACKS; + return id == ID_SEGMENT_INFO || id == ID_CLUSTER || id == ID_CUES || id == ID_TRACKS || id == ID_ATTACHMENTS; } /** @@ -786,6 +808,13 @@ protected void startMasterElement(int id, long contentPosition, long contentSize currentTrack = new Track(); currentTrack.isWebm = isWebm; break; + case ID_ATTACHED_FILE: + Log.d("Sonny - mkvStartElem: ", "We have Attachments!"); + attachmentFileName = null; + attachmentMediaType = null; + attachmentFileData = null; + attachmentFileUid = Format.NO_VALUE; + break; case ID_MASTERING_METADATA: getCurrentTrack(id).hasColorInfo = true; break; @@ -906,6 +935,40 @@ protected void endMasterElement(int id) throws ParserException { } extractorOutput.endTracks(); break; + case ID_ATTACHED_FILE: + Set FONT_MIME_TYPES = new HashSet<>(Set.of( + "application/x-truetype-font", + "application/vnd.ms-opentype", + "application/x-font-otf", + "application/x-font-ttf", + "application/x-font", + "application/font-sfnt", + "font/collection", + "font/otf", + "font/sfnt", + "font/ttf" + )); + Set FONT_FILE_EXTENSION = new HashSet<>(Set.of( + "ttf", + "ttc", + "otf", + "otc" + )); + + String attachmentFileExtension = attachmentFileName.substring(attachmentFileName.lastIndexOf(".") + 1).toLowerCase(); + if (FONT_MIME_TYPES.contains(attachmentMediaType) || (attachmentMediaType.equals("application/octet-stream") && FONT_FILE_EXTENSION.contains(attachmentFileExtension))) { + fontAttachments.add(new FontMetadataEntry(attachmentFileName, attachmentFileData, attachmentFileUid)); + } + break; + case ID_ATTACHMENTS: + for (int i = 0; i < tracks.size(); i++) { + Track attachmentTrack = tracks.valueAt(i); + // Check if this is any kind of subtitle track + if (attachmentTrack.output instanceof SubtitleTranscodingTrackOutput) { + ((SubtitleTranscodingTrackOutput) attachmentTrack.output).setFonts(fontAttachments); + } + } + break; default: break; } @@ -1131,6 +1194,9 @@ protected void integerElement(int id, long value) throws ParserException { case ID_BLOCK_ADD_ID: blockAdditionalId = (int) value; break; + case ID_FILE_UID: + attachmentFileUid = value; + break; default: break; } @@ -1218,6 +1284,10 @@ protected void stringElement(int id, String value) throws ParserException { break; case ID_LANGUAGE: getCurrentTrack(id).language = value; + case ID_FILE_NAME: + attachmentFileName = value; + case ID_FILE_MEDIA_TYPE: + attachmentMediaType = value; break; default: break; @@ -1416,6 +1486,10 @@ protected void binaryElement(int id, int contentSize, ExtractorInput input) thro handleBlockAdditionalData( tracks.get(blockTrackNumber), blockAdditionalId, input, contentSize); break; + case ID_FILE_DATA: + attachmentFileData = new byte[contentSize]; + input.readFully(attachmentFileData, 0, contentSize); + break; default: throw ParserException.createForMalformedContainer( "Unexpected id: " + id, /* cause= */ null); diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java index f132ae7443e..a448e4f249f 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java @@ -25,13 +25,16 @@ import androidx.media3.common.C; import androidx.media3.common.DataReader; import androidx.media3.common.Format; +import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Log; import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.Util; import androidx.media3.extractor.TrackOutput; +import androidx.media3.extractor.mkv.FontMetadataEntry; import java.io.EOFException; import java.io.IOException; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -39,7 +42,7 @@ * MimeTypes#APPLICATION_SUBRIP} to ExoPlayer's internal binary cue representation ({@link * MimeTypes#APPLICATION_MEDIA3_CUES}). */ -/* package */ final class SubtitleTranscodingTrackOutput implements TrackOutput { +public final class SubtitleTranscodingTrackOutput implements TrackOutput { private static final String TAG = "SubtitleTranscodingTO"; @@ -95,20 +98,23 @@ public void format(Format format) { ? subtitleParserFactory.create(format) : null; } + + // Ensure currentFormat always matches what is sent to the delegate, + // as it's used later in setFonts(). if (currentSubtitleParser == null) { - delegate.format(format); + currentFormat = format; } else { - delegate.format( - format - .buildUpon() - .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) - .setCodecs(format.sampleMimeType) - // Reset this value to the default. All non-default timestamp adjustments are done - // below in sampleMetadata() and there are no 'subsamples' after transcoding. - .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) - .setCueReplacementBehavior(subtitleParserFactory.getCueReplacementBehavior(format)) - .build()); + currentFormat = format + .buildUpon() + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCodecs(format.sampleMimeType) + // Reset this value to the default. All non-default timestamp adjustments are done + // below in sampleMetadata() and there are no 'subsamples' after transcoding. + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) + .setCueReplacementBehavior(subtitleParserFactory.getCueReplacementBehavior(format)) + .build(); } + delegate.format(currentFormat); } @Override @@ -180,6 +186,16 @@ public void sampleMetadata( } } + public void setFonts(List fonts) { + checkStateNotNull(currentFormat); // format() must be called before addFont() + + Format.Builder formatBuilder = currentFormat.buildUpon(); + formatBuilder.setMetadata(new Metadata(fonts)); + currentFormat = formatBuilder.build(); + + delegate.format(currentFormat); + } + private void outputSample(CuesWithTiming cuesWithTiming, long timeUs, @C.BufferFlags int flags) { checkStateNotNull(currentFormat); // format() must be called before sampleMetadata() byte[] cuesWithDurationBytes = From 617f6dc752bafecced69ce685fb099d17a92ea58 Mon Sep 17 00:00:00 2001 From: moi15moi <80980684+moi15moi@users.noreply.github.com> Date: Wed, 9 Apr 2025 17:01:00 -0400 Subject: [PATCH 5/6] Add decoder_ass This news native decoder use libass to parse and render the ssa/ass subtitle. Co-authored-by: Vincent Zauhar Co-authored-by: SonnyUplavan Co-authored-by: Raphael Dupuis <78157341+Raphael863@users.noreply.github.com> --- .gitignore | 3 + core_settings.gradle | 2 + demos/main/build.gradle | 1 + libraries/decoder_ass/README.md | 141 ++++++ libraries/decoder_ass/build.gradle | 47 ++ libraries/decoder_ass/proguard-rules.txt | 16 + .../decoder_ass/src/main/AndroidManifest.xml | 17 + .../media3/decoder/ass/AssLibrary.java | 55 +++ .../media3/decoder/ass/AssRenderResult.java | 29 ++ .../media3/decoder/ass/AssRenderer.java | 403 ++++++++++++++++++ .../media3/decoder/ass/LibassJNI.java | 320 ++++++++++++++ .../media3/decoder/ass/package-info.java | 19 + .../decoder_ass/src/main/jni/CMakeLists.txt | 57 +++ .../decoder_ass/src/main/jni/ass_jni.cpp | 287 +++++++++++++ .../decoder_ass/src/main/jni/build_libass.sh | 291 +++++++++++++ .../exoplayer/DefaultRenderersFactory.java | 29 ++ .../text/SubtitleTranscodingTrackOutput.java | 33 +- .../media3/extractor/text/ssa/SsaParser.java | 2 +- 18 files changed, 1747 insertions(+), 5 deletions(-) create mode 100644 libraries/decoder_ass/README.md create mode 100644 libraries/decoder_ass/build.gradle create mode 100644 libraries/decoder_ass/proguard-rules.txt create mode 100644 libraries/decoder_ass/src/main/AndroidManifest.xml create mode 100644 libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssLibrary.java create mode 100644 libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderResult.java create mode 100644 libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java create mode 100644 libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java create mode 100644 libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/package-info.java create mode 100644 libraries/decoder_ass/src/main/jni/CMakeLists.txt create mode 100644 libraries/decoder_ass/src/main/jni/ass_jni.cpp create mode 100755 libraries/decoder_ass/src/main/jni/build_libass.sh diff --git a/.gitignore b/.gitignore index 700b94e9e4e..983058ddc17 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,9 @@ tmp .externalNativeBuild .cxx +# ASS decoder extension +libraries/decoder_ass/src/main/jni/ass + # VP9 decoder extension libraries/decoder_vp9/src/main/jni/libvpx libraries/decoder_vp9/src/main/jni/libvpx_android_configs diff --git a/core_settings.gradle b/core_settings.gradle index 0d3a3a303da..1d0358e7e52 100644 --- a/core_settings.gradle +++ b/core_settings.gradle @@ -71,6 +71,8 @@ project(modulePrefix + 'lib-datasource-okhttp').projectDir = new File(rootDir, ' include modulePrefix + 'lib-decoder' project(modulePrefix + 'lib-decoder').projectDir = new File(rootDir, 'libraries/decoder') +include modulePrefix + 'lib-decoder-ass' +project(modulePrefix + 'lib-decoder-ass').projectDir = new File(rootDir, 'libraries/decoder_ass') include modulePrefix + 'lib-decoder-av1' project(modulePrefix + 'lib-decoder-av1').projectDir = new File(rootDir, 'libraries/decoder_av1') include modulePrefix + 'lib-decoder-ffmpeg' diff --git a/demos/main/build.gradle b/demos/main/build.gradle index 8b8daf420f3..8bda1ede098 100644 --- a/demos/main/build.gradle +++ b/demos/main/build.gradle @@ -82,6 +82,7 @@ dependencies { implementation project(modulePrefix + 'lib-ui') implementation project(modulePrefix + 'lib-datasource-cronet') implementation project(modulePrefix + 'lib-exoplayer-ima') + withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-ass') withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-av1') withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-ffmpeg') withDecoderExtensionsImplementation project(modulePrefix + 'lib-decoder-flac') diff --git a/libraries/decoder_ass/README.md b/libraries/decoder_ass/README.md new file mode 100644 index 00000000000..b849d24039c --- /dev/null +++ b/libraries/decoder_ass/README.md @@ -0,0 +1,141 @@ +# ASS decoder module + +The ASS module provides `AssRenderer`, which uses libass for decoding +and can render ssa/ass subtitle. + +## License note + +Please note that whilst the code in this repository is licensed under +[Apache 2.0][], using this module also requires building and including one or +more external libraries as described below. These are licensed separately. + +[Apache 2.0]: ../../LICENSE + +## Build instructions + +### Prerequisites + +Before running the build script for libass, you will need the following dependencies installed on +your system. You may use whatever package manager to install these: + +* build-essential (or equivalent build tools on non-Debian systems) +* pkg-config +* autoconf +* automake +* libtool +* wget +* gperf +* meson and its dependencies (ninja-build, python3, etc.) + +### Building on Linux or macOS + +1. In a terminal, navigate to this current directory, to make sure the build of libass is done in + the correct directory, as it will create an `ass` directory directly inside `jni`: + ```bash + cd /path/to/media/libraries/decoder_ass/src/main/jni + ``` +2. Locate the path of your Android NDK. You can manually download it from the + official [Android NDK website](https://developer.android.com/ndk/downloads): + ```bash + NDK_PATH="" + ``` +3. Set the host platform: + * For Linux: + ```bash + HOST_PLATFORM="linux-x86_64" + ``` + * For macOS: + ```bash + HOST_PLATFORM="darwin-x86_64" + ``` +4. Set the ABI version for native code (typically equal to your minSdk, and must not exceed it): + ```bash + ANDROID_ABI_VERSION=21 + ``` +5. Execute `build_libass.sh` to build libass for all architectures: + ```bash + ./build_libass.sh "${NDK_PATH}" "${HOST_PLATFORM}" "${ANDROID_ABI_VERSION}" + ``` + +Be aware that you can always edit the script to only build for a specific architecture. The build +process may take some time minutes depending on your system. When complete, the built libraries will +be available in `ass//usr/local/lib/ directories`. + +## Build instructions (Windows) + +We do not provide official support for building this module directly on Windows. However, it is +possible to build using Windows Subsystem for Linux +[WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) with the appropriate tools installed: + +1. Install WSL2 and a Linux distribution (like Ubuntu) from the Microsoft Store +2. Download the [Android NDK](https://developer.android.com/ndk/downloads) for Linux within your + WSL2 environment +3. Follow the Linux build instructions above, setting the NDK path to its location in your WSL2 + filesystem + +For example, if you downloaded and extracted the NDK to your home directory in WSL2, the command +might look like: + +```bash +./build_libass.sh ~/android-ndk-r27c linux-x86_64 21 +``` + +## Note about the build script + +The following script, `build_libass.sh`, as the name suggests, builds libass and its dependencies +for Android platforms. It automates the process of cross-compiling libass and all its dependencies +for Android. This enables Android applications to render complex subtitle formats with advanced +styling and positioning. + +The script builds the following libraries in sequence: + +1. [HarfBuzz (v11.0.0)](https://github.com/harfbuzz/harfbuzz) - An OpenType text shaping engine +2. [FreeType (v2.13.3)](https://freetype.org/) - A font rendering library +3. [FriBidi (v1.0.16)](https://github.com/fribidi/fribidi/) - A library implementing the Unicode + Bidirectional Algorithm +4. [UniBreak (v6.1)](https://github.com/adah1972/libunibreak/) - A line breaking library + implementing the Unicode Line Breaking Algorithm +5. [Expat (v2.7.1)](https://github.com/libexpat/libexpat) - An XML parser library +6. [Fontconfig (v2.16.0)](https://gitlab.freedesktop.org/fontconfig/fontconfig) - A library for font + customization and configuration +7. [libass (v0.17.3)](https://github.com/libass/libass) - The subtitle rendering library + +All libraries are built as static libraries for four Android architectures: + +* x86_64 +* x86 (i686) +* armeabi-v7a (armv7a) +* arm64-v8a (aarch64) + + +## Troubleshooting + +If you encounter issues during the build process: + +1. Ensure all prerequisites are properly installed +2. Verify your NDK path is correct and the NDK version is compatible +3. Check that the HOST_PLATFORM matches your system +4. Make sure ANDROID_ABI_VERSION is appropriate for your project + +Common errors: + +* `meson: command not found` - Install meson using pip: + ```bash + pip3 install meson + ``` +* NDK-related errors - Double-check your NDK path and ensure it's a complete installation +* Library download failures - Verify your internet connection and try again + +For issues with specific dependencies, individual build functions in the script can be modified as +needed. + +## Using the Built Libraries + +After a successful build, the compiled libraries and headers can be used in your Android project. +They will be located in: + +* Libraries: ass//usr/local/lib/ +* Headers: ass//usr/local/include/ + +These files are already be referenced in your project's CMakeLists.txt file to link against the +static libraries. \ No newline at end of file diff --git a/libraries/decoder_ass/build.gradle b/libraries/decoder_ass/build.gradle new file mode 100644 index 00000000000..c2b5250ba6c --- /dev/null +++ b/libraries/decoder_ass/build.gradle @@ -0,0 +1,47 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +apply from: "$gradle.ext.androidxMediaSettingsDir/common_library_config.gradle" + +android { + namespace 'androidx.media3.decoder.ass' + + // TODO(Internal: b/372449691): Remove packagingOptions once AGP is updated + // to version 8.5.1 or higher. + packagingOptions { + jniLibs { + useLegacyPackaging true + } + } +} + +// Configure the native build only if ass is present to avoid gradle sync +// failures if ass hasn't been built according to the README instructions. +if (project.file('src/main/jni/ass').exists()) { + android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt' + // LINT.IfChange + // Should match cmake_minimum_required. + android.externalNativeBuild.cmake.version = '3.21.0+' + // LINT.ThenChange(src/main/jni/CMakeLists.txt) +} + +dependencies { + api project(modulePrefix + 'lib-decoder') + // TODO(b/203752526): Remove this dependency. + implementation project(modulePrefix + 'lib-exoplayer') + implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion + compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion + compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion + testImplementation project(modulePrefix + 'test-utils') + testImplementation 'org.robolectric:robolectric:' + robolectricVersion +} diff --git a/libraries/decoder_ass/proguard-rules.txt b/libraries/decoder_ass/proguard-rules.txt new file mode 100644 index 00000000000..aedb8e120ef --- /dev/null +++ b/libraries/decoder_ass/proguard-rules.txt @@ -0,0 +1,16 @@ +# Proguard rules specific to the libass extension. + +# This prevents the names of native methods from being obfuscated. +-keepclasseswithmembernames class * { + native ; +} + +# Some members of these classes are being accessed from other modules via reflection. Keep them unobfuscated. +-keep class androidx.media3.decoder.ass.AssLibrary { + *; +} + +# Some members of this class are being accessed from native methods. Keep them unobfuscated. +-keep class androidx.media3.decoder.ass.AssRenderResult { + *; +} diff --git a/libraries/decoder_ass/src/main/AndroidManifest.xml b/libraries/decoder_ass/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..2a16d7be08b --- /dev/null +++ b/libraries/decoder_ass/src/main/AndroidManifest.xml @@ -0,0 +1,17 @@ + + + + diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssLibrary.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssLibrary.java new file mode 100644 index 00000000000..92f9f8c6070 --- /dev/null +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssLibrary.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.ass; + +import androidx.media3.common.MediaLibraryInfo; +import androidx.media3.common.util.LibraryLoader; +import androidx.media3.common.util.UnstableApi; + +/** Configures and queries the underlying native library. */ +@UnstableApi +public final class AssLibrary { + + static { + MediaLibraryInfo.registerModule("media3.decoder.ass"); + } + + private static final LibraryLoader LOADER = + new LibraryLoader("assJNI") { + @Override + protected void loadLibrary(String name) { + System.loadLibrary(name); + } + }; + + private AssLibrary() {} + + /** + * Override the names of the Ass native libraries. If an application wishes to call this method, + * it must do so before calling any other method defined by this class, and before instantiating + * any {@link AssRenderer} instances. + * + * @param libraries The names of the Ass native libraries. + */ + public static void setLibraries(String... libraries) { + LOADER.setLibraries(libraries); + } + + /** Returns whether the underlying library is available, loading it if necessary. */ + public static boolean isAvailable() { + return LOADER.isAvailable(); + } +} diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderResult.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderResult.java new file mode 100644 index 00000000000..581a036e02b --- /dev/null +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderResult.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.ass; + +import android.graphics.Bitmap; +import androidx.annotation.Nullable; + +public class AssRenderResult { + @Nullable public final Bitmap bitmap; + public final boolean changedSinceLastCall; + + public AssRenderResult(@Nullable Bitmap bitmap, boolean changedSinceLastCall) { + this.bitmap = bitmap; + this.changedSinceLastCall = changedSinceLastCall; + } +} diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java new file mode 100644 index 00000000000..c3540a5ec13 --- /dev/null +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java @@ -0,0 +1,403 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.ass; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; + +import android.app.Activity; +import android.graphics.Bitmap; +import android.os.Handler; +import android.os.Handler.Callback; +import android.os.Looper; +import android.os.Message; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.Metadata; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.VideoSize; +import androidx.media3.common.text.Cue; +import androidx.media3.common.text.CueGroup; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.exoplayer.BaseRenderer; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.exoplayer.FormatHolder; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; +import androidx.media3.exoplayer.text.TextOutput; +import androidx.media3.extractor.mkv.FontMetadataEntry; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.checkerframework.dataflow.qual.SideEffectFree; + +/** + * A {@link Renderer} for text. + * + *

This implementations decodes sample data to {@link Cue} instances. The actual rendering is + * delegated to a {@link TextOutput}. + */ +@UnstableApi +public final class AssRenderer extends BaseRenderer implements Callback { + + private static final String TAG = "AssRenderer"; + + private static final int MSG_UPDATE_OUTPUT = 1; + + private final DecoderInputBuffer assLineDecoderInputBuffer; + @Nullable private final Handler outputHandler; + private final TextOutput output; + private final FormatHolder formatHolder; + private boolean outputStreamEnded; + @Nullable private Format streamFormat; + private long lastRendererPositionUs; + private long finalStreamEndPositionUs; + @Nullable private IOException streamError; + @Nullable private LibassJNI libassJNI; + + // Track management + private @Nullable String currentTrackId = null; + + private final Set processedFontUids; + private long lastTimestampUs; + // The amount of time to read samples ahead of the current time. + private static final int SAMPLE_WINDOW_DURATION_US = 100_000; + + + /** + * @param output The output. + * @param outputLooper The looper associated with the thread on which the output should be called. + * If the output makes use of standard Android UI components, then this should normally be the + * looper associated with the application's main thread, which can be obtained using {@link + * Activity#getMainLooper()}. Null may be passed if the output should be called + * directly on the player's internal rendering thread. + */ + public AssRenderer( + TextOutput output, + @Nullable Looper outputLooper) { + super(C.TRACK_TYPE_TEXT); + this.output = checkNotNull(output); + this.outputHandler = + outputLooper == null ? null : Util.createHandler(outputLooper, /* callback= */ this); + this.assLineDecoderInputBuffer = + new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + formatHolder = new FormatHolder(); + finalStreamEndPositionUs = C.TIME_UNSET; + lastRendererPositionUs = C.TIME_UNSET; + this.libassJNI = null; + this.processedFontUids = new HashSet<>(); + } + + @Override + public String getName() { + return TAG; + } + + @Override + public @Capabilities int supportsFormat(Format format) { + @Nullable String mimeType = format.sampleMimeType; + if (AssLibrary.isAvailable() && Objects.equals(mimeType, MimeTypes.TEXT_SSA)) { + return RendererCapabilities.create( + format.cryptoType == C.CRYPTO_TYPE_NONE ? C.FORMAT_HANDLED : C.FORMAT_UNSUPPORTED_DRM); + } else { + return RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE); + } + } + + @Override + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs, + MediaSource.MediaPeriodId mediaPeriodId) { + streamFormat = formats[0]; + Metadata metadata = streamFormat.metadata; + maybeInitLibassJNI(); + + // Get a unique key for the subtitle format + String formatId = getTrackId(streamFormat); + currentTrackId = formatId; + libassJNI.createTrack(formatId); + + // Process font metadata + if (metadata != null) { + for (int i = 0; i < metadata.length(); i++) { + Metadata.Entry entry = metadata.get(i); + if (entry instanceof FontMetadataEntry) { + FontMetadataEntry fontEntry = (FontMetadataEntry) entry; + long uid = fontEntry.getUid(); + if (processedFontUids.contains(uid)) { + continue; + } + + String fontFileName = fontEntry.getFileName(); + byte[] fontData = fontEntry.getFontData(); + libassJNI.loadFont(fontFileName, fontData); + processedFontUids.add(uid); + } + } + } + + // Process format initialization data (ASS headers) + assert streamFormat != null; + List assHeaders = streamFormat.initializationData; + if (assHeaders.size() < 2) { + throw new IllegalStateException("Invalid ASS format: missing header data. Found " + + assHeaders.size() + " initialization data entries, expected at least 2."); + } + libassJNI.processCodecPrivate(currentTrackId, assHeaders.get(1)); + } + + + @Override + protected void onPositionReset(long positionUs, boolean joining) { + lastRendererPositionUs = positionUs; + clearOutput(); + outputStreamEnded = false; + finalStreamEndPositionUs = C.TIME_UNSET; + lastTimestampUs = Long.MIN_VALUE; + } + + /** + * Creates an instance of the Libass library if it doesn't already exist + */ + public void maybeInitLibassJNI() { + if (this.libassJNI != null) { + return; + } + + this.libassJNI = new LibassJNI(); + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) { + if (isCurrentStreamFinal() + && finalStreamEndPositionUs != C.TIME_UNSET + && positionUs >= finalStreamEndPositionUs) { + outputStreamEnded = true; + } + + if (outputStreamEnded) { + return; + } + + maybeInitLibassJNI(); + + if (currentTrackId == null) { + Log.w(TAG, "No track ID found. Skipping rendering."); + return; + } + + while (!hasReadStreamToEnd() && lastTimestampUs < positionUs + SAMPLE_WINDOW_DURATION_US) { + assLineDecoderInputBuffer.clear(); + @ReadDataResult int result = readSource(formatHolder, assLineDecoderInputBuffer, /* readFlags= */ 0); + if (result != C.RESULT_BUFFER_READ || assLineDecoderInputBuffer.isEndOfStream()) { + break; + } + + lastTimestampUs = assLineDecoderInputBuffer.timeUs; + boolean isDecodeOnly = lastTimestampUs < getLastResetPositionUs(); + if (isDecodeOnly) { + continue; + } + + assLineDecoderInputBuffer.flip(); + + long subtitleStartTimestamp = getPresentationTimeUs(assLineDecoderInputBuffer.timeUs) / 1000; + ByteBuffer textData = checkNotNull(assLineDecoderInputBuffer.data); + libassJNI.prepareProcessChunk(textData.array(), textData.position(), textData.remaining(), subtitleStartTimestamp, currentTrackId); + } + + // Render current subtitles at the current position + if (currentTrackId != null) { + long renderTimeMs = getPresentationTimeUs(positionUs) / 1000; + AssRenderResult renderResult = libassJNI.renderFrame(currentTrackId, renderTimeMs); + + if (renderResult.changedSinceLastCall) { + if (renderResult.bitmap != null) { + CueGroup cueGroup = bitmapToCueGroup(renderResult.bitmap, positionUs); + updateOutput(cueGroup); + } else { + // No subtitles to show at this time + clearOutput(); + } + } + } + } + + @Override + protected void onDisabled() { + streamFormat = null; + finalStreamEndPositionUs = C.TIME_UNSET; + clearOutput(); + lastRendererPositionUs = C.TIME_UNSET; + + // Release all tracks + if (libassJNI != null && currentTrackId != null) { + libassJNI.releaseTrack(currentTrackId); + currentTrackId = null; + } + } + + @Override + public boolean isEnded() { + return outputStreamEnded; + } + + @Override + public boolean isReady() { + if (streamFormat == null) { + return true; + } + if (streamError == null) { + try { + maybeThrowStreamError(); + } catch (IOException e) { + Log.e(TAG, "Stream error", e); + streamError = e; + return false; + } + } + // Don't block playback whilst subtitles are loading. + // Note: To change this behavior, it will be necessary to consider [Internal: b/12949941]. + return true; + } + + /** + * Updates the output with the given CueGroup. + * This method is called when the subtitles are ready to be displayed. + * @param cueGroup The CueGroup to be displayed. + */ + private void updateOutput(CueGroup cueGroup) { + if (outputHandler != null) { + outputHandler.obtainMessage(MSG_UPDATE_OUTPUT, cueGroup).sendToTarget(); + } else { + invokeUpdateOutputInternal(cueGroup); + } + } + + /** + * Clears the output by sending an empty CueGroup to the output. + * This is used to clear the output when there are no subtitles to display. + */ + private void clearOutput() { + updateOutput(new CueGroup(ImmutableList.of(), getPresentationTimeUs(lastRendererPositionUs))); + } + + @Override + public boolean handleMessage(Message msg) { + if (msg.what == MSG_UPDATE_OUTPUT) { + invokeUpdateOutputInternal((CueGroup) msg.obj); + return true; + } + throw new IllegalStateException(); + } + + @Override + public void handleMessage(@MessageType int messageType, @Nullable Object message) + throws ExoPlaybackException { + maybeInitLibassJNI(); + + switch (messageType) { + case MSG_SET_VIDEO_OUTPUT_RESOLUTION: + Size surfaceSize = (Size) checkNotNull(message, "Surface size message cannot be null"); + libassJNI.setFrameSize(surfaceSize.getWidth(), surfaceSize.getHeight()); + break; + + case MSG_EVENT_VIDEO_SIZE_CHANGED: + VideoSize videoSize = (VideoSize) checkNotNull(message, "Video size message cannot be null"); + libassJNI.setStorageSize(videoSize.width, videoSize.height); + break; + + case MSG_EVENT_VIDEO_FORMAT_CHANGED: + // TODO: Handle this case properly in the alpha blending + Format videoFormat = (Format) checkNotNull(message, "Video format message cannot be null"); + if (videoFormat.colorInfo != null) { + Log.d(TAG, "videoFormat.colorInfo = " + videoFormat.colorInfo); + } else { + Log.d(TAG, "The video format does not have a defined color info."); + } + break; + + default: + super.handleMessage(messageType, message); + } + } + + /** + * We need to call both onCues methods for backward compatibility. + * @param cueGroup The CueGroup to be passed to the output. + */ + @SuppressWarnings("deprecation") + private void invokeUpdateOutputInternal(CueGroup cueGroup) { + output.onCues(cueGroup.cues); + output.onCues(cueGroup); + } + + /** + * Used to calculate the presentation time of the subtitles. + * @param positionUs The current playback position in microseconds. + * @return The presentation time in microseconds. + */ + @SideEffectFree + private long getPresentationTimeUs(long positionUs) { + checkState(positionUs != C.TIME_UNSET); + return positionUs - getStreamOffsetUs(); + } + + /** + * Generate and returns a unique key for a subtitle format to identify tracks, based on format + * attributes that would define a unique subtitle track. + * + * @param format The format to generate a key for. + * @return A unique key for the format. + */ + private String getTrackId(Format format) { + return format.id + "_" + + format.language + "_" + + format.selectionFlags + "_" + + format.roleFlags; + } + + /** + * Converts a bitmap containing rendered subtitles into a CueGroup that can be displayed. + * + * @param bitmap The bitmap containing the rendered subtitles + * @param positionUs The current playback position in microseconds + * @return A CueGroup containing a bitmap cue + */ + private CueGroup bitmapToCueGroup(Bitmap bitmap, long positionUs) { + Cue bitmapCue = new Cue.Builder() + .setBitmap(bitmap) + .setPosition(0.0f) + .setPositionAnchor(Cue.ANCHOR_TYPE_START) + .setLine(0.0f, Cue.LINE_TYPE_FRACTION) + .setLineAnchor(Cue.ANCHOR_TYPE_START) + .setSize(1.0f) + .build(); + + return new CueGroup(ImmutableList.of(bitmapCue), getPresentationTimeUs(positionUs)); + } + +} diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java new file mode 100644 index 00000000000..e7108c83a51 --- /dev/null +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java @@ -0,0 +1,320 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.ass; + +import androidx.annotation.Nullable; +import androidx.media3.common.util.Log; +import androidx.media3.extractor.text.ssa.SsaParser; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +public class LibassJNI { + + private final String TAG = "LibassJNI"; + private final long assLibraryPtr; + private final long assRendererPtr; + private final Map assTrackPtrs = new HashMap<>(); + + @Nullable + private Integer frame_width = null; + @Nullable + private Integer frame_height = null; + + public LibassJNI() { + if (!AssLibrary.isAvailable()) { + throw new RuntimeException("Libass native library is not available"); + } + + assLibraryPtr = assLibraryInit(); + if (assLibraryPtr == 0) { + throw new RuntimeException("Failed to initialize ASS_Library"); + } + + assRendererPtr = assRendererInit(assLibraryPtr); + if (assRendererPtr == 0) { + throw new RuntimeException("Failed to initialize ASS_Renderer"); + } + } + + /** + * Sets the frame size for the ASS_Renderer. + * + * @param width The width of the frame in pixels. + * @param height The height of the frame in pixels. + */ + public void setFrameSize(int width, int height) { + frame_width = width; + frame_height = height; + assSetFrameSize(assRendererPtr, width, height); + } + + /** + * Sets the storage size for the ASS_Renderer. + * + * @param width The width of the storage in pixels. + * @param height The height of the storage in pixels. + */ + public void setStorageSize(int width, int height) { + assSetStorageSize(assRendererPtr, width, height); + } + + /** + * Creates a new ASS_Track instance if it does not already exist for the given format ID. + * + * @param formatId The unique identifier for the format. + * @throws RuntimeException if the ASS_Track creation fails. + */ + public void createTrack(String formatId) { + if (assTrackPtrs.containsKey(formatId)) { + return; + } + + long trackPtr = assNewTrack(assLibraryPtr); + if (trackPtr == 0) { + throw new RuntimeException("Failed to create ASS_Track"); + } + assTrackPtrs.put(formatId, trackPtr); + } + + /** + * Releases a track when it's no longer needed. + * + * @param trackId The ID of the track to release. + */ + public void releaseTrack(String trackId) { + Long trackPtr = assTrackPtrs.remove(trackId); + assFreeTrack(trackPtr); + } + + /** + * Prepares and formats data to then call {@link #assProcessChunk}()}. + * + * @param data The ass subtitle event. + * @param offset The index in {@code data} to start reading from (inclusive). + * @param length The number of bytes to read from {@code data}. + * @param timecode The timestamp in milliseconds. + * @param trackId The ID of the track to process subtitles from. + */ + public void prepareProcessChunk( + byte[] data, + int offset, + int length, + long timecode, + String trackId) { + Long trackPtr = assTrackPtrs.get(trackId); + if (trackPtr == null) { + Log.w(TAG, "The trackID '" + trackId + "' isn't registered."); + return; + } + + // Find the first and second comma positions + int firstComma = -1, secondComma = -1, commaCount = 0; + for (int i = offset; i < offset + length; i++) { + if (data[i] == ',') { + commaCount++; + if (commaCount == 1) { + firstComma = i; + } else { + secondComma = i; + break; + } + } + } + + // If event formatting is wrong + if (secondComma == -1) { + String dialogueLine = new String(data, offset, length, StandardCharsets.UTF_8); + Log.w(TAG, "Skipping dialogue line with fewer columns than 2: " + dialogueLine); + return; + } + + // Extract the timestamp + int timestampLength = secondComma - firstComma - 1; + byte[] timestampBytes = new byte[timestampLength]; + System.arraycopy(data, firstComma + 1, timestampBytes, 0, timestampLength); + + long durationMs = SsaParser.parseTimecodeUs(new String(timestampBytes)) / 1000; + + // Isolate the part after the end time. + // Ex: + // From: "Dialogue: 0:00:00:00,0:00:05:00,1,0,Default,,0,0,0,,Line Text" + // Result: "1,0,Default,,0,0,0,,Line Text" + int line_offset = secondComma + 1; + int line_length = offset + length - secondComma - 1; + + assProcessChunk(trackPtr, data, line_offset, line_length, timecode, durationMs); + } + + /** + * Processes codec private data (subtitle headers) for a specific track. + * + * @param trackId The ID of the track to process the data for. + * @param data The codec private data bytes. + */ + public void processCodecPrivate(String trackId, byte[] data) { + Long trackPtr = assTrackPtrs.get(trackId); + assProcessCodecPrivate(trackPtr, data); + } + + /** + * Renders a frame for a specific track at the given timestamp. + * + * @param trackId The ID of the track to render. + * @param timeMs The timestamp in milliseconds. + * @return A bitmap with the rendered subtitle image, or null if no image was rendered. + */ + @Nullable + public AssRenderResult renderFrame(String trackId, long timeMs) { + Long trackPtr = assTrackPtrs.get(trackId); + if (trackPtr == null) { + Log.w(TAG, "The trackID '" + trackId + "' isn't registered."); + return null; + } + if (frame_width == null || frame_height == null) { + throw new RuntimeException("Frame size has not been set"); + } + return assRenderFrame(assRendererPtr, trackPtr, frame_width, frame_height, timeMs); + } + + /** + * Loads a font from its raw byte data and adds it to the ASS_Library. + * + * @param fileName The name of the font file. + * @param fontData The raw byte data of the font. + */ + public void loadFont(String fileName, byte[] fontData) { + assAddFont(assLibraryPtr, fileName, fontData); + } + + @Override + protected void finalize() throws Throwable { + try { + for (Map.Entry entry : assTrackPtrs.entrySet()) { + if (entry.getValue() != 0) { + assFreeTrack(entry.getValue()); + } + } + + if (assRendererPtr != 0) { + assRendererDone(assRendererPtr); + } + if (assLibraryPtr != 0) { + assLibraryDone(assLibraryPtr); + } + } finally { + super.finalize(); + } + } + + /** + * Adds a font to the ASS_Library. + * + * @param assLibraryPtr The pointer to the native ASS_Library instance. + * @param fontName The name of the font. + * @param fontData The raw byte data of the font. + */ + private native void assAddFont(long assLibraryPtr, String fontName, byte[] fontData); + + /** + * Initializes the native ASS_Library and returns its pointer as a long. + * This pointer must be passed to native methods that require it. + */ + private native long assLibraryInit(); + + /** + * Destroys the native ASS_Library instance. + * + * @param assLibraryPtr The pointer to the native ASS_Library instance. + */ + private native void assLibraryDone(long assLibraryPtr); + + /** + * Initializes the native ASS_Renderer and returns its pointer as a long. + * This pointer must be passed to native methods that require it. + * + * @param assLibraryPtr The pointer to the native ASS_Library instance. + * @return The pointer to the native ASS_Renderer instance. + */ + private native long assRendererInit(long assLibraryPtr); + + /** + * Destroys the native ASS_Renderer instance. + * + * @param assRendererPtr The pointer to the native ASS_Renderer instance. + */ + private native void assRendererDone(long assRendererPtr); + + /** + * Sets the frame size for the ASS_Renderer. + * + * @param assRendererPtr The pointer to the native ASS_Renderer instance. + * @param width The width of the frame in pixels. + * @param height The height of the frame in pixels. + */ + private native void assSetFrameSize(long assRendererPtr, int width, int height); + + /** + * Sets the storage size for the ASS_Renderer. + * + * @param assRendererPtr The pointer to the native ASS_Renderer instance. + * @param width The width of the storage in pixels. + * @param height The height of the storage in pixels. + */ + private native void assSetStorageSize(long assRendererPtr, int width, int height); + + /** + * Creates a new ASS_Track instance. + * + * @param assLibraryPtr The pointer to the native ASS_Library instance. + * @return The pointer to the created ASS_Track instance. + */ + private native long assNewTrack(long assLibraryPtr); + + /** + * Destroys the ASS_Track instance. + * + * @param assTrackPtr The pointer to the native ASS_Track instance. + */ + private native void assFreeTrack(long assTrackPtr); + + + /** + * Process a chunk of subtitle stream format + * + * @param assTrackPtr The pointer to the native ASS_Track instance. + * @param eventData The ass subtitle event. + * @param offset The index in {@code eventData} to start reading from (inclusive). + * @param length The number of bytes to read from {@code eventData}. + * @param timecode The timestamp in milliseconds. + * @param duration The duration of the event. + */ + private native void assProcessChunk(long assTrackPtr, byte[] eventData, int offset, int length, + long timecode, long duration); + + + private native AssRenderResult assRenderFrame(long assRendererPtr, long assTrackPtr, + int frame_width, int frame_height, long timeMs); + + + /** + * Processes codec private data (subtitle headers) for the ASS_Track. + * + * @param assTrackPtr The pointer to the native ASS_Track instance. + * @param data The codec private data bytes. + */ + private native void assProcessCodecPrivate(long assTrackPtr, byte[] data); +} diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/package-info.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/package-info.java new file mode 100644 index 00000000000..9eb0cc8ca3a --- /dev/null +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package androidx.media3.decoder.ass; + +import androidx.media3.common.util.NonNullApi; diff --git a/libraries/decoder_ass/src/main/jni/CMakeLists.txt b/libraries/decoder_ass/src/main/jni/CMakeLists.txt new file mode 100644 index 00000000000..d13ab5f24cb --- /dev/null +++ b/libraries/decoder_ass/src/main/jni/CMakeLists.txt @@ -0,0 +1,57 @@ +# +# Copyright 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) + +# Enable C++11 features. +set(CMAKE_CXX_STANDARD 14) + +# Define project name for your JNI module +project(libassJNI C CXX) + +set(libass_location "${CMAKE_CURRENT_SOURCE_DIR}/ass/${ANDROID_ABI}") +set(libass_binaries "${libass_location}/usr/local/lib") + +foreach(libass_lib fribidi harfbuzz freetype expat fontconfig ass) + set(libass_lib_filename lib${libass_lib}.a) + set(libass_lib_file_path ${libass_binaries}/${libass_lib_filename}) + + add_library( + ${libass_lib} + STATIC + IMPORTED) + set_target_properties( + ${libass_lib} PROPERTIES + IMPORTED_LOCATION + ${libass_lib_file_path}) +endforeach() + +include_directories("${libass_location}/usr/local/include") + +add_library(assJNI + SHARED + ass_jni.cpp) + +target_link_libraries(assJNI + android + log + jnigraphics + ass + fribidi + harfbuzz + freetype + fontconfig + expat) \ No newline at end of file diff --git a/libraries/decoder_ass/src/main/jni/ass_jni.cpp b/libraries/decoder_ass/src/main/jni/ass_jni.cpp new file mode 100644 index 00000000000..2b750a63dea --- /dev/null +++ b/libraries/decoder_ass/src/main/jni/ass_jni.cpp @@ -0,0 +1,287 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "ass/ass.h" +#include "ass/ass_types.h" + +#define LIBASS_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE Java_androidx_media3_decoder_ass_LibassJNI_##NAME( \ + JNIEnv* env, jobject thiz, ##__VA_ARGS__); \ + } \ + JNIEXPORT RETURN_TYPE Java_androidx_media3_decoder_ass_LibassJNI_##NAME( \ + JNIEnv* env, jobject thiz, ##__VA_ARGS__) + +static void draw_ass_rgba(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src, ptrdiff_t src_stride, + int w, int h, uint32_t color) { + const uint8_t ass_r = (color >> 24) & 0xff; // Red (bits 24-31) + const uint8_t ass_g = (color >> 16) & 0xff; // Green (bits 16-23) + const uint8_t ass_b = (color >> 8) & 0xff; // Blue (bits 8-15) + const uint8_t ass_a = 0xff - (color & 0xff); // Inverted Alpha (ASS uses 0 = opaque) + + // From libass: https://github.com/libass/libass/blob/1b699559025185e34d21a24cac477ca360cb917d/test/test.c#L149-L165 + const uint16_t ROUNDING_OFFSET = 255 * 255 / 2; + for (size_t y = 0; y < h; y++) { + for (size_t x = 0; x < w; x++) { + uint16_t k = src[x] * ass_a; + dst[x * 4 + 0] = (k * ass_r + (255 * 255 - k) * dst[x * 4 + 0] + ROUNDING_OFFSET) / (255 * 255); + dst[x * 4 + 1] = (k * ass_g + (255 * 255 - k) * dst[x * 4 + 1] + ROUNDING_OFFSET) / (255 * 255); + dst[x * 4 + 2] = (k * ass_b + (255 * 255 - k) * dst[x * 4 + 2] + ROUNDING_OFFSET) / (255 * 255); + dst[x * 4 + 3] = (k * 255 + (255 * 255 - k) * dst[x * 4 + 3] + ROUNDING_OFFSET) / (255 * 255); + } + src += src_stride; + dst += dst_stride; + } +} + +// Callback de libass pour les messages d'erreur +void libass_msg_callback(int level, const char *fmt, va_list args, void *data) { + if (level < 6) { + __android_log_vprint(ANDROID_LOG_DEBUG, "LIBASS_LOG", fmt, args); + } +} + +// Function to initialize the ASS_Library +LIBASS_FUNC(jlong, assLibraryInit) { + ASS_Library *library = ass_library_init(); + if (!library) { + return reinterpret_cast(nullptr); + } + + ass_set_message_cb(library, libass_msg_callback, nullptr); + return reinterpret_cast(library); +} + +// Destroy ASS_Library +LIBASS_FUNC(void, assLibraryDone, jlong ass_library_ptr) { + ASS_Library *library = reinterpret_cast(ass_library_ptr); + if (library) { + ass_library_done(library); + } +} + +// add fonts to the library +LIBASS_FUNC(void, assAddFont, jlong ass_library_ptr, jstring font_name, jbyteArray font_data) { + ASS_Library *library = reinterpret_cast(ass_library_ptr); + if (!library) { + return; + } + + // Convert jstring to const char * + const char *name = env->GetStringUTFChars(font_name, nullptr); + if (!name) { + return; + } + + // Convert jbyteArray to const char * + jbyte *data = env->GetByteArrayElements(font_data, nullptr); + if (!data) { + env->ReleaseStringUTFChars(font_name, name); + return; + } + jsize data_size = env->GetArrayLength(font_data); + + ass_add_font(library, name, reinterpret_cast(data), data_size); + + // Release the JNI resources + env->ReleaseStringUTFChars(font_name, name); + env->ReleaseByteArrayElements(font_data, data, 0); +} + +// Prepare data for processChunk +LIBASS_FUNC(void, assProcessChunk, jlong track, jbyteArray eventData, + jint offset, jint length, jlong timecode, jlong duration) { + jbyte *data = env->GetByteArrayElements(eventData, nullptr); + if (!data) { + return; + } + + ass_process_chunk(reinterpret_cast(track), + reinterpret_cast(data + offset), length, timecode, duration); + env->ReleaseByteArrayElements(eventData, data, 0); +} + + +// Initialize the ASS_Renderer +LIBASS_FUNC(jlong, assRendererInit, jlong ass_library_ptr) { + ASS_Library *library = reinterpret_cast(ass_library_ptr); + if (!library) { + return NULL; + } + + ASS_Renderer *renderer = ass_renderer_init(library); + if (!renderer) { + return NULL; + } + + ass_set_fonts(renderer, NULL, NULL, ASS_FONTPROVIDER_AUTODETECT, NULL, 1); + return reinterpret_cast(renderer); +} + + +// Destroy Ass Renderer instance +LIBASS_FUNC(void, assRendererDone, jlong ass_renderer_ptr) { + ASS_Renderer *renderer = reinterpret_cast(ass_renderer_ptr); + if (renderer) { + ass_renderer_done(renderer); + } +} + +// Sets the frame size for the ASS_Renderer. +LIBASS_FUNC(void, assSetFrameSize, jlong ass_renderer_ptr, jint width, jint height) { + + ASS_Renderer *renderer = reinterpret_cast(ass_renderer_ptr); + if (!renderer) { + return; + } + + ass_set_frame_size(renderer, width, height); +} + + +// Sets the storage size for the ASS_Renderer. +LIBASS_FUNC(void, assSetStorageSize, jlong ass_renderer_ptr, jint width, jint height) { + ASS_Renderer *renderer = reinterpret_cast(ass_renderer_ptr); + if (!renderer) { + return; + } + + ass_set_storage_size(renderer, width, height); +} + +// Creates new ASS_Track +LIBASS_FUNC(jlong, assNewTrack, jlong ass_library_ptr) { + auto *library = reinterpret_cast(ass_library_ptr); + if (!library) { + return NULL; + } + + ASS_Track *track = ass_new_track(library); + if (!track) { + return NULL; + } + + return reinterpret_cast(track); +} + + +// Destroys the ASS_Track instance. +LIBASS_FUNC(void, assFreeTrack, jlong ass_track_ptr) { + ASS_Track *track = reinterpret_cast(ass_track_ptr); + if (track) { + ass_free_track(track); + } +} + +// Process codec private data +LIBASS_FUNC(void, assProcessCodecPrivate, jlong ass_track_ptr, jbyteArray data) { + ASS_Track *track = reinterpret_cast(ass_track_ptr); + if (!track) { + return; + } + + // Convert jbyteArray to const char * + jbyte *data_bytes = env->GetByteArrayElements(data, nullptr); + if (!data_bytes) { + return; + } + jsize data_size = env->GetArrayLength(data); + + // Call the libass function + ass_process_codec_private(track, reinterpret_cast(data_bytes), data_size); + + // Release the JNI resources + env->ReleaseByteArrayElements(data, data_bytes, JNI_OK); +} + + +// Renders a frame for a specific track at the given timestamp. +LIBASS_FUNC(jobject, assRenderFrame, jlong ass_renderer_ptr, jlong ass_track_ptr, jint frame_width, jint frame_height, jlong time_ms, jint video_color_space, jint video_color_range) { + jclass resultClass = env->FindClass("androidx/media3/decoder/ass/AssRenderResult"); + jmethodID resultConstructor = env->GetMethodID(resultClass, "", "(Landroid/graphics/Bitmap;Z)V"); + + ASS_Renderer *renderer = reinterpret_cast(ass_renderer_ptr); + ASS_Track *track = reinterpret_cast(ass_track_ptr); + + if (!renderer || !track) { + return env->NewObject(resultClass, resultConstructor, (jobject) nullptr, JNI_FALSE); + } + + int detect_change; + ASS_Image *img = ass_render_frame(renderer, track, time_ms, &detect_change); + jboolean changedSinceLastCall = detect_change ? JNI_TRUE : JNI_FALSE; + + if (!detect_change || !img) { + return env->NewObject(resultClass, resultConstructor, (jobject) nullptr, changedSinceLastCall); + } + + // Create an Android Bitmap + jclass bitmapClass = env->FindClass("android/graphics/Bitmap"); + jmethodID createBitmapMethod = env->GetStaticMethodID( + bitmapClass, + "createBitmap", + "(IILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;" + ); + + // Use ARGB_8888 configuration for transparent bitmap + jobject bitmap = env->CallStaticObjectMethod( + bitmapClass, + createBitmapMethod, + frame_width, + frame_height, + env->GetStaticObjectField( + env->FindClass("android/graphics/Bitmap$Config"), + env->GetStaticFieldID(env->FindClass("android/graphics/Bitmap$Config"), + "ARGB_8888", "Landroid/graphics/Bitmap$Config;") + ) + ); + + AndroidBitmapInfo bitmapInfo; + void *pixels = nullptr; + if (AndroidBitmap_getInfo(env, bitmap, &bitmapInfo) != ANDROID_BITMAP_RESULT_SUCCESS || + AndroidBitmap_lockPixels(env, bitmap, &pixels) != ANDROID_BITMAP_RESULT_SUCCESS) { + return env->NewObject(resultClass, resultConstructor, (jobject) nullptr, changedSinceLastCall); + } + + for (ASS_Image *current = img; current; current = current->next) { + if (current->w <= 0 || current->h <= 0) { + continue; + } + + uint8_t *dst = reinterpret_cast(pixels) + + current->dst_y * bitmapInfo.stride + // Vertical offset + current->dst_x * 4; // Horizontal offset (4 bytes per pixel) + + draw_ass_rgba( + dst, + bitmapInfo.stride, + current->bitmap, + current->stride, + current->w, + current->h, + current->color + ); + } + + AndroidBitmap_unlockPixels(env, bitmap); + + return env->NewObject(resultClass, resultConstructor, bitmap, changedSinceLastCall); +} \ No newline at end of file diff --git a/libraries/decoder_ass/src/main/jni/build_libass.sh b/libraries/decoder_ass/src/main/jni/build_libass.sh new file mode 100755 index 00000000000..b528ffd58b9 --- /dev/null +++ b/libraries/decoder_ass/src/main/jni/build_libass.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +# If an error occurs, stop the script +set -eu + +# Base directory (where the script is located) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Build directory +mkdir -p "ass" +BUILD_DIR="$SCRIPT_DIR/ass" + +# Android NDK configuration +NDK_PATH="$1" +HOST_PLATFORM="$2" +ANDROID_ABI_VERSION="$3" +TOOLCHAIN="$NDK_PATH/toolchains/llvm/prebuilt/$HOST_PLATFORM" +CROSS_FILE_PATH="$BUILD_DIR/cross-file.tmp" + + +# Function to check if NDK is properly set up +check_ndk_setup() { + if [ ! -d "$NDK_PATH" ]; then + echo "Error: Android NDK not found at $NDK_PATH" + echo "Please verify the NDK path. Current path is set to: $NDK_PATH" + exit 1 + fi + + if [ ! -d "$TOOLCHAIN" ]; then + echo "Error: Toolchain not found at $TOOLCHAIN" + echo "Please verify that the NDK installation is complete" + exit 1 + fi +} + +# Function to create a Meson cross file for cross-compilation +create_meson_cross_file() { + local cpu_family=$CPU + [ "$cpu_family" == "i686" ] && cpu_family=x86 + + cat > "$CROSS_FILE_PATH" < out) { out.add(new TextRenderer(output, outputLooper)); + + if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { + return; + } + int extensionRendererIndex = out.size(); + /*if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { + extensionRendererIndex--; + }*/ + // TODO Remove hack. We suppose that when EXTENSION_RENDERER_MODE_ON, we actually mean EXTENSION_RENDERER_MODE_PREFER + // This is necessary because + extensionRendererIndex--; + + try { + Class clazz = Class.forName("androidx.media3.decoder.ass.AssRenderer"); + Constructor constructor = + clazz.getConstructor( + TextOutput.class, + Looper.class); + // LINT.ThenChange(../../../../../../proguard-rules.txt) + Renderer renderer = + (Renderer) constructor.newInstance(output, outputLooper); + out.add(extensionRendererIndex++, renderer); + Log.i(TAG, "Loaded AssRenderer."); + } catch (ClassNotFoundException e) { + // Expected if the app was built without the extension. + } catch (Exception e) { + // The extension is present, but instantiation failed. + throw new IllegalStateException("Error instantiating ASS extension", e); + } } /** diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java index a448e4f249f..4daecb4d219 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java @@ -34,6 +34,7 @@ import androidx.media3.extractor.mkv.FontMetadataEntry; import java.io.EOFException; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -93,10 +94,34 @@ public void format(Format format) { checkArgument(MimeTypes.getTrackType(format.sampleMimeType) == C.TRACK_TYPE_TEXT); if (!format.equals(currentFormat)) { currentFormat = format; - currentSubtitleParser = - subtitleParserFactory.supportsFormat(format) - ? subtitleParserFactory.create(format) - : null; + switch (format.sampleMimeType) { + case MimeTypes.TEXT_SSA: + // TODO Remove hack. We suppose that when extensionRendererMode is NOT EXTENSION_RENDERER_MODE_OFF, + // we suppose it is EXTENSION_RENDERER_MODE_PREFER (but it may be EXTENSION_RENDERER_MODE_ON), so the renderer may be TextRenderer (not AssRenderer). + // This means we may send the ass file to TextRenderer instead of Cues. + // This also depends on the hack in the method `buildTextRenderers` of DefaultRenderersFactory.java + boolean isAssNativeLibraryAvailable = false; + try { + isAssNativeLibraryAvailable = + Boolean.TRUE.equals( + Class.forName("androidx.media3.decoder.ass.AssLibrary") + .getMethod("isAvailable") + .invoke(/* obj= */ null)); + } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e){ + } + + if (isAssNativeLibraryAvailable || !subtitleParserFactory.supportsFormat(format)) { + currentSubtitleParser = null; + } else { + currentSubtitleParser = subtitleParserFactory.create(format); + } + break; + default: + currentSubtitleParser = + subtitleParserFactory.supportsFormat(format) + ? subtitleParserFactory.create(format) + : null; + } } // Ensure currentFormat always matches what is sent to the delegate, diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/ssa/SsaParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/ssa/SsaParser.java index 6576ca05e75..3deb3cbe12d 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/ssa/SsaParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/ssa/SsaParser.java @@ -367,7 +367,7 @@ private void parseDialogueLine( * @param timeString The string to parse. * @return The parsed timestamp in microseconds. */ - private static long parseTimecodeUs(String timeString) { + public static long parseTimecodeUs(String timeString) { Matcher matcher = SSA_TIMECODE_PATTERN.matcher(timeString.trim()); if (!matcher.matches()) { return C.TIME_UNSET; From ec2a1f9eae332ae073bb21f0a3d9b49f69895560 Mon Sep 17 00:00:00 2001 From: moi15moi <80980684+moi15moi@users.noreply.github.com> Date: Mon, 7 Apr 2025 16:56:16 -0400 Subject: [PATCH 6/6] Do the "VSFilter mangled colors" The "VSFilter mangled colors" is requested by libass. See: https://github.com/libass/libass/blob/5b44c4d690254a7bd8de2e048fa23820036b671b/libass/ass_types.h#L157-L262 The math formula are from: https://en.wikipedia.org/wiki/YCbCr#R'G'B'_to_Y%E2%80%B2PbPr Co-authored-by: Vincent Zauhar --- libraries/decoder_ass/build.gradle | 5 +- .../src/androidTest/AndroidManifest.xml | 32 +++ .../media3/decoder/ass/LibassJNITest.java | 146 ++++++++++ .../media3/decoder/ass/AssRenderer.java | 7 +- .../media3/decoder/ass/LibassJNI.java | 25 +- .../decoder_ass/src/main/jni/CMakeLists.txt | 4 +- .../decoder_ass/src/main/jni/ass_jni.cpp | 261 +++++++++++++++++- 7 files changed, 459 insertions(+), 21 deletions(-) create mode 100644 libraries/decoder_ass/src/androidTest/AndroidManifest.xml create mode 100644 libraries/decoder_ass/src/androidTest/java/androidx/media3/decoder/ass/LibassJNITest.java diff --git a/libraries/decoder_ass/build.gradle b/libraries/decoder_ass/build.gradle index c2b5250ba6c..b9d8443ab6a 100644 --- a/libraries/decoder_ass/build.gradle +++ b/libraries/decoder_ass/build.gradle @@ -41,7 +41,6 @@ dependencies { implementation project(modulePrefix + 'lib-exoplayer') implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion - compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion - testImplementation project(modulePrefix + 'test-utils') - testImplementation 'org.robolectric:robolectric:' + robolectricVersion + androidTestImplementation project(modulePrefix + 'test-utils') + androidTestImplementation 'androidx.test:runner:' + androidxTestRunnerVersion } diff --git a/libraries/decoder_ass/src/androidTest/AndroidManifest.xml b/libraries/decoder_ass/src/androidTest/AndroidManifest.xml new file mode 100644 index 00000000000..6b9f8dd3c94 --- /dev/null +++ b/libraries/decoder_ass/src/androidTest/AndroidManifest.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + diff --git a/libraries/decoder_ass/src/androidTest/java/androidx/media3/decoder/ass/LibassJNITest.java b/libraries/decoder_ass/src/androidTest/java/androidx/media3/decoder/ass/LibassJNITest.java new file mode 100644 index 00000000000..657f2401c01 --- /dev/null +++ b/libraries/decoder_ass/src/androidTest/java/androidx/media3/decoder/ass/LibassJNITest.java @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.ass; + +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Color; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit test for {@link LibassJNITest}. + */ +@RunWith(AndroidJUnit4.class) +public class LibassJNITest { + + private String buildHeader(@Nullable String ycbcrMatrix) { + return "[Script Info]\n" + + "ScriptType: v4.00+\n" + + "WrapStyle: 0\n" + + "ScaledBorderAndShadow: yes\n" + + (ycbcrMatrix != null ? "YCbCr Matrix: " + ycbcrMatrix + "\n" : "") + + "PlayResX: 640\n" + + "PlayResY: 480\n" + + "\n" + + "[V4+ Styles]\n" + + "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n" + + "Style: Default,Arial,48,&H00000000,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,0,0,2,0,0,0,1\n"; + } + + private LibassJNI setupLibassJNI(@Nullable String ycbcrMatrix, int videoColorSpace, int videoColorRange, + int initialRed, int initialGreen, int initialBlue) { + LibassJNI libassJNI = new LibassJNI(); + libassJNI.setFrameSize(640, 480); + libassJNI.setStorageSize(640, 480); + libassJNI.setVideoColorProperties(videoColorSpace, videoColorRange); + libassJNI.createTrack("0"); + + libassJNI.processCodecPrivate("0", buildHeader(ycbcrMatrix).getBytes(StandardCharsets.UTF_8)); + + String colorTag = "\\c&H" + String.format("%02x", initialBlue) + String.format("%02x", initialGreen) + String.format("%02x", initialRed) + "&"; + String line = "Dialogue: 0:00:00:00,0:00:05:00,1,0,Default,,0,0,0,,{\\an7\\pos(0,0)\\p1" + colorTag + "}m 0 0 l 640 0 640 480 0 480"; + byte[] line_bytes = line.getBytes(StandardCharsets.UTF_8); + ByteBuffer line_buffer = ByteBuffer.wrap(line_bytes); + libassJNI.prepareProcessChunk(line_buffer.array(), line_buffer.position(), line_buffer.remaining(), 0, "0"); + return libassJNI; + } + + private void assertCenterPixel(AssRenderResult result, int expectedRed, int expectedGreen, int expectedBlue) { + assertThat(result.changedSinceLastCall).isTrue(); + assertThat(result.bitmap).isNotNull(); + int color = result.bitmap.getPixel(640 / 2, 480 / 2); + assertThat(Color.red(color)).isEqualTo(expectedRed); + assertThat(Color.green(color)).isEqualTo(expectedGreen); + assertThat(Color.blue(color)).isEqualTo(expectedBlue); + assertThat(Color.alpha(color)).isEqualTo(255); + } + + @Test + public void renderFrameTV601ToTV709() { + LibassJNI libassJNI = setupLibassJNI("TV.601", C.COLOR_SPACE_BT709, C.COLOR_RANGE_LIMITED, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 155, 104, 78); + } + + @Test + public void renderFrameTV601ToPC709() { + LibassJNI libassJNI = setupLibassJNI("TV.601", C.COLOR_SPACE_BT709, C.COLOR_RANGE_FULL, 150, 100, 80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 150, 105, 83); + } + + @Test + public void renderFrameTV709ToTV601() { + LibassJNI libassJNI = setupLibassJNI("TV.709", C.COLOR_SPACE_BT601, C.COLOR_RANGE_LIMITED, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 146, 96, 81); + } + + @Test + public void renderFrameTV709ToPC601() { + LibassJNI libassJNI = setupLibassJNI("TV.709", C.COLOR_SPACE_BT601, C.COLOR_RANGE_FULL, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 142, 98, 85); + } + + @Test + public void renderFrameTVFCCToTV709() { + LibassJNI libassJNI = setupLibassJNI("TV.FCC", C.COLOR_SPACE_BT709, C.COLOR_RANGE_LIMITED, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 155, 104, 79); + } + + @Test + public void renderFrameTVFCCToPC709() { + LibassJNI libassJNI = setupLibassJNI("TV.FCC", C.COLOR_SPACE_BT709, C.COLOR_RANGE_FULL, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 150, 105, 83); + } + + @Test + public void renderFrameTV240MToTV709() { + LibassJNI libassJNI = setupLibassJNI("TV.240M", C.COLOR_SPACE_BT709, C.COLOR_RANGE_LIMITED, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 150, 100, 80); + } + + @Test + public void renderFrameTV240MToPC709() { + LibassJNI libassJNI = setupLibassJNI("TV.240M", C.COLOR_SPACE_BT709, C.COLOR_RANGE_FULL, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 146, 101, 84); + } + + @Test + public void renderFrameNone() { + LibassJNI libassJNI = setupLibassJNI("None", C.COLOR_SPACE_BT709, C.COLOR_RANGE_LIMITED, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 150,100,80); + } + + @Test + public void renderFrameNoYcbcrMatrix() { + LibassJNI libassJNI = setupLibassJNI(null, C.COLOR_SPACE_BT709, C.COLOR_RANGE_LIMITED, 150,100,80); + AssRenderResult result = libassJNI.renderFrame("0", 0); + assertCenterPixel(result, 155, 104, 78); + } +} \ No newline at end of file diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java index c3540a5ec13..3de18f71314 100644 --- a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/AssRenderer.java @@ -331,12 +331,11 @@ public void handleMessage(@MessageType int messageType, @Nullable Object message break; case MSG_EVENT_VIDEO_FORMAT_CHANGED: - // TODO: Handle this case properly in the alpha blending Format videoFormat = (Format) checkNotNull(message, "Video format message cannot be null"); - if (videoFormat.colorInfo != null) { - Log.d(TAG, "videoFormat.colorInfo = " + videoFormat.colorInfo); + if (videoFormat.colorInfo == null) { + libassJNI.setVideoColorProperties(Format.NO_VALUE, Format.NO_VALUE); } else { - Log.d(TAG, "The video format does not have a defined color info."); + libassJNI.setVideoColorProperties(videoFormat.colorInfo.colorSpace, videoFormat.colorInfo.colorRange); } break; diff --git a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java index e7108c83a51..53517edc37b 100644 --- a/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java +++ b/libraries/decoder_ass/src/main/java/androidx/media3/decoder/ass/LibassJNI.java @@ -16,6 +16,8 @@ package androidx.media3.decoder.ass; import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; import androidx.media3.common.util.Log; import androidx.media3.extractor.text.ssa.SsaParser; import java.nio.charset.StandardCharsets; @@ -34,6 +36,9 @@ public class LibassJNI { @Nullable private Integer frame_height = null; + private @C.ColorSpace int videoColorSpace; + private @C.ColorRange int videoColorRange; + public LibassJNI() { if (!AssLibrary.isAvailable()) { throw new RuntimeException("Libass native library is not available"); @@ -48,6 +53,9 @@ public LibassJNI() { if (assRendererPtr == 0) { throw new RuntimeException("Failed to initialize ASS_Renderer"); } + + this.videoColorSpace = Format.NO_VALUE; + this.videoColorRange = Format.NO_VALUE; } /** @@ -72,6 +80,18 @@ public void setStorageSize(int width, int height) { assSetStorageSize(assRendererPtr, width, height); } + /** + * Sets the video color space and color range + * that will be used for the alpha blending. + * + * @param videoColorSpace The video color space. + * @param videoColorRange The video color range. + */ + public void setVideoColorProperties(@C.ColorSpace int videoColorSpace, @C.ColorRange int videoColorRange) { + this.videoColorSpace = videoColorSpace; + this.videoColorRange = videoColorRange; + } + /** * Creates a new ASS_Track instance if it does not already exist for the given format ID. * @@ -187,7 +207,8 @@ public AssRenderResult renderFrame(String trackId, long timeMs) { if (frame_width == null || frame_height == null) { throw new RuntimeException("Frame size has not been set"); } - return assRenderFrame(assRendererPtr, trackPtr, frame_width, frame_height, timeMs); + return assRenderFrame(assRendererPtr, trackPtr, frame_width, frame_height, timeMs, + videoColorSpace, videoColorRange); } /** @@ -307,7 +328,7 @@ private native void assProcessChunk(long assTrackPtr, byte[] eventData, int offs private native AssRenderResult assRenderFrame(long assRendererPtr, long assTrackPtr, - int frame_width, int frame_height, long timeMs); + int frame_width, int frame_height, long timeMs, int colorSpace, int colorRange); /** diff --git a/libraries/decoder_ass/src/main/jni/CMakeLists.txt b/libraries/decoder_ass/src/main/jni/CMakeLists.txt index d13ab5f24cb..70661b5a4fd 100644 --- a/libraries/decoder_ass/src/main/jni/CMakeLists.txt +++ b/libraries/decoder_ass/src/main/jni/CMakeLists.txt @@ -16,8 +16,8 @@ cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) -# Enable C++11 features. -set(CMAKE_CXX_STANDARD 14) +# Enable C++17 features. +set(CMAKE_CXX_STANDARD 17) # Define project name for your JNI module project(libassJNI C CXX) diff --git a/libraries/decoder_ass/src/main/jni/ass_jni.cpp b/libraries/decoder_ass/src/main/jni/ass_jni.cpp index 2b750a63dea..e986c9961a4 100644 --- a/libraries/decoder_ass/src/main/jni/ass_jni.cpp +++ b/libraries/decoder_ass/src/main/jni/ass_jni.cpp @@ -29,22 +29,190 @@ JNIEXPORT RETURN_TYPE Java_androidx_media3_decoder_ass_LibassJNI_##NAME( \ JNIEnv* env, jobject thiz, ##__VA_ARGS__) + +class RGB { + public: + double r_prime, g_prime, b_prime; + uint8_t r, g, b; + + RGB() = default; + + explicit RGB(uint8_t r, uint8_t g, uint8_t b) { + this->r = r; + this->g = g; + this->b = b; + + this->r_prime = this->r / 255.0; + this->g_prime = this->g / 255.0; + this->b_prime = this->b / 255.0; + } + + explicit RGB(double r_prime, double g_prime, double b_prime) { + this->r_prime = r_prime; + this->g_prime = g_prime; + this->b_prime = b_prime; + + this->r = std::lround(this->r_prime * 255.0); + this->g = std::lround(this->g_prime * 255.0); + this->b = std::lround(this->b_prime * 255.0); + } +}; + +struct YCbCr { + double y, cb, cr; +}; + +struct ColorSpace { + double kr, kg, kb; +}; + +enum class ColorSpaceEnum { + UNKNOWN, + BT601, + BT709, + FCC, + SMPTE_240M, + BT2020 +}; + +// Must match https://developer.android.com/reference/androidx/media3/common/C.ColorSpace +enum ColorSpaceMedia3 { + COLOR_SPACE_NO_VALUE = -1, + COLOR_SPACE_BT601 = 2, + COLOR_SPACE_BT709 = 1, + COLOR_SPACE_BT2020 = 6, +}; + +constexpr ColorSpaceEnum getColorSpaceFromMedia3ColorSpace(ColorSpaceMedia3 color_space) { + switch (color_space) { + case ColorSpaceMedia3::COLOR_SPACE_NO_VALUE: case ColorSpaceMedia3::COLOR_SPACE_BT709: return ColorSpaceEnum::BT709; + case ColorSpaceMedia3::COLOR_SPACE_BT601: return ColorSpaceEnum::BT601; + case ColorSpaceMedia3::COLOR_SPACE_BT2020: return ColorSpaceEnum::BT2020; + default: return ColorSpaceEnum::UNKNOWN; + } +} + +constexpr ColorSpace getColorSpace(ColorSpaceEnum space) { + switch (space) { + case ColorSpaceEnum::BT601: return {0.299, 0.587, 0.114}; + case ColorSpaceEnum::BT709: return {0.2126, 0.7152, 0.0722}; + case ColorSpaceEnum::FCC: return {0.3, 0.59, 0.11}; + case ColorSpaceEnum::SMPTE_240M: return {0.212, 0.701, 0.087}; + case ColorSpaceEnum::BT2020: return {0.2627, 0.6780, 0.0593}; + default: throw std::invalid_argument("Unsupported ColorSpaceEnum"); + } +} + +enum class ColorRange { + UNKNOWN, + FULL, + LIMITED +}; + +// Must match https://developer.android.com/reference/androidx/media3/common/C.ColorRange +enum ColorRangeMedia3 { + COLOR_RANGE_NO_VALUE = -1, + COLOR_RANGE_FULL = 1, + COLOR_RANGE_LIMITED = 2, +}; + +constexpr ColorRange getColorRangeFromMedia3ColorRange(ColorRangeMedia3 color_range) { + switch (color_range) { + case ColorRangeMedia3::COLOR_RANGE_NO_VALUE: case ColorRangeMedia3::COLOR_RANGE_LIMITED: return ColorRange::LIMITED; + case ColorRangeMedia3::COLOR_RANGE_FULL: return ColorRange::FULL; + default: return ColorRange::UNKNOWN; + } +} + +class ColorConverter { + private: + ColorSpace color_space; + ColorRange color_range; + + public: + ColorConverter(ColorSpace color_space, ColorRange color_range) + : color_space(color_space), color_range(color_range) {} + + YCbCr rgb_to_ycbcr(double r, double g, double b) const { + // y_prime is [0, 1] + // pb/pr are [-0.5, 0.5] + double y_prime = color_space.kr * r + color_space.kg * g + color_space.kb * b; + double pb = 0.5 * (b - y_prime) / (1.0 - color_space.kb); + double pr = 0.5 * (r - y_prime) / (1.0 - color_space.kr); + + double y, cb, cr; + if (color_range == ColorRange::FULL) { + // y is [0, 255] + // cb/cr are [0.5, 255.5] (but, anything above 255.5 is clipped to 255) + y = y_prime * 255.0; + cb = std::min(pb * 255.0 + 128.0, 255.0); + cr = std::min(pr * 255.0 + 128.0, 255.0); + } else if (color_range == ColorRange::LIMITED) { + // y is [16, 235] + // cb/cr are [16, 240] + y = y_prime * 219.0 + 16.0; + cb = pb * 224.0 + 128.0; + cr = pr * 224.0 + 128.0; + } else { + throw std::invalid_argument("Unsupported ColorRange"); + } + return YCbCr{y, cb, cr}; + } + + RGB ycbcr_to_rgb(double y, double cb, double cr) const { + double y_prime, pb, pr; + if (color_range == ColorRange::FULL) { + // y is [0, 255] -> y_prime is [0, 1] + // cb/cr are [0.5, 255] -> pb/pr are [-0.5, 0.5] + y_prime = y / 255.0; + pb = (cb - 128.0) / 255.0; + pr = (cr - 128.0) / 255.0; + } else if (color_range == ColorRange::LIMITED) { + // y is [16, 235] -> y_prime is [0, 1] + // cb/cr are [16, 240] -> pb/pr are [-0.5, 0.5] + y_prime = (y - 16.0) / 219.0; + pb = (cb - 128.0) / 224.0; + pr = (cr - 128.0) / 224.0; + } else { + throw std::invalid_argument("Unsupported ColorRange"); + } + + double r = y_prime + pr * (1.0 - color_space.kr) * 2.0; + double b = y_prime + pb * (1.0 - color_space.kb) * 2.0; + double g = (y_prime - color_space.kr * r - color_space.kb * b) / color_space.kg; + + return RGB(std::clamp(r, 0.0, 1.0), std::clamp(g, 0.0, 1.0), std::clamp(b, 0.0, 1.0)); + } + + static RGB rgb_to_rgb( + const ColorSpace src_color_space, + const ColorRange src_color_range, + const ColorSpace dst_color_space, + const ColorRange dst_color_range, + const RGB src_rgb + ) { + ColorConverter src_converter(src_color_space, src_color_range); + ColorConverter dst_converter(dst_color_space, dst_color_range); + + YCbCr ycbcr = src_converter.rgb_to_ycbcr(src_rgb.r_prime, src_rgb.g_prime, src_rgb.b_prime); + return dst_converter.ycbcr_to_rgb(ycbcr.y, ycbcr.cb, ycbcr.cr); + } +}; + + + static void draw_ass_rgba(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, ptrdiff_t src_stride, - int w, int h, uint32_t color) { - const uint8_t ass_r = (color >> 24) & 0xff; // Red (bits 24-31) - const uint8_t ass_g = (color >> 16) & 0xff; // Green (bits 16-23) - const uint8_t ass_b = (color >> 8) & 0xff; // Blue (bits 8-15) - const uint8_t ass_a = 0xff - (color & 0xff); // Inverted Alpha (ASS uses 0 = opaque) + int w, int h, RGB color, uint8_t alpha) { // From libass: https://github.com/libass/libass/blob/1b699559025185e34d21a24cac477ca360cb917d/test/test.c#L149-L165 const uint16_t ROUNDING_OFFSET = 255 * 255 / 2; for (size_t y = 0; y < h; y++) { for (size_t x = 0; x < w; x++) { - uint16_t k = src[x] * ass_a; - dst[x * 4 + 0] = (k * ass_r + (255 * 255 - k) * dst[x * 4 + 0] + ROUNDING_OFFSET) / (255 * 255); - dst[x * 4 + 1] = (k * ass_g + (255 * 255 - k) * dst[x * 4 + 1] + ROUNDING_OFFSET) / (255 * 255); - dst[x * 4 + 2] = (k * ass_b + (255 * 255 - k) * dst[x * 4 + 2] + ROUNDING_OFFSET) / (255 * 255); + uint16_t k = src[x] * alpha; + dst[x * 4 + 0] = (k * color.r + (255 * 255 - k) * dst[x * 4 + 0] + ROUNDING_OFFSET) / (255 * 255); + dst[x * 4 + 1] = (k * color.g + (255 * 255 - k) * dst[x * 4 + 1] + ROUNDING_OFFSET) / (255 * 255); + dst[x * 4 + 2] = (k * color.b + (255 * 255 - k) * dst[x * 4 + 2] + ROUNDING_OFFSET) / (255 * 255); dst[x * 4 + 3] = (k * 255 + (255 * 255 - k) * dst[x * 4 + 3] + ROUNDING_OFFSET) / (255 * 255); } src += src_stride; @@ -233,6 +401,65 @@ LIBASS_FUNC(jobject, assRenderFrame, jlong ass_renderer_ptr, jlong ass_track_ptr return env->NewObject(resultClass, resultConstructor, (jobject) nullptr, changedSinceLastCall); } + ColorSpaceEnum dst_color_space_enum = getColorSpaceFromMedia3ColorSpace((ColorSpaceMedia3) video_color_space); + if (dst_color_space_enum == ColorSpaceEnum::UNKNOWN) { + std::string errorMessage = "The color space " + std::to_string(video_color_space) + " is invalid"; + env->ThrowNew(env->FindClass("java/lang/IllegalArgumentException"), errorMessage.c_str()); + return env->NewObject(resultClass, resultConstructor, (jobject) nullptr, changedSinceLastCall); + } + + ColorRange dst_color_range = getColorRangeFromMedia3ColorRange((ColorRangeMedia3) video_color_range); + if (dst_color_range == ColorRange::UNKNOWN) { + std::string errorMessage = "The color range " + std::to_string(video_color_range) + " is invalid"; + env->ThrowNew(env->FindClass("java/lang/IllegalArgumentException"), errorMessage.c_str()); + return env->NewObject(resultClass, resultConstructor, (jobject) nullptr, changedSinceLastCall); + } + + ColorSpaceEnum src_color_space_enum = ColorSpaceEnum::UNKNOWN; + ColorRange src_color_range = ColorRange::UNKNOWN; + switch (track->YCbCrMatrix) { + case YCBCR_DEFAULT: + case YCBCR_UNKNOWN: + case YCBCR_BT601_TV: + src_color_space_enum = ColorSpaceEnum::BT601; + src_color_range = ColorRange::LIMITED; + break; + case YCBCR_BT601_PC: + src_color_space_enum = ColorSpaceEnum::BT601; + src_color_range = ColorRange::FULL; + break; + case YCBCR_BT709_TV: + src_color_space_enum = ColorSpaceEnum::BT709; + src_color_range = ColorRange::LIMITED; + break; + case YCBCR_BT709_PC: + src_color_space_enum = ColorSpaceEnum::BT709; + src_color_range = ColorRange::FULL; + break; + case YCBCR_SMPTE240M_TV: + src_color_space_enum = ColorSpaceEnum::SMPTE_240M; + src_color_range = ColorRange::LIMITED; + break; + case YCBCR_SMPTE240M_PC: + src_color_space_enum = ColorSpaceEnum::SMPTE_240M; + src_color_range = ColorRange::FULL; + break; + case YCBCR_FCC_TV: + src_color_space_enum = ColorSpaceEnum::FCC; + src_color_range = ColorRange::LIMITED; + break; + case YCBCR_FCC_PC: + src_color_space_enum = ColorSpaceEnum::FCC; + src_color_range = ColorRange::FULL; + break; + } + + ColorSpace src_color_space; + ColorSpace dst_color_space = getColorSpace(dst_color_space_enum); + if (src_color_space_enum != ColorSpaceEnum::UNKNOWN) { + src_color_space = getColorSpace(src_color_space_enum); + } + // Create an Android Bitmap jclass bitmapClass = env->FindClass("android/graphics/Bitmap"); jmethodID createBitmapMethod = env->GetStaticMethodID( @@ -266,6 +493,19 @@ LIBASS_FUNC(jobject, assRenderFrame, jlong ass_renderer_ptr, jlong ass_track_ptr continue; } + const uint8_t ass_r = (current->color >> 24) & 0xff; // Red (bits 24-31) + const uint8_t ass_g = (current->color >> 16) & 0xff; // Green (bits 16-23) + const uint8_t ass_b = (current->color >> 8) & 0xff; // Blue (bits 8-15) + const uint8_t ass_a = 0xff - (current->color & 0xff); // Inverted Alpha (ASS uses 0 = opaque) + + RGB src_rgb = RGB(ass_r, ass_g, ass_b); + RGB dst_rgb; + if (src_color_space_enum == ColorSpaceEnum::UNKNOWN || src_color_range == ColorRange::UNKNOWN) { + dst_rgb = src_rgb; + } else { + dst_rgb = ColorConverter::rgb_to_rgb(src_color_space, src_color_range, dst_color_space, dst_color_range, src_rgb); + } + uint8_t *dst = reinterpret_cast(pixels) + current->dst_y * bitmapInfo.stride + // Vertical offset current->dst_x * 4; // Horizontal offset (4 bytes per pixel) @@ -277,7 +517,8 @@ LIBASS_FUNC(jobject, assRenderFrame, jlong ass_renderer_ptr, jlong ass_track_ptr current->stride, current->w, current->h, - current->color + dst_rgb, + ass_a ); }