Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add captions to m.file, m.image, m.video, and m.audio #2059

Open
wants to merge 11 commits into
base: dev
Choose a base branch
from
+118 −73
Open

Add captions to m.file, m.image, m.video, and m.audio #2059

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
dccf87d
Add rendering image captions
nexy7574 Nov 17, 2024
9a3cfd0
Handle sending captions for images
nexy7574 Nov 17, 2024
905d796
Fix caption rendering on m.video and m.audio too
nexy7574 Nov 17, 2024
ea0f601
Remove unused renderBody() parameter
nexy7574 Nov 17, 2024
3d92efd
Fix m.file rendering body instead of filename where possible
nexy7574 Nov 17, 2024
78ffa88
Add caption rendering for generic files
nexy7574 Nov 17, 2024
a073aa8
Use m.text for captions & render on demand
nexy7574 Dec 7, 2024
41e426d
Allow custom HTML in sending captions
nexy7574 Dec 7, 2024
0dd3d4e
Don't *send* captions
nexy7574 Dec 15, 2024
eb24ed2
mvoe content const into renderCaption()
nexy7574 Dec 30, 2024
b6b98f8
Merge branch 'dev' into nexy7574/image-captions
ajbura Jan 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 104 additions & 68 deletions src/app/components/RenderMessageContent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { ImageViewer } from './image-viewer';
import { PdfViewer } from './Pdf-viewer';
import { TextViewer } from './text-viewer';
import { testMatrixTo } from '../plugins/matrix-to';
import {IImageContent} from "../../types/matrix/common";

type RenderMessageContentProps = {
displayName: string;
Expand Down Expand Up @@ -67,38 +68,62 @@ export function RenderMessageContent({
</UrlPreviewHolder>
);
};
const content: IImageContent = getContent();
const renderCaption = content.filename && content.filename !== content.body;
let renderedCaption: React.ReactNode = null;
if(renderCaption) {
nexy7574 marked this conversation as resolved.
Show resolved Hide resolved
renderedCaption = (
<MNotice
nexy7574 marked this conversation as resolved.
Show resolved Hide resolved
edited={edited}
content={getContent()}
renderBody={(props) => (
<RenderBody
{...props}
highlightRegex={highlightRegex}
htmlReactParserOptions={htmlReactParserOptions}
linkifyOpts={linkifyOpts}
/>
)}
renderUrlsPreview={urlPreview ? renderUrlsPreview : undefined}
/>
)
}

const renderFile = () => (
<MFile
content={getContent()}
renderFileContent={({ body, mimeType, info, encInfo, url }) => (
<FileContent
body={body}
mimeType={mimeType}
renderAsPdfFile={() => (
<ReadPdfFile
body={body}
mimeType={mimeType}
url={url}
encInfo={encInfo}
renderViewer={(p) => <PdfViewer {...p} />}
/>
)}
renderAsTextFile={() => (
<ReadTextFile
<>
<MFile
content={getContent()}
renderFileContent={({ body, mimeType, info, encInfo, url }) => (
<FileContent
body={body}
mimeType={mimeType}
url={url}
encInfo={encInfo}
renderViewer={(p) => <TextViewer {...p} />}
/>
)}
>
<DownloadFile body={body} mimeType={mimeType} url={url} encInfo={encInfo} info={info} />
</FileContent>
)}
outlined={outlineAttachment}
/>
renderAsPdfFile={() => (
<ReadPdfFile
body={body}
mimeType={mimeType}
url={url}
encInfo={encInfo}
renderViewer={(p) => <PdfViewer {...p} />}
/>
)}
renderAsTextFile={() => (
<ReadTextFile
body={body}
mimeType={mimeType}
url={url}
encInfo={encInfo}
renderViewer={(p) => <TextViewer {...p} />}
/>
)}
>
<DownloadFile body={body} mimeType={mimeType} url={url} encInfo={encInfo} info={info} />
</FileContent>

)}
outlined={outlineAttachment}
/>
{renderedCaption}
</>
);

if (msgType === MsgType.Text) {
Expand Down Expand Up @@ -158,63 +183,74 @@ export function RenderMessageContent({

if (msgType === MsgType.Image) {
return (
<MImage
content={getContent()}
renderImageContent={(props) => (
<ImageContent
{...props}
autoPlay={mediaAutoLoad}
renderImage={(p) => <Image {...p} loading="lazy" />}
renderViewer={(p) => <ImageViewer {...p} />}
/>
)}
outlined={outlineAttachment}
/>
<>
<MImage
content={getContent()}
renderImageContent={(props) => (
<ImageContent
{...props}
autoPlay={mediaAutoLoad}
renderImage={(p) => <Image {...p} loading="lazy" />}
renderViewer={(p) => <ImageViewer {...p} />}
/>
)}
outlined={outlineAttachment}
/>
{renderedCaption}
</>
);
}

if (msgType === MsgType.Video) {
return (
<MVideo
content={getContent()}
renderAsFile={renderFile}
renderVideoContent={({ body, info, mimeType, url, encInfo }) => (
<VideoContent
body={body}
info={info}
mimeType={mimeType}
url={url}
encInfo={encInfo}
renderThumbnail={
mediaAutoLoad
? () => (
<>
<MVideo
content={getContent()}
renderAsFile={renderFile}
renderVideoContent={({ body, info, mimeType, url, encInfo }) => (
<VideoContent
body={body}
info={info}
mimeType={mimeType}
url={url}
encInfo={encInfo}
renderThumbnail={
mediaAutoLoad
? () => (
<ThumbnailContent
info={info}
renderImage={(src) => (
<Image alt={body} title={body} src={src} loading="lazy" />
)}
/>
)
: undefined
}
renderVideo={(p) => <Video {...p} />}
/>
)}
outlined={outlineAttachment}
/>
: undefined
}
renderVideo={(p) => <Video {...p} />}
/>
)}
outlined={outlineAttachment}
/>
{renderedCaption}
</>

);
}

if (msgType === MsgType.Audio) {
return (
<MAudio
content={getContent()}
renderAsFile={renderFile}
renderAudioContent={(props) => (
<AudioContent {...props} renderMediaControl={(p) => <MediaControl {...p} />} />
)}
outlined={outlineAttachment}
/>
<>
<MAudio
content={getContent()}
renderAsFile={renderFile}
renderAudioContent={(props) => (
<AudioContent {...props} renderMediaControl={(p) => <MediaControl {...p} />} />
)}
outlined={outlineAttachment}
/>
{renderedCaption}
</>

);
}

Expand Down
7 changes: 4 additions & 3 deletions src/app/components/message/MsgTypeRenderers.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ export function MNotice({ edited, content, renderBody, renderUrlsPreview }: MNot

type RenderImageContentProps = {
body: string;
filename?: string;
info?: IImageInfo & IThumbnailContent;
mimeType?: string;
url: string;
Expand Down Expand Up @@ -282,7 +283,7 @@ export function MAudio({ content, renderAsFile, renderAudioContent, outlined }:
return (
<Attachment outlined={outlined}>
<AttachmentHeader>
<FileHeader body={content.body ?? 'Audio'} mimeType={safeMimeType} />
<FileHeader body={content.filename ?? content.body ?? 'Audio'} mimeType={safeMimeType} />
</AttachmentHeader>
<AttachmentBox>
<AttachmentContent>
Expand Down Expand Up @@ -322,14 +323,14 @@ export function MFile({ content, renderFileContent, outlined }: MFileProps) {
<Attachment outlined={outlined}>
<AttachmentHeader>
<FileHeader
body={content.body ?? 'Unnamed File'}
body={content.filename ?? content.body ?? 'Unnamed File'}
mimeType={fileInfo?.mimetype ?? FALLBACK_MIMETYPE}
/>
</AttachmentHeader>
<AttachmentBox>
<AttachmentContent>
{renderFileContent({
body: content.body ?? 'File',
body: content.filename ?? content.body ?? 'File',
info: fileInfo ?? {},
mimeType: fileInfo?.mimetype ?? FALLBACK_MIMETYPE,
url: mxcUrl,
Expand Down
13 changes: 8 additions & 5 deletions src/app/features/room/RoomInput.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,17 +227,18 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
const contentsPromises = uploads.map(async (upload) => {
const fileItem = selectedFiles.find((f) => f.file === upload.file);
if (!fileItem) throw new Error('Broken upload');
const caption = toPlainText(editor.children).trim() || undefined;
nexy7574 marked this conversation as resolved.
Show resolved Hide resolved
nexy7574 marked this conversation as resolved.
Show resolved Hide resolved

if (fileItem.file.type.startsWith('image')) {
return getImageMsgContent(mx, fileItem, upload.mxc);
return getImageMsgContent(mx, fileItem, upload.mxc, caption);
}
if (fileItem.file.type.startsWith('video')) {
return getVideoMsgContent(mx, fileItem, upload.mxc);
return getVideoMsgContent(mx, fileItem, upload.mxc, caption);
}
if (fileItem.file.type.startsWith('audio')) {
return getAudioMsgContent(fileItem, upload.mxc);
return getAudioMsgContent(fileItem, upload.mxc, caption);
}
return getFileMsgContent(fileItem, upload.mxc);
return getFileMsgContent(fileItem, upload.mxc, caption);
});
handleCancelUpload(uploads);
const contents = fulfilledPromiseSettledResult(await Promise.allSettled(contentsPromises));
Expand Down Expand Up @@ -318,7 +319,9 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
content['m.relates_to'].is_falling_back = false;
}
}
mx.sendMessage(roomId, content);
if (!uploadBoardHandlers.current) {
mx.sendMessage(roomId, content);
}
nexy7574 marked this conversation as resolved.
Show resolved Hide resolved
resetEditor(editor);
resetEditorHistory(editor);
setReplyDraft(undefined);
Expand Down
21 changes: 13 additions & 8 deletions src/app/features/room/msgContent.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,17 @@ const generateThumbnailContent = async (
export const getImageMsgContent = async (
mx: MatrixClient,
item: TUploadItem,
mxc: string
mxc: string,
caption?: string
): Promise<IContent> => {
const { file, originalFile, encInfo } = item;
const [imgError, imgEl] = await to(loadImageElement(getImageFileUrl(originalFile)));
if (imgError) console.warn(imgError);

const content: IContent = {
msgtype: MsgType.Image,
body: file.name,
filename: file.name,
body: caption || file.name,
};
if (imgEl) {
const blurHash = encodeBlurHash(imgEl, 512, scaleYDimension(imgEl.width, 512, imgEl.height));
Expand All @@ -74,7 +76,8 @@ export const getImageMsgContent = async (
export const getVideoMsgContent = async (
mx: MatrixClient,
item: TUploadItem,
mxc: string
mxc: string,
caption?: string
): Promise<IContent> => {
const { file, originalFile, encInfo } = item;

Expand All @@ -83,7 +86,8 @@ export const getVideoMsgContent = async (

const content: IContent = {
msgtype: MsgType.Video,
body: file.name,
filename: file.name,
body: caption || file.name,
};
if (videoEl) {
const [thumbError, thumbContent] = await to(
Expand Down Expand Up @@ -118,11 +122,12 @@ export const getVideoMsgContent = async (
return content;
};

export const getAudioMsgContent = (item: TUploadItem, mxc: string): IContent => {
export const getAudioMsgContent = (item: TUploadItem, mxc: string, caption?: string): IContent => {
const { file, encInfo } = item;
const content: IContent = {
msgtype: MsgType.Audio,
body: file.name,
filename: file.name,
body: caption || file.name,
info: {
mimetype: file.type,
size: file.size,
Expand All @@ -139,11 +144,11 @@ export const getAudioMsgContent = (item: TUploadItem, mxc: string): IContent =>
return content;
};

export const getFileMsgContent = (item: TUploadItem, mxc: string): IContent => {
export const getFileMsgContent = (item: TUploadItem, mxc: string, caption?: string): IContent => {
const { file, encInfo } = item;
const content: IContent = {
msgtype: MsgType.File,
body: file.name,
body: caption ?? file.name,
filename: file.name,
info: {
mimetype: file.type,
Expand Down
4 changes: 4 additions & 0 deletions src/types/matrix/common.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export type IThumbnailContent = {
export type IImageContent = {
msgtype: MsgType.Image;
body?: string;
filename?: string;
url?: string;
info?: IImageInfo & IThumbnailContent;
file?: IEncryptedFile;
Expand All @@ -51,6 +52,7 @@ export type IImageContent = {
export type IVideoContent = {
msgtype: MsgType.Video;
body?: string;
filename?: string;
url?: string;
info?: IVideoInfo & IThumbnailContent;
file?: IEncryptedFile;
Expand All @@ -59,6 +61,7 @@ export type IVideoContent = {
export type IAudioContent = {
msgtype: MsgType.Audio;
body?: string;
filename?: string;
url?: string;
info?: IAudioInfo;
file?: IEncryptedFile;
Expand All @@ -67,6 +70,7 @@ export type IAudioContent = {
export type IFileContent = {
msgtype: MsgType.File;
body?: string;
filename?: string;
url?: string;
info?: IFileInfo & IThumbnailContent;
file?: IEncryptedFile;
Expand Down