From 7d74dfaae24fe38e25239d267566e5835cb119d1 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 6 Jan 2025 19:08:05 +0100 Subject: [PATCH] alright --- .../lambda/src/api/get-render-progress.ts | 2 + .../lambda/src/api/render-still-on-lambda.ts | 4 +- .../src/functions/aws-implementation.ts | 8 -- .../functions/aws-server-implementation.ts | 7 + .../lambda/src/test/mock-implementation.ts | 12 +- packages/serverless/src/constants.ts | 2 +- .../serverless/src/create-post-render-data.ts | 4 +- packages/serverless/src/handlers/launch.ts | 4 +- packages/serverless/src/handlers/progress.ts | 8 +- packages/serverless/src/handlers/renderer.ts | 2 +- packages/serverless/src/handlers/start.ts | 23 +++- packages/serverless/src/handlers/still.ts | 122 +++++++++--------- packages/serverless/src/index.ts | 4 +- packages/serverless/src/inner-routine.ts | 8 +- packages/serverless/src/inspect-error.ts | 7 +- packages/serverless/src/invoke-webhook.ts | 2 +- packages/serverless/src/make-timeout-error.ts | 10 +- .../serverless/src/make-timeout-message.ts | 12 +- .../serverless/src/most-expensive-chunks.ts | 4 +- .../serverless/src/overall-render-progress.ts | 6 +- packages/serverless/src/plan-frame-ranges.ts | 10 +- packages/serverless/src/progress.ts | 10 +- .../serverless/src/provider-implementation.ts | 2 +- packages/serverless/src/render-progress.ts | 2 +- .../serverless/src/streaming/streaming.ts | 16 +-- .../src/test/best-frames-per-function.test.ts | 2 +- .../src/test/most-expensive.test.ts | 10 +- .../src/test/plan-frame-ranges.test.ts | 6 +- packages/serverless/src/types.ts | 23 ++-- ...bda-error.ts => write-error-to-storage.ts} | 6 +- 30 files changed, 189 insertions(+), 149 deletions(-) rename packages/serverless/src/{write-lambda-error.ts => write-error-to-storage.ts} (89%) diff --git a/packages/lambda/src/api/get-render-progress.ts b/packages/lambda/src/api/get-render-progress.ts index a027e0d60f5..bd5ae0a2176 100644 --- a/packages/lambda/src/api/get-render-progress.ts +++ b/packages/lambda/src/api/get-render-progress.ts @@ -5,6 +5,7 @@ import { awsImplementation, type AwsProvider, } from '../functions/aws-implementation'; +import {serverAwsImplementation} from '../functions/aws-server-implementation'; import {parseFunctionName} from '../functions/helpers/parse-function-name'; import type {AwsRegion} from '../regions'; import type {RenderProgress} from '../shared/constants'; @@ -51,6 +52,7 @@ export const getRenderProgress = async ( memorySizeInMb: parsed.memorySizeInMb, timeoutInMilliseconds: parsed.timeoutInSeconds * 1000, functionName: input.functionName, + serverProviderSpecifics: serverAwsImplementation, }); } diff --git a/packages/lambda/src/api/render-still-on-lambda.ts b/packages/lambda/src/api/render-still-on-lambda.ts index 7f6f77d06d2..6beaf09e9c4 100644 --- a/packages/lambda/src/api/render-still-on-lambda.ts +++ b/packages/lambda/src/api/render-still-on-lambda.ts @@ -11,7 +11,7 @@ import {wrapWithErrorHandling} from '@remotion/renderer/error-handling'; import type { CostsInfo, ReceivedArtifact, - RenderStillLambdaResponsePayload, + RenderStillFunctionResponsePayload, } from '@remotion/serverless'; import type {OutNameInput, Privacy} from '@remotion/serverless/client'; import { @@ -87,7 +87,7 @@ const internalRenderStillOnLambda = async ( try { const payload = await makeLambdaRenderStillPayload(input); const res = await new Promise< - RenderStillLambdaResponsePayload + RenderStillFunctionResponsePayload >((resolve, reject) => { awsImplementation .callFunctionStreaming({ diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index 0b0eb4909df..862f6517e62 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -104,14 +104,6 @@ export const awsImplementation: ProviderSpecifics = { callFunctionAsync: callFunctionAsyncImplementation, callFunctionStreaming: callFunctionWithStreamingImplementation, callFunctionSync: callFunctionSyncImplementation, - getCurrentFunctionName() { - const name = process.env.AWS_LAMBDA_FUNCTION_NAME; - if (!name) { - throw new Error('Expected AWS_LAMBDA_FUNCTION_NAME to be set'); - } - - return name; - }, getEphemeralStorageForPriceCalculation() { // We cannot determine the ephemeral storage size, so we // overestimate the price, but will only have a miniscule effect (~0.2%) diff --git a/packages/lambda/src/functions/aws-server-implementation.ts b/packages/lambda/src/functions/aws-server-implementation.ts index 8f2c8385e1f..6b1a13c060e 100644 --- a/packages/lambda/src/functions/aws-server-implementation.ts +++ b/packages/lambda/src/functions/aws-server-implementation.ts @@ -15,4 +15,11 @@ export const serverAwsImplementation: ServerProviderSpecifics = { return generateRandomHashWithLifeCycleRule({deleteAfter, randomHashFn}); }, deleteTmpDir: () => Promise.resolve(deleteTmpDir()), + getCurrentFunctionName: () => { + if (!process.env.AWS_LAMBDA_FUNCTION_NAME) { + throw new Error('Expected AWS_LAMBDA_FUNCTION_NAME to be set'); + } + + return process.env.AWS_LAMBDA_FUNCTION_NAME; + }, }; diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index 12396093326..908469460ff 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -158,12 +158,6 @@ export const mockImplementation: ProviderSpecifics = { callFunctionAsync: getMockCallFunctionAsync, callFunctionStreaming: getMockCallFunctionStreaming, callFunctionSync: getMockCallFunctionSync, - getCurrentFunctionName: () => - speculateFunctionName({ - diskSizeInMb: 10240, - memorySizeInMb: 3009, - timeoutInSeconds: 120, - }), estimatePrice, getOutputUrl: () => { return { @@ -182,4 +176,10 @@ export const mockServerImplementation: ServerProviderSpecifics = { }), deleteTmpDir: () => Promise.resolve(), generateRandomId: randomHashImplementation, + getCurrentFunctionName: () => + speculateFunctionName({ + diskSizeInMb: 10240, + memorySizeInMb: 3009, + timeoutInSeconds: 120, + }), }; diff --git a/packages/serverless/src/constants.ts b/packages/serverless/src/constants.ts index f7ec168d100..7083f4b8f61 100644 --- a/packages/serverless/src/constants.ts +++ b/packages/serverless/src/constants.ts @@ -14,7 +14,7 @@ import type { import type {BrowserSafeApis} from '@remotion/renderer/client'; import type {ExpensiveChunk} from './most-expensive-chunks'; import type {ChunkRetry, CloudProvider, ReceivedArtifact} from './types'; -import type {EnhancedErrorInfo} from './write-lambda-error'; +import type {EnhancedErrorInfo} from './write-error-to-storage'; // Needs to be in sync with renderer/src/options/delete-after.ts#L7 export const expiryDays = { diff --git a/packages/serverless/src/create-post-render-data.ts b/packages/serverless/src/create-post-render-data.ts index 43f8b270e64..e0695529f83 100644 --- a/packages/serverless/src/create-post-render-data.ts +++ b/packages/serverless/src/create-post-render-data.ts @@ -9,7 +9,7 @@ import type {OverallRenderProgress} from './overall-render-progress'; import type {ProviderSpecifics} from './provider-implementation'; import type {RenderMetadata} from './render-metadata'; import type {CloudProvider} from './types'; -import type {EnhancedErrorInfo} from './write-lambda-error'; +import type {EnhancedErrorInfo} from './write-error-to-storage'; export const createPostRenderData = ({ region, @@ -96,7 +96,7 @@ export const createPostRenderData = ({ ? [] : getMostExpensiveChunks({ parsedTimings, - framesPerLambda: renderMetadata.framesPerLambda, + framesPerFunction: renderMetadata.framesPerLambda, firstFrame: renderMetadata.frameRange[0], lastFrame: renderMetadata.frameRange[1], }), diff --git a/packages/serverless/src/handlers/launch.ts b/packages/serverless/src/handlers/launch.ts index 41c696294bd..3d1fbfc33d2 100644 --- a/packages/serverless/src/handlers/launch.ts +++ b/packages/serverless/src/handlers/launch.ts @@ -47,7 +47,7 @@ import {validateComposition} from '../validate-composition'; import {validateFramesPerFunction} from '../validate-frames-per-function'; import {validateOutname} from '../validate-outname'; import {validatePrivacy} from '../validate-privacy'; -import {getTmpDirStateIfENoSp} from '../write-lambda-error'; +import {getTmpDirStateIfENoSp} from '../write-error-to-storage'; type Options = { expectedBucketOwner: string; @@ -188,7 +188,7 @@ const innerLaunchHandler = async ({ RenderInternals.validatePuppeteerTimeout(params.timeoutInMilliseconds); const {chunks} = planFrameRanges({ - framesPerLambda, + framesPerFunction: framesPerLambda, frameRange: realFrameRange, everyNthFrame: params.everyNthFrame, }); diff --git a/packages/serverless/src/handlers/progress.ts b/packages/serverless/src/handlers/progress.ts index 563ddcb3ba2..a66175c3bb1 100644 --- a/packages/serverless/src/handlers/progress.ts +++ b/packages/serverless/src/handlers/progress.ts @@ -2,7 +2,10 @@ import {VERSION} from 'remotion/version'; import type {ServerlessPayload} from '../constants'; import {ServerlessRoutines} from '../constants'; import {getProgress} from '../progress'; -import type {ProviderSpecifics} from '../provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from '../provider-implementation'; import type {GenericRenderProgress} from '../render-progress'; import type {CloudProvider} from '../types'; @@ -11,6 +14,7 @@ type Options = { timeoutInMilliseconds: number; retriesRemaining: number; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }; export const progressHandler = async ( @@ -45,6 +49,7 @@ export const progressHandler = async ( providerSpecifics: options.providerSpecifics, forcePathStyle: lambdaParams.forcePathStyle, functionName: process.env.AWS_LAMBDA_FUNCTION_NAME as string, + serverProviderSpecifics: options.serverProviderSpecifics, }); return progress; } catch (err) { @@ -61,6 +66,7 @@ export const progressHandler = async ( timeoutInMilliseconds: options.timeoutInMilliseconds, retriesRemaining: options.retriesRemaining - 1, providerSpecifics: options.providerSpecifics, + serverProviderSpecifics: options.serverProviderSpecifics, }); } diff --git a/packages/serverless/src/handlers/renderer.ts b/packages/serverless/src/handlers/renderer.ts index 792a7a398e7..bb10afd45e9 100644 --- a/packages/serverless/src/handlers/renderer.ts +++ b/packages/serverless/src/handlers/renderer.ts @@ -28,7 +28,7 @@ import type {OnStream} from '../streaming/streaming'; import {truthy} from '../truthy'; import type {CloudProvider, ObjectChunkTimingData} from '../types'; import {enableNodeIntrospection} from '../why-is-node-running'; -import {getTmpDirStateIfENoSp} from '../write-lambda-error'; +import {getTmpDirStateIfENoSp} from '../write-error-to-storage'; type Options = { expectedBucketOwner: string; diff --git a/packages/serverless/src/handlers/start.ts b/packages/serverless/src/handlers/start.ts index 7fc3726f331..0b94c6d7dd4 100644 --- a/packages/serverless/src/handlers/start.ts +++ b/packages/serverless/src/handlers/start.ts @@ -3,7 +3,10 @@ import type {ServerlessPayload} from '../constants'; import {ServerlessRoutines, overallProgressKey} from '../constants'; import {internalGetOrCreateBucket} from '../get-or-create-bucket'; import {makeInitialOverallRenderProgress} from '../overall-render-progress'; -import type {ProviderSpecifics} from '../provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from '../provider-implementation'; import type {CloudProvider} from '../types'; type Options = { @@ -12,11 +15,17 @@ type Options = { renderId: string; }; -export const startHandler = async ( - params: ServerlessPayload, - options: Options, - providerSpecifics: ProviderSpecifics, -) => { +export const startHandler = async ({ + params, + options, + providerSpecifics, + serverProviderSpecifics, +}: { + params: ServerlessPayload; + options: Options; + providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; +}) => { if (params.type !== ServerlessRoutines.start) { throw new TypeError('Expected type start'); } @@ -118,7 +127,7 @@ export const startHandler = async ( }; await providerSpecifics.callFunctionAsync({ - functionName: providerSpecifics.getCurrentFunctionName(), + functionName: serverProviderSpecifics.getCurrentFunctionName(), type: ServerlessRoutines.launch, payload, region, diff --git a/packages/serverless/src/handlers/still.ts b/packages/serverless/src/handlers/still.ts index 3733f98ff43..4151d6827f3 100644 --- a/packages/serverless/src/handlers/still.ts +++ b/packages/serverless/src/handlers/still.ts @@ -30,13 +30,13 @@ import type {OnStream} from '../streaming/streaming'; import type { CloudProvider, ReceivedArtifact, - RenderStillLambdaResponsePayload, + RenderStillFunctionResponsePayload, } from '../types'; import {validateComposition} from '../validate-composition'; import {validateDownloadBehavior} from '../validate-download-behavior'; import {validateOutname} from '../validate-outname'; import {validatePrivacy} from '../validate-privacy'; -import {getTmpDirStateIfENoSp} from '../write-lambda-error'; +import {getTmpDirStateIfENoSp} from '../write-error-to-storage'; type Options = { params: ServerlessPayload; @@ -50,7 +50,7 @@ type Options = { const innerStillHandler = async ( { - params: lambdaParams, + params, expectedBucketOwner, renderId, onStream, @@ -60,26 +60,26 @@ const innerStillHandler = async ( }: Options, cleanup: CleanupFn[], ) => { - if (lambdaParams.type !== ServerlessRoutines.still) { + if (params.type !== ServerlessRoutines.still) { throw new TypeError('Expected still type'); } - if (lambdaParams.version !== VERSION) { - if (!lambdaParams.version) { + if (params.version !== VERSION) { + if (!params.version) { throw new Error( `Version mismatch: When calling renderStillOnLambda(), you called the function ${process.env.AWS_LAMBDA_FUNCTION_NAME} which has the version ${VERSION} but the @remotion/lambda package is an older version. Deploy a new function and use it to call renderStillOnLambda(). See: https://www.remotion.dev/docs/lambda/upgrading`, ); } throw new Error( - `Version mismatch: When calling renderStillOnLambda(), you passed ${process.env.AWS_LAMBDA_FUNCTION_NAME} as the function, which has the version ${VERSION}, but the @remotion/lambda package you used to invoke the function has version ${lambdaParams.version}. Deploy a new function and use it to call renderStillOnLambda(). See: https://www.remotion.dev/docs/lambda/upgrading`, + `Version mismatch: When calling renderStillOnLambda(), you passed ${process.env.AWS_LAMBDA_FUNCTION_NAME} as the function, which has the version ${VERSION}, but the @remotion/lambda package you used to invoke the function has version ${params.version}. Deploy a new function and use it to call renderStillOnLambda(). See: https://www.remotion.dev/docs/lambda/upgrading`, ); } - validateDownloadBehavior(lambdaParams.downloadBehavior); - validatePrivacy(lambdaParams.privacy, true); + validateDownloadBehavior(params.downloadBehavior); + validatePrivacy(params.privacy, true); validateOutname({ - outName: lambdaParams.outName, + outName: params.outName, codec: null, audioCodecSetting: null, separateAudioTo: null, @@ -88,20 +88,20 @@ const innerStillHandler = async ( const start = Date.now(); const browserInstancePromise = serverProviderSpecifics.getBrowserInstance({ - logLevel: lambdaParams.logLevel, + logLevel: params.logLevel, indent: false, - chromiumOptions: lambdaParams.chromiumOptions, + chromiumOptions: params.chromiumOptions, providerSpecifics, serverProviderSpecifics, }); const bucketNamePromise = - lambdaParams.bucketName ?? + params.bucketName ?? internalGetOrCreateBucket({ region: providerSpecifics.getCurrentRegionInFunction(), enableFolderExpiry: null, customCredentials: null, providerSpecifics, - forcePathStyle: lambdaParams.forcePathStyle, + forcePathStyle: params.forcePathStyle, skipPutAcl: false, }).then((b) => b.bucketName); @@ -115,14 +115,14 @@ const innerStillHandler = async ( bucketName, expectedBucketOwner, region, - serialized: lambdaParams.inputProps, + serialized: params.inputProps, propsType: 'input-props', providerSpecifics, - forcePathStyle: lambdaParams.forcePathStyle, + forcePathStyle: params.forcePathStyle, }); const serveUrl = providerSpecifics.convertToServeUrl({ - urlOrId: lambdaParams.serveUrl, + urlOrId: params.serveUrl, region, bucketName, }); @@ -134,10 +134,9 @@ const innerStillHandler = async ( indent: false, port: null, remotionRoot: process.cwd(), - logLevel: lambdaParams.logLevel, + logLevel: params.logLevel, webpackConfigOrServeUrl: serveUrl, - offthreadVideoCacheSizeInBytes: - lambdaParams.offthreadVideoCacheSizeInBytes, + offthreadVideoCacheSizeInBytes: params.offthreadVideoCacheSizeInBytes, binariesDirectory: null, forceIPv4: false, }, @@ -152,17 +151,17 @@ const innerStillHandler = async ( const composition = await validateComposition({ serveUrl, browserInstance: browserInstance.instance, - composition: lambdaParams.composition, + composition: params.composition, serializedInputPropsWithCustomSchema, - envVariables: lambdaParams.envVariables ?? {}, - chromiumOptions: lambdaParams.chromiumOptions, - timeoutInMilliseconds: lambdaParams.timeoutInMilliseconds, + envVariables: params.envVariables ?? {}, + chromiumOptions: params.chromiumOptions, + timeoutInMilliseconds: params.timeoutInMilliseconds, port: null, - forceHeight: lambdaParams.forceHeight, - forceWidth: lambdaParams.forceWidth, - logLevel: lambdaParams.logLevel, + forceHeight: params.forceHeight, + forceWidth: params.forceWidth, + logLevel: params.logLevel, server, - offthreadVideoCacheSizeInBytes: lambdaParams.offthreadVideoCacheSizeInBytes, + offthreadVideoCacheSizeInBytes: params.offthreadVideoCacheSizeInBytes, onBrowserDownload: () => { throw new Error('Should not download a browser in Lambda'); }, @@ -173,31 +172,31 @@ const innerStillHandler = async ( const renderMetadata: RenderMetadata = { startedDate: Date.now(), codec: null, - compositionId: lambdaParams.composition, + compositionId: params.composition, estimatedTotalLambdaInvokations: 1, estimatedRenderLambdaInvokations: 1, siteId: serveUrl, totalChunks: 1, type: 'still', - imageFormat: lambdaParams.imageFormat, - inputProps: lambdaParams.inputProps, + imageFormat: params.imageFormat, + inputProps: params.inputProps, lambdaVersion: VERSION, framesPerLambda: 1, memorySizeInMb: Number(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE), region: providerSpecifics.getCurrentRegionInFunction(), renderId, - outName: lambdaParams.outName ?? undefined, - privacy: lambdaParams.privacy, + outName: params.outName ?? undefined, + privacy: params.privacy, audioCodec: null, - deleteAfter: lambdaParams.deleteAfter, + deleteAfter: params.deleteAfter, numberOfGifLoops: null, - downloadBehavior: lambdaParams.downloadBehavior, + downloadBehavior: params.downloadBehavior, audioBitrate: null, metadata: null, functionName: process.env.AWS_LAMBDA_FUNCTION_NAME as string, dimensions: { - height: composition.height * (lambdaParams.scale ?? 1), - width: composition.width * (lambdaParams.scale ?? 1), + height: composition.height * (params.scale ?? 1), + width: composition.width * (params.scale ?? 1), }, }; @@ -213,7 +212,7 @@ const innerStillHandler = async ( expectedBucketOwner, downloadBehavior: null, customCredentials: null, - forcePathStyle: lambdaParams.forcePathStyle, + forcePathStyle: params.forcePathStyle, }); const onBrowserDownload = () => { @@ -225,7 +224,7 @@ const innerStillHandler = async ( const {key, renderBucketName, customCredentials} = getExpectedOutName( renderMetadata, bucketName, - getCredentialsFromOutName(lambdaParams.outName), + getCredentialsFromOutName(params.outName), ); const onArtifact = (artifact: EmittedArtifact): {alreadyExisted: boolean} => { @@ -246,7 +245,7 @@ const innerStillHandler = async ( const startTime = Date.now(); RenderInternals.Log.info( - {indent: false, logLevel: lambdaParams.logLevel}, + {indent: false, logLevel: params.logLevel}, 'Writing artifact ' + artifact.filename + ' to S3', ); providerSpecifics @@ -255,21 +254,21 @@ const innerStillHandler = async ( key: storageKey, body: artifact.content, region, - privacy: lambdaParams.privacy, + privacy: params.privacy, expectedBucketOwner, - downloadBehavior: lambdaParams.downloadBehavior, + downloadBehavior: params.downloadBehavior, customCredentials, - forcePathStyle: lambdaParams.forcePathStyle, + forcePathStyle: params.forcePathStyle, }) .then(() => { RenderInternals.Log.info( - {indent: false, logLevel: lambdaParams.logLevel}, + {indent: false, logLevel: params.logLevel}, `Wrote artifact to S3 in ${Date.now() - startTime}ms`, ); }) .catch((err) => { RenderInternals.Log.error( - {indent: false, logLevel: lambdaParams.logLevel}, + {indent: false, logLevel: params.logLevel}, 'Failed to write artifact to S3', err, ); @@ -281,35 +280,34 @@ const innerStillHandler = async ( composition, output: outputPath, serveUrl, - envVariables: lambdaParams.envVariables ?? {}, + envVariables: params.envVariables ?? {}, frame: RenderInternals.convertToPositiveFrameIndex({ - frame: lambdaParams.frame, + frame: params.frame, durationInFrames: composition.durationInFrames, }), - imageFormat: lambdaParams.imageFormat as StillImageFormat, + imageFormat: params.imageFormat as StillImageFormat, serializedInputPropsWithCustomSchema, overwrite: false, puppeteerInstance: browserInstance.instance, - jpegQuality: - lambdaParams.jpegQuality ?? RenderInternals.DEFAULT_JPEG_QUALITY, - chromiumOptions: lambdaParams.chromiumOptions, - scale: lambdaParams.scale, - timeoutInMilliseconds: lambdaParams.timeoutInMilliseconds, + jpegQuality: params.jpegQuality ?? RenderInternals.DEFAULT_JPEG_QUALITY, + chromiumOptions: params.chromiumOptions, + scale: params.scale, + timeoutInMilliseconds: params.timeoutInMilliseconds, browserExecutable: providerSpecifics.getChromiumPath(), cancelSignal: null, indent: false, onBrowserLog: null, - onDownload: onDownloadsHelper(lambdaParams.logLevel), + onDownload: onDownloadsHelper(params.logLevel), port: null, server, - logLevel: lambdaParams.logLevel, + logLevel: params.logLevel, serializedResolvedPropsWithCustomSchema: NoReactInternals.serializeJSONWithDate({ indent: undefined, staticBase: null, data: composition.props, }).serializedString, - offthreadVideoCacheSizeInBytes: lambdaParams.offthreadVideoCacheSizeInBytes, + offthreadVideoCacheSizeInBytes: params.offthreadVideoCacheSizeInBytes, binariesDirectory: null, onBrowserDownload, onArtifact, @@ -320,22 +318,22 @@ const innerStillHandler = async ( await providerSpecifics.writeFile({ bucketName: renderBucketName, key, - privacy: lambdaParams.privacy, + privacy: params.privacy, body: fs.createReadStream(outputPath), expectedBucketOwner, region: providerSpecifics.getCurrentRegionInFunction(), - downloadBehavior: lambdaParams.downloadBehavior, + downloadBehavior: params.downloadBehavior, customCredentials, - forcePathStyle: lambdaParams.forcePathStyle, + forcePathStyle: params.forcePathStyle, }); await Promise.all([ fs.promises.rm(outputPath, {recursive: true}), cleanupSerializedInputProps({ region: providerSpecifics.getCurrentRegionInFunction(), - serialized: lambdaParams.inputProps, + serialized: params.inputProps, providerSpecifics, - forcePathStyle: lambdaParams.forcePathStyle, + forcePathStyle: params.forcePathStyle, }), server.closeServer(true), ]); @@ -355,7 +353,7 @@ const innerStillHandler = async ( currentRegion: providerSpecifics.getCurrentRegionInFunction(), }); - const payload: RenderStillLambdaResponsePayload = { + const payload: RenderStillFunctionResponsePayload = { type: 'success' as const, output: url, size, diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index f639a7c1726..216c355b9f8 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -48,6 +48,6 @@ export { } from './validate-webhook'; export { EnhancedErrorInfo, - LambdaErrorInfo, + FunctionErrorInfo as LambdaErrorInfo, getTmpDirStateIfENoSp, -} from './write-lambda-error'; +} from './write-error-to-storage'; diff --git a/packages/serverless/src/inner-routine.ts b/packages/serverless/src/inner-routine.ts index 272e53a2dff..241cae718b5 100644 --- a/packages/serverless/src/inner-routine.ts +++ b/packages/serverless/src/inner-routine.ts @@ -157,15 +157,16 @@ export const innerHandler = async ({ ); } - const response = await startHandler( + const response = await startHandler({ params, - { + options: { expectedBucketOwner: currentUserId, timeoutInMilliseconds, renderId, }, providerSpecifics, - ); + serverProviderSpecifics, + }); await responseWriter.write(Buffer.from(JSON.stringify(response))); await responseWriter.end(); @@ -218,6 +219,7 @@ export const innerHandler = async ({ timeoutInMilliseconds, retriesRemaining: 2, providerSpecifics, + serverProviderSpecifics, }); await responseWriter.write(Buffer.from(JSON.stringify(response))); diff --git a/packages/serverless/src/inspect-error.ts b/packages/serverless/src/inspect-error.ts index 1bbd4b9e930..a3d2947163b 100644 --- a/packages/serverless/src/inspect-error.ts +++ b/packages/serverless/src/inspect-error.ts @@ -4,7 +4,10 @@ import { isBrowserCrashedError, isErrInsufficientResourcesErr, } from './error-category'; -import type {EnhancedErrorInfo, LambdaErrorInfo} from './write-lambda-error'; +import type { + EnhancedErrorInfo, + FunctionErrorInfo, +} from './write-error-to-storage'; const FAILED_TO_LAUNCH_TOKEN = 'Failed to launch browser.'; @@ -45,7 +48,7 @@ const getExplanation = (stack: string) => { export const inspectErrors = ({ errors, }: { - errors: LambdaErrorInfo[]; + errors: FunctionErrorInfo[]; }): EnhancedErrorInfo[] => { return errors.map((e): EnhancedErrorInfo => { return { diff --git a/packages/serverless/src/invoke-webhook.ts b/packages/serverless/src/invoke-webhook.ts index 52d9480314e..9fd032320f6 100644 --- a/packages/serverless/src/invoke-webhook.ts +++ b/packages/serverless/src/invoke-webhook.ts @@ -4,7 +4,7 @@ import type https from 'https'; import * as Crypto from 'node:crypto'; import type http from 'node:http'; import type {AfterRenderCost} from './constants'; -import type {EnhancedErrorInfo} from './write-lambda-error'; +import type {EnhancedErrorInfo} from './write-error-to-storage'; export function calculateSignature(payload: string, secret: string | null) { if (!secret) { diff --git a/packages/serverless/src/make-timeout-error.ts b/packages/serverless/src/make-timeout-error.ts index 24ef43655ce..e1138dbb1a6 100644 --- a/packages/serverless/src/make-timeout-error.ts +++ b/packages/serverless/src/make-timeout-error.ts @@ -1,8 +1,11 @@ import {makeTimeoutMessage} from './make-timeout-message'; -import type {ProviderSpecifics} from './provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from './provider-implementation'; import type {RenderMetadata} from './render-metadata'; import type {CloudProvider} from './types'; -import type {EnhancedErrorInfo} from './write-lambda-error'; +import type {EnhancedErrorInfo} from './write-error-to-storage'; export const makeTimeoutError = ({ timeoutInMilliseconds, @@ -12,6 +15,7 @@ export const makeTimeoutError = ({ functionName, region, providerSpecifics, + serverProviderSpecifics, }: { timeoutInMilliseconds: number; renderMetadata: RenderMetadata; @@ -20,6 +24,7 @@ export const makeTimeoutError = ({ functionName: string; region: Provider['region']; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }): EnhancedErrorInfo => { const message = makeTimeoutMessage({ missingChunks, @@ -29,6 +34,7 @@ export const makeTimeoutError = ({ functionName, region, providerSpecifics, + serverProviderSpecifics, }); const error = new Error(message); diff --git a/packages/serverless/src/make-timeout-message.ts b/packages/serverless/src/make-timeout-message.ts index 71846b67448..85aa2d9b7e2 100644 --- a/packages/serverless/src/make-timeout-message.ts +++ b/packages/serverless/src/make-timeout-message.ts @@ -1,6 +1,9 @@ import {ServerlessRoutines} from './constants'; import {DOCS_URL} from './docs-url'; -import type {ProviderSpecifics} from './provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from './provider-implementation'; import type {RenderMetadata} from './render-metadata'; import type {CloudProvider} from './types'; @@ -11,11 +14,13 @@ const makeChunkMissingMessage = ({ renderMetadata, region, providerSpecifics, + serverProviderSpecifics, }: { missingChunks: number[]; renderMetadata: RenderMetadata; region: Provider['region']; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }) => { if (missingChunks.length === 0) { return 'All chunks have been successfully rendered, but the main function has timed out.'; @@ -43,7 +48,7 @@ const makeChunkMissingMessage = ({ msg, `▸ Logs for chunk ${ch}: ${providerSpecifics.getLoggingUrlForRendererFunction( { - functionName: providerSpecifics.getCurrentFunctionName(), + functionName: serverProviderSpecifics.getCurrentFunctionName(), region, rendererFunctionName: null, renderId: renderMetadata.renderId, @@ -64,6 +69,7 @@ export const makeTimeoutMessage = ({ functionName, region, providerSpecifics, + serverProviderSpecifics, }: { timeoutInMilliseconds: number; missingChunks: number[]; @@ -72,6 +78,7 @@ export const makeTimeoutMessage = ({ region: Provider['region']; functionName: string; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }) => { const cloudWatchRendererUrl = providerSpecifics.getLoggingUrlForRendererFunction({ @@ -96,6 +103,7 @@ export const makeTimeoutMessage = ({ renderMetadata, region, providerSpecifics, + serverProviderSpecifics, }), '', `Consider increasing the timeout of your function.`, diff --git a/packages/serverless/src/most-expensive-chunks.ts b/packages/serverless/src/most-expensive-chunks.ts index 412425fd8ab..a3045852530 100644 --- a/packages/serverless/src/most-expensive-chunks.ts +++ b/packages/serverless/src/most-expensive-chunks.ts @@ -10,12 +10,12 @@ export type ExpensiveChunk = { export const getMostExpensiveChunks = ({ parsedTimings, - framesPerLambda, + framesPerFunction: framesPerLambda, firstFrame, lastFrame, }: { parsedTimings: ParsedTiming[]; - framesPerLambda: number; + framesPerFunction: number; firstFrame: number; lastFrame: number; }): ExpensiveChunk[] => { diff --git a/packages/serverless/src/overall-render-progress.ts b/packages/serverless/src/overall-render-progress.ts index 3ca94d2e17f..b6184955de8 100644 --- a/packages/serverless/src/overall-render-progress.ts +++ b/packages/serverless/src/overall-render-progress.ts @@ -10,7 +10,7 @@ import type { ParsedTiming, ReceivedArtifact, } from './types'; -import type {LambdaErrorInfo} from './write-lambda-error'; +import type {FunctionErrorInfo} from './write-error-to-storage'; export type OverallRenderProgress = { chunks: number[]; @@ -25,7 +25,7 @@ export type OverallRenderProgress = { postRenderData: PostRenderData | null; timings: ParsedTiming[]; renderMetadata: RenderMetadata | null; - errors: LambdaErrorInfo[]; + errors: FunctionErrorInfo[]; timeoutTimestamp: number; functionLaunched: number; serveUrlOpened: number | null; @@ -55,7 +55,7 @@ export type OverallProgressHelper = { addRetry: (retry: ChunkRetry) => void; setPostRenderData: (postRenderData: PostRenderData) => void; setRenderMetadata: (renderMetadata: RenderMetadata) => void; - addErrorWithoutUpload: (errorInfo: LambdaErrorInfo) => void; + addErrorWithoutUpload: (errorInfo: FunctionErrorInfo) => void; setExpectedChunks: (expectedChunks: number) => void; get: () => OverallRenderProgress; setServeUrlOpened: (timestamp: number) => void; diff --git a/packages/serverless/src/plan-frame-ranges.ts b/packages/serverless/src/plan-frame-ranges.ts index 75492f48b5c..4f2a91b6c97 100644 --- a/packages/serverless/src/plan-frame-ranges.ts +++ b/packages/serverless/src/plan-frame-ranges.ts @@ -1,11 +1,11 @@ import {RenderInternals} from '@remotion/renderer'; export const planFrameRanges = ({ - framesPerLambda, + framesPerFunction, frameRange, everyNthFrame, }: { - framesPerLambda: number; + framesPerFunction: number; frameRange: [number, number]; everyNthFrame: number; }): {chunks: [number, number][]} => { @@ -13,15 +13,15 @@ export const planFrameRanges = ({ frameRange, everyNthFrame, ); - const chunkCount = Math.ceil(framesToRender.length / framesPerLambda); + const chunkCount = Math.ceil(framesToRender.length / framesPerFunction); const firstFrame = frameRange[0]; return { chunks: new Array(chunkCount).fill(1).map((_, i) => { - const start = i * framesPerLambda * everyNthFrame + firstFrame; + const start = i * framesPerFunction * everyNthFrame + firstFrame; const end = Math.min( framesToRender[framesToRender.length - 1], - (i + 1) * framesPerLambda * everyNthFrame - 1 + firstFrame, + (i + 1) * framesPerFunction * everyNthFrame - 1 + firstFrame, ); return [start, end]; diff --git a/packages/serverless/src/progress.ts b/packages/serverless/src/progress.ts index 1e15de906f7..ae78533e305 100644 --- a/packages/serverless/src/progress.ts +++ b/packages/serverless/src/progress.ts @@ -10,12 +10,15 @@ import {getOverallProgress} from './get-overall-progress'; import {getOverallProgressFromStorage} from './get-overall-progress-from-storage'; import {inspectErrors} from './inspect-error'; import {makeTimeoutError} from './make-timeout-error'; -import type {ProviderSpecifics} from './provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from './provider-implementation'; import {lambdaRenderHasAudioVideo} from './render-has-audio-video'; import type {CleanupInfo, GenericRenderProgress} from './render-progress'; import {truthy} from './truthy'; import type {CloudProvider} from './types'; -import type {EnhancedErrorInfo} from './write-lambda-error'; +import type {EnhancedErrorInfo} from './write-error-to-storage'; export const getProgress = async ({ bucketName, @@ -28,6 +31,7 @@ export const getProgress = async ({ providerSpecifics, forcePathStyle, functionName, + serverProviderSpecifics, }: { bucketName: string; renderId: string; @@ -37,6 +41,7 @@ export const getProgress = async ({ timeoutInMilliseconds: number; customCredentials: CustomCredentials | null; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; forcePathStyle: boolean; functionName: string; }): Promise> => { @@ -255,6 +260,7 @@ export const getProgress = async ({ region, functionName, providerSpecifics, + serverProviderSpecifics, }) : null, ...errorExplanations, diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index bba9b5d5615..856d73a1960 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -239,6 +239,7 @@ export type ServerProviderSpecifics = { timer: DebuggingTimer; generateRandomId: GenerateRenderId; deleteTmpDir: () => Promise; + getCurrentFunctionName: () => string; }; export type ProviderSpecifics = { @@ -262,7 +263,6 @@ export type ProviderSpecifics = { callFunctionAsync: CallFunctionAsync; callFunctionStreaming: CallFunctionStreaming; callFunctionSync: CallFunctionSync; - getCurrentFunctionName: () => string; estimatePrice: EstimatePrice; getLoggingUrlForRendererFunction: GetLoggingUrlForRendererFunction; getLoggingUrlForMethod: GetLoggingUrlForMethod; diff --git a/packages/serverless/src/render-progress.ts b/packages/serverless/src/render-progress.ts index 08853097bde..566881b5f98 100644 --- a/packages/serverless/src/render-progress.ts +++ b/packages/serverless/src/render-progress.ts @@ -6,7 +6,7 @@ import type { CostsInfo, ReceivedArtifact, } from './types'; -import type {EnhancedErrorInfo} from './write-lambda-error'; +import type {EnhancedErrorInfo} from './write-error-to-storage'; export type CleanupInfo = { doneIn: number | null; diff --git a/packages/serverless/src/streaming/streaming.ts b/packages/serverless/src/streaming/streaming.ts index bc0ea5ee04d..5cb15363968 100644 --- a/packages/serverless/src/streaming/streaming.ts +++ b/packages/serverless/src/streaming/streaming.ts @@ -1,7 +1,7 @@ import {makeStreamPayloadMessage} from '@remotion/streaming'; import type {SerializedArtifact} from '../serialize-artifact'; -import type {CloudProvider, RenderStillLambdaResponsePayload} from '../types'; -import type {LambdaErrorInfo} from '../write-lambda-error'; +import type {CloudProvider, RenderStillFunctionResponsePayload} from '../types'; +import type {FunctionErrorInfo} from '../write-error-to-storage'; const framesRendered = 'frames-rendered' as const; const errorOccurred = 'error-occurred' as const; @@ -10,7 +10,7 @@ const videoChunkRendered = 'video-chunk-rendered' as const; const audioChunkRendered = 'audio-chunk-rendered' as const; const chunkComplete = 'chunk-complete' as const; const stillRendered = 'still-rendered' as const; -const lambdaInvoked = 'lambda-invoked' as const; +const functionInvoked = 'lambda-invoked' as const; const artifactEmitted = 'artifact-emitted' as const; const messageTypes = { @@ -21,7 +21,7 @@ const messageTypes = { '5': {type: audioChunkRendered}, '6': {type: stillRendered}, '7': {type: chunkComplete}, - '8': {type: lambdaInvoked}, + '8': {type: functionInvoked}, '9': {type: artifactEmitted}, } as const; @@ -36,7 +36,7 @@ export const formatMap: {[key in MessageType]: 'json' | 'binary'} = { [audioChunkRendered]: 'binary', [stillRendered]: 'json', [chunkComplete]: 'json', - [lambdaInvoked]: 'json', + [functionInvoked]: 'json', [artifactEmitted]: 'json', }; @@ -61,7 +61,7 @@ export type StreamingPayload = payload: { error: string; shouldRetry: boolean; - errorInfo: LambdaErrorInfo; + errorInfo: FunctionErrorInfo; }; } | { @@ -72,7 +72,7 @@ export type StreamingPayload = } | { type: typeof stillRendered; - payload: RenderStillLambdaResponsePayload; + payload: RenderStillFunctionResponsePayload; } | { type: typeof chunkComplete; @@ -82,7 +82,7 @@ export type StreamingPayload = }; } | { - type: typeof lambdaInvoked; + type: typeof functionInvoked; payload: { attempt: number; }; diff --git a/packages/serverless/src/test/best-frames-per-function.test.ts b/packages/serverless/src/test/best-frames-per-function.test.ts index 60d96b23d10..77a2fa76707 100644 --- a/packages/serverless/src/test/best-frames-per-function.test.ts +++ b/packages/serverless/src/test/best-frames-per-function.test.ts @@ -1,7 +1,7 @@ import {expect, test} from 'bun:test'; import {bestFramesPerFunctionParam} from '../best-frames-per-function-param'; -test('Get reasonable framesPerLambda defaults', () => { +test('Get reasonable framesPerFunction defaults', () => { expect(bestFramesPerFunctionParam(20)).toEqual(20); expect(bestFramesPerFunctionParam(21)).toEqual(11); expect(bestFramesPerFunctionParam(100)).toEqual(20); diff --git a/packages/serverless/src/test/most-expensive.test.ts b/packages/serverless/src/test/most-expensive.test.ts index bf3ef7b3397..e07d583bf18 100644 --- a/packages/serverless/src/test/most-expensive.test.ts +++ b/packages/serverless/src/test/most-expensive.test.ts @@ -40,7 +40,7 @@ test('Should calculate most expensive chunks', () => { start: 2000, }, ], - framesPerLambda: 10, + framesPerFunction: 10, firstFrame: 0, lastFrame: 99, }); @@ -55,7 +55,7 @@ test('Should calculate most expensive chunks', () => { }); test('Render starting from frame 10 should have correct offset', () => { - const framesPerLambda = 10; + const framesPerFunction = 10; const firstFrame = 10; const lastFrame = 99; const most = getMostExpensiveChunks({ @@ -96,7 +96,7 @@ test('Render starting from frame 10 should have correct offset', () => { start: 2000, }, ], - framesPerLambda, + framesPerFunction, firstFrame, lastFrame, }); @@ -111,7 +111,7 @@ test('Render starting from frame 10 should have correct offset', () => { }); test('Render starting from frame 10 and last chunk in most expensive should be corret', () => { - const framesPerLambda = 10; + const framesPerFunction = 10; const firstFrame = 10; const lastFrame = 79; const most = getMostExpensiveChunks({ @@ -152,7 +152,7 @@ test('Render starting from frame 10 and last chunk in most expensive should be c start: 2000, }, ], - framesPerLambda, + framesPerFunction, firstFrame, lastFrame, }); diff --git a/packages/serverless/src/test/plan-frame-ranges.test.ts b/packages/serverless/src/test/plan-frame-ranges.test.ts index 831c96cbc07..0a93e372266 100644 --- a/packages/serverless/src/test/plan-frame-ranges.test.ts +++ b/packages/serverless/src/test/plan-frame-ranges.test.ts @@ -3,7 +3,7 @@ import {planFrameRanges} from '../plan-frame-ranges'; test('Plan frame ranges should respect everyNthFrame', () => { const planned = planFrameRanges({ - framesPerLambda: 8, + framesPerFunction: 8, everyNthFrame: 2, frameRange: [0, 99], }); @@ -20,7 +20,7 @@ test('Plan frame ranges should respect everyNthFrame', () => { test('Should remove ranges that are not going to render', () => { const planned = planFrameRanges({ - framesPerLambda: 11, + framesPerFunction: 11, everyNthFrame: 1, frameRange: [0, 22], }); @@ -33,7 +33,7 @@ test('Should remove ranges that are not going to render', () => { test('Should not have a bug that was reported', () => { const planned = planFrameRanges({ - framesPerLambda: 138, + framesPerFunction: 138, everyNthFrame: 1, frameRange: [15000, 35559], }); diff --git a/packages/serverless/src/types.ts b/packages/serverless/src/types.ts index eb32d098e14..7d878baa79d 100644 --- a/packages/serverless/src/types.ts +++ b/packages/serverless/src/types.ts @@ -26,17 +26,18 @@ export type CostsInfo = { disclaimer: string; }; -export type RenderStillLambdaResponsePayload = { - type: 'success'; - output: string; - outKey: string; - size: number; - bucketName: string; - sizeInBytes: number; - estimatedPrice: CostsInfo; - renderId: string; - receivedArtifacts: ReceivedArtifact[]; -}; +export type RenderStillFunctionResponsePayload = + { + type: 'success'; + output: string; + outKey: string; + size: number; + bucketName: string; + sizeInBytes: number; + estimatedPrice: CostsInfo; + renderId: string; + receivedArtifacts: ReceivedArtifact[]; + }; export type ChunkRetry = { chunk: number; diff --git a/packages/serverless/src/write-lambda-error.ts b/packages/serverless/src/write-error-to-storage.ts similarity index 89% rename from packages/serverless/src/write-lambda-error.ts rename to packages/serverless/src/write-error-to-storage.ts index 0e39fb67415..ee64c102c78 100644 --- a/packages/serverless/src/write-lambda-error.ts +++ b/packages/serverless/src/write-error-to-storage.ts @@ -3,7 +3,7 @@ import type {FileNameAndSize} from './get-files-in-folder'; import type {ProviderSpecifics} from './provider-implementation'; import type {CloudProvider} from './types'; -export type LambdaErrorInfo = { +export type FunctionErrorInfo = { type: 'renderer' | 'browser' | 'stitcher' | 'webhook' | 'artifact'; message: string; name: string; @@ -20,7 +20,7 @@ export type LambdaErrorInfo = { export const getTmpDirStateIfENoSp = ( err: string, providerSpecifics: ProviderSpecifics, -): LambdaErrorInfo['tmpDir'] => { +): FunctionErrorInfo['tmpDir'] => { if (!errorIsOutOfSpaceError(err)) { return null; } @@ -36,7 +36,7 @@ export const getTmpDirStateIfENoSp = ( }; }; -export type EnhancedErrorInfo = LambdaErrorInfo & { +export type EnhancedErrorInfo = FunctionErrorInfo & { /** * @deprecated Will always be an empty string. */