Merge pull request #4700 from remotion-dev/serverless-runtime

package.json

@@ -14,6 +14,7 @@
 		"ci": "turbo run make test --concurrency=1 --no-update-notifier",
 		"watch": "turbo watch make --concurrency=2",
 		"watchwebcodecs": "turbo watch make --filter='@remotion/media-parser' --filter='@remotion/webcodecs'",
+		"watchserverless": "turbo watch make --filter='@remotion/lambda' --filter='@remotion/serverless' --filter='@remotion/streaming'",
 		"release-alpha": "pnpm recursive publish --tag=alpha --no-git-checks && pnpm recursive run --sequential publishprivate",
 		"release": "pnpm recursive publish && pnpm recursive run --sequential publishprivate && git push --tags && git push",
 		"clean": "turbo run clean && rm -rf packages/**/dist && rm -rf packages/**/node_modules && rm -rf node_modules && rm -rf .cache && rm -rf packages/**/tsconfig.tsbuildinfo && rm -f packages/tsconfig.tsbuildinfo && rm -rf packages/**/.turbo && rm -rf .turbo"

packages/it-tests/src/lambda/lambda-browser-bundle.test.ts

@@ -43,7 +43,8 @@ describe('Should be able to bundle @remotion/lambda/client with ESBuild', () =>
 
 	test('Bundle should not include Renderer', async () => {
 		const file = await fs.promises.readFile(outfile, 'utf-8');
-		expect(file).not.toContain('@remotion/renderer');
+		expect(file.includes('"@remotion/renderer"')).toBe(false);
+		expect(file.includes("'@remotion/renderer'")).toBe(false);
 	});
 
 	test('Should be able to delete it', () => {

packages/lambda/src/api/delete-render.ts

@@ -1,12 +1,12 @@
 import type {ProviderSpecifics} from '@remotion/serverless';
 import {
 	getExpectedOutName,
+	getOverallProgressFromStorage,
 	rendersPrefix,
 	type CustomCredentials,
 } from '@remotion/serverless/client';
 import type {AwsProvider} from '../functions/aws-implementation';
 import {awsImplementation} from '../functions/aws-implementation';
-import {getOverallProgressS3} from '../functions/helpers/get-overall-progress-s3';
 import type {AwsRegion} from '../regions';
 import {getAccountId} from '../shared/get-account-id';
 import {cleanItems} from './clean-items';
@@ -28,7 +28,7 @@ export const internalDeleteRender = async (
 	const expectedBucketOwner = await getAccountId({
 		region: input.region,
 	});
-	const progress = await getOverallProgressS3({
+	const progress = await getOverallProgressFromStorage({
 		bucketName: input.bucketName,
 		expectedBucketOwner,
 		region: input.region,

packages/lambda/src/api/deploy-site.ts

@@ -3,7 +3,7 @@ import type {ToOptions} from '@remotion/renderer';
 import type {BrowserSafeApis} from '@remotion/renderer/client';
 import {wrapWithErrorHandling} from '@remotion/renderer/error-handling';
 import type {ProviderSpecifics} from '@remotion/serverless';
-import {validateBucketName} from '@remotion/serverless/client';
+import {validateBucketName, validatePrivacy} from '@remotion/serverless/client';
 import fs from 'node:fs';
 import type {AwsProvider} from '../functions/aws-implementation';
 import {awsImplementation} from '../functions/aws-implementation';
@@ -14,7 +14,6 @@ import {getAccountId} from '../shared/get-account-id';
 import {getS3DiffOperations} from '../shared/get-s3-operations';
 import {makeS3ServeUrl} from '../shared/make-s3-url';
 import {validateAwsRegion} from '../shared/validate-aws-region';
-import {validatePrivacy} from '../shared/validate-privacy';
 import {validateSiteName} from '../shared/validate-site-name';
 import type {UploadDirProgress} from './upload-dir';
 import {uploadDir} from './upload-dir';

packages/lambda/src/api/download-media.ts

@@ -3,12 +3,12 @@ import {RenderInternals} from '@remotion/renderer';
 import type {ProviderSpecifics} from '@remotion/serverless';
 import {
 	getExpectedOutName,
+	getOverallProgressFromStorage,
 	type CustomCredentials,
 } from '@remotion/serverless/client';
 import path from 'node:path';
 import type {AwsProvider} from '../functions/aws-implementation';
 import {awsImplementation} from '../functions/aws-implementation';
-import {getOverallProgressS3} from '../functions/helpers/get-overall-progress-s3';
 import type {LambdaReadFileProgress} from '../functions/helpers/read-with-progress';
 import {lambdaDownloadFileWithProgress} from '../functions/helpers/read-with-progress';
 import type {AwsRegion} from '../regions';
@@ -39,7 +39,7 @@ export const internalDownloadMedia = async (
 	const expectedBucketOwner = await getAccountId({
 		region: input.region,
 	});
-	const overallProgress = await getOverallProgressS3({
+	const overallProgress = await getOverallProgressFromStorage({
 		bucketName: input.bucketName,
 		expectedBucketOwner,
 		region: input.region,

packages/lambda/src/api/get-aws-client.ts

@@ -5,8 +5,8 @@ import * as S3SDK from '@aws-sdk/client-s3';
 import * as ServiceQuotasSDK from '@aws-sdk/client-service-quotas';
 import * as StsSdk from '@aws-sdk/client-sts';
 import type {CustomCredentials} from '@remotion/serverless/client';
-import type {AwsRegion} from '../client';
 import type {AwsProvider} from '../functions/aws-implementation';
+import type {AwsRegion} from '../regions';
 import {
 	getServiceClient,
 	type ServiceMapping,

packages/lambda/src/api/get-compositions-on-lambda.ts

@@ -8,9 +8,8 @@ import {
 } from '@remotion/serverless/client';
 import type {VideoConfig} from 'remotion/no-react';
 import {VERSION} from 'remotion/version';
-import type {AwsRegion} from '../client';
 import {awsImplementation} from '../functions/aws-implementation';
-import {callLambda} from '../shared/call-lambda';
+import type {AwsRegion} from '../regions';
 
 export type GetCompositionsOnLambdaInput = {
 	chromiumOptions?: ChromiumOptions;
@@ -66,10 +65,11 @@ export const getCompositionsOnLambda = async ({
 	});
 
 	try {
-		const res = await callLambda({
+		const res = await awsImplementation.callFunctionSync({
 			functionName,
 			type: ServerlessRoutines.compositions,
 			payload: {
+				type: ServerlessRoutines.compositions,
 				chromiumOptions: chromiumOptions ?? {},
 				serveUrl,
 				envVariables,

packages/lambda/src/api/get-render-progress.ts

@@ -1,14 +1,12 @@
 import type {LogLevel} from '@remotion/renderer';
 import type {CustomCredentials} from '@remotion/serverless/client';
-import {ServerlessRoutines} from '@remotion/serverless/client';
+import {getProgress, ServerlessRoutines} from '@remotion/serverless/client';
 import {
 	awsImplementation,
 	type AwsProvider,
 } from '../functions/aws-implementation';
-import {getProgress} from '../functions/helpers/get-progress';
 import {parseFunctionName} from '../functions/helpers/parse-function-name';
 import type {AwsRegion} from '../regions';
-import {callLambda} from '../shared/call-lambda';
 import type {RenderProgress} from '../shared/constants';
 import {getRenderProgressPayload} from './make-lambda-payload';
 
@@ -56,12 +54,13 @@ export const getRenderProgress = async (
 		});
 	}
 
-	const result = await callLambda<AwsProvider, ServerlessRoutines.status>({
-		functionName: input.functionName,
-		type: ServerlessRoutines.status,
-		payload: getRenderProgressPayload(input),
-		region: input.region,
-		timeoutInTest: 120000,
-	});
+	const result =
+		await awsImplementation.callFunctionSync<ServerlessRoutines.status>({
+			functionName: input.functionName,
+			type: ServerlessRoutines.status,
+			payload: getRenderProgressPayload(input),
+			region: input.region,
+			timeoutInTest: 120000,
+		});
 	return result;
 };

packages/lambda/src/api/make-lambda-payload.ts

@@ -26,16 +26,17 @@ import {
 	compressInputProps,
 	getNeedsToUpload,
 	serializeOrThrow,
+	validateDownloadBehavior,
+	validateFramesPerFunction,
 } from '@remotion/serverless/client';
 import {VERSION} from 'remotion/version';
-import type {AwsRegion, DeleteAfter} from '../client';
+import type {DeleteAfter} from '../client';
 import type {AwsProvider} from '../functions/aws-implementation';
 import {awsImplementation} from '../functions/aws-implementation';
 
 import {validateWebhook} from '@remotion/serverless/client';
 import {NoReactInternals} from 'remotion/no-react';
-import {validateDownloadBehavior} from '../shared/validate-download-behavior';
-import {validateFramesPerLambda} from '../shared/validate-frames-per-lambda';
+import type {AwsRegion} from '../regions';
 import {validateLambdaCodec} from '../shared/validate-lambda-codec';
 import {validateServeUrl} from '../shared/validate-serveurl';
 import type {GetRenderProgressInput} from './get-render-progress';
@@ -136,7 +137,7 @@ export const makeLambdaRenderMediaPayload = async ({
 > => {
 	const actualCodec = validateLambdaCodec(codec);
 	validateServeUrl(serveUrl);
-	validateFramesPerLambda({
+	validateFramesPerFunction({
 		framesPerLambda: framesPerLambda ?? null,
 		durationInFrames: 1,
 	});

packages/lambda/src/api/render-media-on-lambda.ts

@@ -17,9 +17,11 @@ import type {
 	WebhookOption,
 } from '@remotion/serverless/client';
 import {ServerlessRoutines} from '@remotion/serverless/client';
-import type {AwsProvider} from '../functions/aws-implementation';
+import {
+	awsImplementation,
+	type AwsProvider,
+} from '../functions/aws-implementation';
 import type {AwsRegion} from '../regions';
-import {callLambda} from '../shared/call-lambda';
 import {
 	getCloudwatchMethodUrl,
 	getCloudwatchRendererUrl,
@@ -88,7 +90,7 @@ export const internalRenderMediaOnLambdaRaw = async (
 	const {functionName, region, rendererFunctionName} = input;
 
 	try {
-		const res = await callLambda({
+		const res = await awsImplementation.callFunctionSync({
 			functionName,
 			type: ServerlessRoutines.start,
 			payload: await makeLambdaRenderMediaPayload(input),

packages/lambda/src/api/render-still-on-lambda.ts

@@ -6,17 +6,19 @@ import type {
 import type {BrowserSafeApis} from '@remotion/renderer/client';
 import type {DownloadBehavior} from '@remotion/serverless/client';
 import {ServerlessRoutines} from '@remotion/serverless/client';
-import {callLambdaWithStreaming} from '../shared/call-lambda';
 
 import {wrapWithErrorHandling} from '@remotion/renderer/error-handling';
 import type {
+	CostsInfo,
 	ReceivedArtifact,
 	RenderStillLambdaResponsePayload,
 } from '@remotion/serverless';
 import type {OutNameInput, Privacy} from '@remotion/serverless/client';
-import type {AwsProvider} from '../functions/aws-implementation';
+import {
+	awsImplementation,
+	type AwsProvider,
+} from '../functions/aws-implementation';
 import type {AwsRegion} from '../regions';
-import type {CostsInfo} from '../shared/constants';
 import {DEFAULT_MAX_RETRIES} from '../shared/constants';
 import {
 	getCloudwatchMethodUrl,
@@ -87,40 +89,41 @@ const internalRenderStillOnLambda = async (
 		const res = await new Promise<
 			RenderStillLambdaResponsePayload<AwsProvider>
 		>((resolve, reject) => {
-			callLambdaWithStreaming<AwsProvider, ServerlessRoutines.still>({
-				functionName,
-				type: ServerlessRoutines.still,
-				payload,
-				region,
-				receivedStreamingPayload: ({message}) => {
-					if (message.type === 'render-id-determined') {
-						onInit?.({
-							renderId: message.payload.renderId,
-							cloudWatchLogs: getCloudwatchMethodUrl({
-								functionName,
-								method: ServerlessRoutines.still,
-								region,
-								rendererFunctionName: null,
+			awsImplementation
+				.callFunctionStreaming<ServerlessRoutines.still>({
+					functionName,
+					type: ServerlessRoutines.still,
+					payload,
+					region,
+					receivedStreamingPayload: ({message}) => {
+						if (message.type === 'render-id-determined') {
+							onInit?.({
 								renderId: message.payload.renderId,
-							}),
-							lambdaInsightsUrl: getLambdaInsightsUrl({
-								functionName,
-								region,
-							}),
-						});
-					}
+								cloudWatchLogs: getCloudwatchMethodUrl({
+									functionName,
+									method: ServerlessRoutines.still,
+									region,
+									rendererFunctionName: null,
+									renderId: message.payload.renderId,
+								}),
+								lambdaInsightsUrl: getLambdaInsightsUrl({
+									functionName,
+									region,
+								}),
+							});
+						}
 
-					if (message.type === 'error-occurred') {
-						reject(new Error(message.payload.error));
-					}
+						if (message.type === 'error-occurred') {
+							reject(new Error(message.payload.error));
+						}
 
-					if (message.type === 'still-rendered') {
-						resolve(message.payload);
-					}
-				},
-				timeoutInTest: 120000,
-				retriesRemaining: input.maxRetries,
-			})
+						if (message.type === 'still-rendered') {
+							resolve(message.payload);
+						}
+					},
+					timeoutInTest: 120000,
+					retriesRemaining: input.maxRetries,
+				})
 				.then(() => {
 					reject(new Error('Expected response to be streamed'));
 				})

packages/lambda/src/cli/args.ts

@@ -1,8 +1,9 @@
 import {CliInternals} from '@remotion/cli';
 import type {BrowserSafeApis} from '@remotion/renderer/client';
-import type {AwsRegion, DeleteAfter} from '../client';
+import type {DeleteAfter} from '../client';
 
 import type {Privacy} from '@remotion/serverless/client';
+import {AwsRegion} from '../regions';
 import type {RuntimePreference} from '../shared/get-layers';
 
 type LambdaCommandLineOptions = {

packages/lambda/src/cli/commands/quotas/increase.ts

@@ -7,8 +7,8 @@ import {
 import type {LogLevel, LogOptions} from '@remotion/renderer';
 import {exit} from 'node:process';
 import {QUOTAS_COMMAND} from '.';
-import type {AwsRegion} from '../../../client';
 import {BINARY_NAME, LAMBDA_CONCURRENCY_LIMIT_QUOTA} from '../../../defaults';
+import {AwsRegion} from '../../../regions';
 import {getServiceQuotasClient} from '../../../shared/aws-clients';
 import {forceFlagProvided} from '../../args';
 import {getAwsRegion} from '../../get-aws-region';

packages/lambda/src/cli/commands/render/render.ts

@@ -10,7 +10,11 @@ import {getRenderProgress} from '../../../api/get-render-progress';
 import {internalRenderMediaOnLambdaRaw} from '../../../api/render-media-on-lambda';
 
 import type {EnhancedErrorInfo, ProviderSpecifics} from '@remotion/serverless';
-import type {ServerlessCodec} from '@remotion/serverless/client';
+import {
+	validateFramesPerFunction,
+	validatePrivacy,
+	type ServerlessCodec,
+} from '@remotion/serverless/client';
 import type {AwsProvider} from '../../../functions/aws-implementation';
 import {parseFunctionName} from '../../../functions/helpers/parse-function-name';
 import {
@@ -19,8 +23,6 @@ import {
 	DEFAULT_OUTPUT_PRIVACY,
 } from '../../../shared/constants';
 import {sleep} from '../../../shared/sleep';
-import {validateFramesPerLambda} from '../../../shared/validate-frames-per-lambda';
-import {validatePrivacy} from '../../../shared/validate-privacy';
 import {validateMaxRetries} from '../../../shared/validate-retries';
 import {validateServeUrl} from '../../../shared/validate-serveurl';
 import {parsedLambdaCli} from '../../args';
@@ -270,7 +272,7 @@ export const renderCommand = async (
 	const privacy = parsedLambdaCli.privacy ?? DEFAULT_OUTPUT_PRIVACY;
 	validatePrivacy(privacy, true);
 	const framesPerLambda = parsedLambdaCli['frames-per-lambda'] ?? undefined;
-	validateFramesPerLambda({framesPerLambda, durationInFrames: 1});
+	validateFramesPerFunction({framesPerLambda, durationInFrames: 1});
 
 	const webhookCustomData = getWebhookCustomData(logLevel);
 

packages/lambda/src/cli/commands/still.ts

@@ -4,6 +4,7 @@ import type {ChromiumOptions, LogLevel} from '@remotion/renderer';
 import {RenderInternals} from '@remotion/renderer';
 import {BrowserSafeApis} from '@remotion/renderer/client';
 import type {ProviderSpecifics} from '@remotion/serverless';
+import {validatePrivacy} from '@remotion/serverless/client';
 import path from 'path';
 import {NoReactInternals} from 'remotion/no-react';
 import {internalDownloadMedia} from '../../api/download-media';
@@ -15,7 +16,6 @@ import {
 	DEFAULT_OUTPUT_PRIVACY,
 } from '../../shared/constants';
 import {getS3RenderUrl} from '../../shared/get-aws-urls';
-import {validatePrivacy} from '../../shared/validate-privacy';
 import {validateMaxRetries} from '../../shared/validate-retries';
 import {validateServeUrl} from '../../shared/validate-serveurl';
 import {parsedLambdaCli} from '../args';

packages/lambda/src/cli/helpers/find-function-name.ts

@@ -1,7 +1,7 @@
 import type {LogLevel, LogOptions} from '@remotion/renderer';
 import {VERSION} from 'remotion/version';
 import {getFunctions} from '../../api/get-functions';
-import {speculateFunctionName} from '../../client';
+import {speculateFunctionName} from '../../api/speculate-function-name';
 import {BINARY_NAME} from '../../shared/constants';
 import {parsedLambdaCli} from '../args';
 import {FUNCTIONS_COMMAND} from '../commands/functions';

packages/lambda/src/cli/index.ts

@@ -1,13 +1,13 @@
 import {CliInternals} from '@remotion/cli';
 import type {LogLevel} from '@remotion/renderer';
 import {RenderInternals} from '@remotion/renderer';
-import type {ProviderSpecifics} from '@remotion/serverless';
+import {type ProviderSpecifics} from '@remotion/serverless';
+import {DOCS_URL} from '@remotion/serverless/client';
 import {ROLE_NAME} from '../api/iam-validation/suggested-policy';
 import {BINARY_NAME} from '../defaults';
 import type {AwsProvider} from '../functions/aws-implementation';
 import {awsImplementation} from '../functions/aws-implementation';
 import {checkCredentials} from '../shared/check-credentials';
-import {DOCS_URL} from '../shared/docs-url';
 import {parsedLambdaCli} from './args';
 import {
 	COMPOSITIONS_COMMAND,