From 980e52d86ffc04882990e48d09da06aa046c6cc4 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 11 Dec 2025 01:05:00 +0700 Subject: [PATCH 1/3] Docker: Improve video recorder/uploader with async tasks and graceful shutdown Signed-off-by: Viet Nguyen Duc --- Video/recorder.conf | 1 - Video/upload.sh | 190 +++++++++++++++++- Video/uploader.conf | 1 - Video/video.sh | 160 +++++++++++++-- ...mpose-v3-video-upload-standalone-arm64.yml | 79 ++++++++ 5 files changed, 411 insertions(+), 20 deletions(-) create mode 100644 docker-compose-v3-video-upload-standalone-arm64.yml diff --git a/Video/recorder.conf b/Video/recorder.conf index 9bbef29f68..790c43ff14 100755 --- a/Video/recorder.conf +++ b/Video/recorder.conf @@ -6,7 +6,6 @@ autostart=%(ENV_SE_RECORD_VIDEO)s startsecs=0 autorestart=%(ENV_SE_RECORD_VIDEO)s stopsignal=TERM -stopwaitsecs=30 ;Logs (all activity redirected to stdout so it can be seen through "docker logs" redirect_stderr=true diff --git a/Video/upload.sh b/Video/upload.sh index 1ea39dddbc..7e0c442b4e 100755 --- a/Video/upload.sh +++ b/Video/upload.sh @@ -8,6 +8,10 @@ UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"} UPLOAD_PIPE_FILE_NAME=${SE_UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} VIDEO_INTERNAL_UPLOAD=${VIDEO_INTERNAL_UPLOAD:-$SE_VIDEO_INTERNAL_UPLOAD} VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} +UPLOAD_RETRY_MAX_ATTEMPTS=${SE_UPLOAD_RETRY_MAX_ATTEMPTS:-3} +UPLOAD_RETRY_DELAY=${SE_UPLOAD_RETRY_DELAY:-5} +UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-1} +UPLOAD_VERIFY_CHECKSUM=${SE_UPLOAD_VERIFY_CHECKSUM:-"true"} ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"} process_name="video.uploader" @@ -41,6 +45,57 @@ function rename_rclone_env() { } list_rclone_pid=() +upload_success_count=0 +upload_failed_count=0 +graceful_exit_called=false + +function verify_file_ready() { + local file=$1 + + # Check if file exists + if [ ! -f "${file}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File does not exist: ${file}" + return 1 + fi + + # Check if file is readable + if [ ! -r "${file}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File is not readable: ${file}" + return 1 + fi + + # Check if file has non-zero size + local file_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + if [ -z "${file_size}" ] || [ "${file_size}" -eq 0 ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File is empty: ${file}" + return 1 + fi + + # Wait for file to be stable (no longer being written) + local initial_size=${file_size} + sleep ${UPLOAD_FILE_READY_WAIT} + local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + + if [ "${initial_size}" != "${final_size}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File is still being written: ${file} (size changed from ${initial_size} to ${final_size})" + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - File verified ready for upload: ${file} (size: ${final_size} bytes)" + return 0 +} + +function calculate_checksum() { + local file=$1 + # Use md5sum for checksum (available on most systems) + if command -v md5sum >/dev/null 2>&1; then + md5sum "${file}" | awk '{print $1}' + elif command -v md5 >/dev/null 2>&1; then + md5 -q "${file}" + else + echo "" + fi +} function check_and_clear_background() { # Wait for a batch rclone processes to finish if [ ${#list_rclone_pid[@]} -eq ${VIDEO_UPLOAD_BATCH_CHECK} ]; then @@ -52,11 +107,89 @@ function check_and_clear_background() { } +function rclone_upload_with_retry() { + local source=$1 + local target=$2 + local attempt=1 + local source_checksum="" + + # Verify file is ready before attempting upload + if ! verify_file_ready "${source}"; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File verification failed, skipping upload: ${source}" + upload_failed_count=$((upload_failed_count + 1)) + return 1 + fi + + # Calculate checksum if verification is enabled + if [ "${UPLOAD_VERIFY_CHECKSUM}" = "true" ]; then + source_checksum=$(calculate_checksum "${source}") + if [ -n "${source_checksum}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Source file checksum: ${source_checksum}" + fi + fi + + # Retry loop + while [ ${attempt} -le ${UPLOAD_RETRY_MAX_ATTEMPTS} ]; do + echo "$(date -u +"${ts_format}") [${process_name}] - Upload attempt ${attempt}/${UPLOAD_RETRY_MAX_ATTEMPTS}: ${source} to ${target}" + + # Execute rclone command + if rclone --config ${RCLONE_CONFIG} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${source}" "${target}"; then + echo "$(date -u +"${ts_format}") [${process_name}] - SUCCESS: Upload completed: ${source} to ${target}" + + # Verify checksum if enabled and using copy command + if [ "${UPLOAD_VERIFY_CHECKSUM}" = "true" ] && [ -n "${source_checksum}" ] && [ "${UPLOAD_COMMAND}" = "copy" ]; then + # For copy command, verify source file still has same checksum + local post_upload_checksum=$(calculate_checksum "${source}") + if [ "${source_checksum}" = "${post_upload_checksum}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Checksum verification passed: ${source_checksum}" + else + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: Checksum mismatch after upload (before: ${source_checksum}, after: ${post_upload_checksum})" + fi + fi + + # Update statistics file (for cross-process tracking) + ( + flock -x 200 + local current_success=$(grep -oP 'upload_success_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_success_count=$((current_success + 1))" >/tmp/upload_stats.txt.tmp + local current_failed=$(grep -oP 'upload_failed_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_failed_count=${current_failed}" >>/tmp/upload_stats.txt.tmp + mv /tmp/upload_stats.txt.tmp /tmp/upload_stats.txt + ) 200>/tmp/upload_stats.lock + + return 0 + else + local exit_code=$? + echo "$(date -u +"${ts_format}") [${process_name}] - FAILED: Upload attempt ${attempt} failed with exit code ${exit_code}: ${source}" + + if [ ${attempt} -lt ${UPLOAD_RETRY_MAX_ATTEMPTS} ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Retrying in ${UPLOAD_RETRY_DELAY} seconds..." + sleep ${UPLOAD_RETRY_DELAY} + fi + + attempt=$((attempt + 1)) + fi + done + + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: All upload attempts failed for: ${source}" + + # Update statistics file (for cross-process tracking) + ( + flock -x 200 + local current_success=$(grep -oP 'upload_success_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_success_count=${current_success}" >/tmp/upload_stats.txt.tmp + local current_failed=$(grep -oP 'upload_failed_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_failed_count=$((current_failed + 1))" >>/tmp/upload_stats.txt.tmp + mv /tmp/upload_stats.txt.tmp /tmp/upload_stats.txt + ) 200>/tmp/upload_stats.lock + + return 1 +} + function rclone_upload() { local source=$1 local target=$2 - echo "$(date -u +"${ts_format}") [${process_name}] - Uploading ${source} to ${target}" - rclone --config ${RCLONE_CONFIG} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${source}" "${target}" & + rclone_upload_with_retry "${source}" "${target}" & list_rclone_pid+=($!) check_and_clear_background } @@ -99,8 +232,30 @@ function wait_until_pipefile_exists() { done } +function wait_for_active_uploads() { + if [ ${#list_rclone_pid[@]} -gt 0 ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${#list_rclone_pid[@]} active upload process(es) to complete" + for pid in "${list_rclone_pid[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for upload process (PID: $pid)" + wait "$pid" 2>/dev/null || true + fi + done + echo "$(date -u +"${ts_format}") [${process_name}] - All active upload processes completed" + list_rclone_pid=() + fi +} + function graceful_exit() { + # Prevent duplicate execution (trap catches both SIGTERM and EXIT) + if [ "$graceful_exit_called" = "true" ]; then + return 0 + fi + graceful_exit_called=true + echo "$(date -u +"${ts_format}") [${process_name}] - Trapped SIGTERM/SIGINT/x so shutting down uploader" + + # Signal the pipe consumer to stop accepting new files if ! check_if_pid_alive "${UPLOAD_PID}"; then consume_pipe_file_in_background & UPLOAD_PID=$! @@ -108,7 +263,22 @@ function graceful_exit() { echo "exit" >>"${UPLOAD_PIPE_FILE}" & wait "${UPLOAD_PID}" echo "$(date -u +"${ts_format}") [${process_name}] - Uploader consumed all files in the pipe" + + # Wait for all active background rclone uploads to complete + wait_for_active_uploads + + # Log upload statistics from file (written by background processes) + if [ -f "/tmp/upload_stats.txt" ]; then + source "/tmp/upload_stats.txt" 2>/dev/null || true + fi + local total_uploads=$((upload_success_count + upload_failed_count)) + echo "$(date -u +"${ts_format}") [${process_name}] - Upload Statistics:" + echo "$(date -u +"${ts_format}") [${process_name}] - Total uploads attempted: ${total_uploads}" + echo "$(date -u +"${ts_format}") [${process_name}] - Successful uploads: ${upload_success_count}" + echo "$(date -u +"${ts_format}") [${process_name}] - Failed uploads: ${upload_failed_count}" + rm -rf "${FORCE_EXIT_FILE}" + rm -rf "/tmp/upload_stats.txt" echo "$(date -u +"${ts_format}") [${process_name}] - Uploader is ready to shutdown" exit 0 } @@ -117,12 +287,28 @@ rename_rclone_env trap graceful_exit SIGTERM SIGINT EXIT while true; do + # Exit main loop if graceful shutdown is in progress + if [ "$graceful_exit_called" = "true" ]; then + break + fi + wait_until_pipefile_exists + + # Exit main loop if graceful shutdown is in progress + if [ "$graceful_exit_called" = "true" ]; then + break + fi + if ! check_if_pid_alive "${UPLOAD_PID}"; then consume_pipe_file_in_background & UPLOAD_PID=$! fi + while check_if_pid_alive "${UPLOAD_PID}"; do + # Exit main loop if graceful shutdown is in progress + if [ "$graceful_exit_called" = "true" ]; then + break 2 # Break out of both while loops + fi sleep 1 done done diff --git a/Video/uploader.conf b/Video/uploader.conf index 0a3162073a..bd63fbfe2f 100644 --- a/Video/uploader.conf +++ b/Video/uploader.conf @@ -6,7 +6,6 @@ autostart=%(ENV_SE_VIDEO_UPLOAD_ENABLED)s startsecs=0 autorestart=%(ENV_SE_VIDEO_UPLOAD_ENABLED)s stopsignal=TERM -stopwaitsecs=30 ;Logs (all activity redirected to stdout so it can be seen through "docker logs" redirect_stderr=true diff --git a/Video/video.sh b/Video/video.sh index 22c683ccd7..b9b4a75708 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -18,6 +18,7 @@ poll_interval=${SE_VIDEO_POLL_INTERVAL:-2} max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50} file_ready_max_attempts=${SE_VIDEO_FILE_READY_WAIT_ATTEMPTS:-5} wait_uploader_shutdown_max_attempts=${SE_VIDEO_WAIT_UPLOADER_SHUTDOWN_ATTEMPTS:-5} +graceful_stop_delay=${SE_VIDEO_GRACEFUL_STOP_DELAY:-5} ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"} process_name="video.recorder" @@ -33,6 +34,7 @@ else NODE_STATUS_ENDPOINT="${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status" fi +BASIC_AUTH="Authorization: Basic YWRtaW46YWRtaW4=" if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then BASIC_AUTH="$(echo -en "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64 -w0)" BASIC_AUTH="Authorization: Basic ${BASIC_AUTH}" @@ -125,6 +127,51 @@ function wait_util_uploader_shutdown() { fi } +function wait_for_pipe_to_drain() { + if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] || [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then + return 0 + fi + + local wait_count=0 + local max_drain_wait=${SE_VIDEO_WAIT_PIPE_DRAIN:-10} + + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for upload pipe to drain before shutdown" + + # Give uploader time to process queued files + while [[ ${wait_count} -lt ${max_drain_wait} ]]; do + local upload_active=false + + # Check for active rclone/upload processes + if [[ "${VIDEO_INTERNAL_UPLOAD}" = "true" ]]; then + # For internal upload, check rclone processes + local rclone_count=$(pgrep rclone 2>/dev/null | wc -l) + if [[ ${rclone_count} -gt 0 ]]; then + upload_active=true + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${rclone_count} active rclone upload(s) to complete (${wait_count}/${max_drain_wait})" + fi + else + # For external upload, just give some time for pipe to be consumed + # We can't check external container processes, so use a fixed wait + if [[ ${wait_count} -lt 3 ]]; then + upload_active=true + echo "$(date -u +"${ts_format}") [${process_name}] - Giving external uploader time to process queue (${wait_count}/3)" + fi + fi + + if [[ "${upload_active}" = "false" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Upload pipe drained, no active uploads" + break + fi + + sleep ${poll_interval} + wait_count=$((wait_count + 1)) + done + + if [[ ${wait_count} -ge ${max_drain_wait} ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: Pipe drain timeout reached, some uploads may not complete" + fi +} + function send_exit_signal_to_uploader() { if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then echo "$(date -u +"${ts_format}") [${process_name}] - Sending a signal to force exit the uploader" @@ -144,7 +191,7 @@ function stop_ffmpeg() { while true; do FFMPEG_PID=$(pgrep -f "ffmpeg -hide_banner" | tr '\n' ' ') if [ -n "$FFMPEG_PID" ]; then - kill -SIGTERM $FFMPEG_PID + kill -SIGINT $FFMPEG_PID wait $FFMPEG_PID fi if ! pgrep -f "ffmpeg -hide_banner" >/dev/null; then @@ -154,18 +201,81 @@ function stop_ffmpeg() { done } +function stop_ffmpeg_graceful_async() { + local video_file_to_finalize="$1" + local video_file_name_param="$2" + local upload_dest_prefix="$3" + local upload_pipe_file="$4" + local should_upload="$5" + local session_id_param="$6" + + ( + # Send SIGINT to FFmpeg + FFMPEG_PID=$(pgrep -f "ffmpeg -hide_banner" | tr '\n' ' ') + if [ -n "$FFMPEG_PID" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Sending SIGINT to FFmpeg PID: $FFMPEG_PID for file: ${video_file_name_param}" + kill -SIGINT $FFMPEG_PID + + # Wait for FFmpeg to finish writing + wait $FFMPEG_PID 2>/dev/null + + # Grace period for metadata finalization + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay} seconds for video metadata finalization" + sleep ${graceful_stop_delay} + + # Verify file integrity after grace period + if [[ -f "${video_file_to_finalize}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file finalized: ${video_file_to_finalize}" + + # Trigger upload AFTER file is finalized + if [[ "${should_upload}" = "true" ]] && [[ -n "${upload_dest_prefix}" ]] && [[ -n "${upload_pipe_file}" ]]; then + upload_destination="${upload_dest_prefix}/${video_file_name_param}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Add to pipe a signal Uploading video to ${upload_destination}" + echo "${video_file_to_finalize} ${upload_dest_prefix}" >>"${upload_pipe_file}" + fi + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: Video file not found after finalization: ${video_file_to_finalize}" + fi + fi + ) & + local bg_pid=$! + background_finalization_pids+=($bg_pid) + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] FFmpeg graceful stop initiated in background (PID: ${bg_pid}) for file: ${video_file_name_param}" +} + function stop_recording() { - stop_ffmpeg - echo "$(date -u +"${ts_format}") [${process_name}] - Video recording stopped" + local use_async=${1:-true} + + if [[ "${use_async}" = "true" ]]; then + # Async stop - allows immediate start of new recording + # Upload will be triggered AFTER FFmpeg finalization in the background + local should_upload="false" + if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then + should_upload="true" + elif [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Stopping video recording for file: ${video_file_name} (async mode)" + stop_ffmpeg_graceful_async "${video_file}" "${video_file_name}" "${UPLOAD_DESTINATION_PREFIX}" "${UPLOAD_PIPE_FILE}" "${should_upload}" "${prev_session_id}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Video recording stopped (async mode - new session can start immediately)" + else + # Sync stop - wait for FFmpeg to fully stop, then upload immediately + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Stopping video recording for file: ${video_file_name} (sync mode)" + stop_ffmpeg + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Video recording stopped (sync mode)" + + if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then + upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Add to pipe a signal Uploading video to $upload_destination" + echo "$video_file ${UPLOAD_DESTINATION_PREFIX}" >>${UPLOAD_PIPE_FILE} & + elif [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." + fi + fi + recorded_count=$((recorded_count + 1)) recording_started="false" - if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then - upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} - echo "$(date -u +"${ts_format}") [${process_name}] - Add to pipe a signal Uploading video to $upload_destination" - echo "$video_file ${UPLOAD_DESTINATION_PREFIX}" >>${UPLOAD_PIPE_FILE} & - elif [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." - fi } function check_if_ffmpeg_running() { @@ -193,8 +303,9 @@ function wait_for_file_integrity() { } function stop_if_recording_inprogress() { + local use_async=${1:-false} if [[ "$recording_started" = "true" ]] || check_if_ffmpeg_running; then - stop_recording + stop_recording "${use_async}" fi } @@ -204,9 +315,24 @@ function log_node_response() { fi } +function wait_for_background_finalization() { + if [ ${#background_finalization_pids[@]} -gt 0 ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${#background_finalization_pids[@]} background finalization process(es) to complete" + for pid in "${background_finalization_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for background finalization process (PID: $pid)" + wait "$pid" 2>/dev/null || true + fi + done + echo "$(date -u +"${ts_format}") [${process_name}] - All background finalization processes completed" + fi +} + function graceful_exit() { echo "$(date -u +"${ts_format}") [${process_name}] - Trapped SIGTERM/SIGINT/x so shutting down recorder" stop_if_recording_inprogress + wait_for_background_finalization + wait_for_pipe_to_drain send_exit_signal_to_uploader wait_util_uploader_shutdown } @@ -250,21 +376,23 @@ else attempts=0 max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} recorded_count=0 + # Track background finalization processes + declare -a background_finalization_pids wait_for_api_respond while curl --noproxy "*" -H "${BASIC_AUTH}" -sk --request GET ${NODE_STATUS_ENDPOINT} >"/tmp/status.json"; do session_id="$(jq -r "${JQ_SESSION_ID_QUERY}" "/tmp/status.json")" if [[ "$session_id" != "null" && "$session_id" != "" && "$session_id" != "reserved" && "$recording_started" = "false" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Session: $session_id is created" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] New session detected" session_capabilities="$(jq -r "${JQ_SESSION_CAPABILITIES_QUERY}" "/tmp/status.json")" return_list=($(python3 "${VIDEO_CONFIG_DIRECTORY}/video_nodeQuery.py" "${session_id}" "${session_capabilities}")) caps_se_video_record="${return_list[0]}" video_file_name="${return_list[1]}.mp4" if [[ "$caps_se_video_record" = "true" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Start recording: $caps_se_video_record, video file name: $video_file_name" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] Start recording enabled, video file: ${video_file_name}" log_node_response video_file="${VIDEO_FOLDER}/$video_file_name" - echo "$(date -u +"${ts_format}") [${process_name}] - Starting to record video" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] Starting FFmpeg to record video" ffmpeg -hide_banner -loglevel warning -threads ${SE_FFMPEG_THREADS:-1} -thread_queue_size 512 \ -probesize 32M -analyzeduration 0 -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} \ -i ${DISPLAY} ${SE_AUDIO_SOURCE} -codec:v ${CODEC} ${PRESET:-"-preset veryfast"} \ @@ -275,7 +403,7 @@ else recording_started="true" prev_session_id=$session_id fi - echo "$(date -u +"${ts_format}") [${process_name}] - Video recording started" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] Video recording started (FFmpeg PID: ${FFMPEG_PID})" sleep ${poll_interval} fi elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]]; then @@ -285,7 +413,7 @@ else exit fi elif [[ $recording_started = "true" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Video recording in progress" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Video recording in progress" sleep ${poll_interval} else sleep ${poll_interval} diff --git a/docker-compose-v3-video-upload-standalone-arm64.yml b/docker-compose-v3-video-upload-standalone-arm64.yml new file mode 100644 index 0000000000..f918866eb1 --- /dev/null +++ b/docker-compose-v3-video-upload-standalone-arm64.yml @@ -0,0 +1,79 @@ +# To execute this docker compose yml file use `docker compose -f docker-compose-v3-video-upload-standalone-arm64.yml up` +# Add the `-d` flag at the end for detached execution +# To stop the execution, hit Ctrl+C, and then `docker compose -f docker-compose-v3-video-upload-standalone-arm64.yml down` +# ${variable_pattern} get value from .env in the same directory +services: + # Start a local FTP server to demonstrate video upload with RCLONE (https://github.com/delfer/docker-alpine-ftp-server) + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + volumes: + # Mount the local directory `/home/${USER}/Videos/upload` to the FTP server's `/ftp/seluser` directory to check out the uploaded videos + - /tmp/upload:/ftp/seluser + command: ["/bin/sh", "-c", "/sbin/tini -- /bin/start_vsftpd.sh && tail -f /dev/null"] + stop_grace_period: 30s + + # File browser to manage the uploaded videos from the FTP server + file_browser: + image: filebrowser/filebrowser:latest + container_name: file_browser + restart: always + ports: + - "8081:80" + volumes: + # Mount the local directory `/tmp/upload` to file browser's `/srv` directory to check out the uploaded videos + - /tmp/upload:/srv + environment: + - FB_NOAUTH=true + + standalone_chrome: + image: selenium/standalone-chromium:4.38.0-20251101 + shm_size: 2gb + ports: + - "4444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_RECORD_VIDEO=true + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s + + standalone_firefox: + image: selenium/standalone-firefox:4.38.0-20251101 + shm_size: 2gb + ports: + - "5444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_RECORD_VIDEO=true + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s From 6a788e2221d018702b40da252e82e09e478cf512 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 11 Dec 2025 12:22:05 +0700 Subject: [PATCH 2/3] Fix file recording broken Signed-off-by: Viet Nguyen Duc --- Video/video.sh | 100 ++++++++++++++++++++++++++++++++++++++----- tests/get_started.py | 2 +- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/Video/video.sh b/Video/video.sh index b9b4a75708..7e82f8d8cd 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -19,6 +19,9 @@ max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50} file_ready_max_attempts=${SE_VIDEO_FILE_READY_WAIT_ATTEMPTS:-5} wait_uploader_shutdown_max_attempts=${SE_VIDEO_WAIT_UPLOADER_SHUTDOWN_ATTEMPTS:-5} graceful_stop_delay=${SE_VIDEO_GRACEFUL_STOP_DELAY:-5} +min_recording_duration=${SE_VIDEO_MIN_RECORDING_DURATION:-5} +video_validation_enabled=${SE_VIDEO_VALIDATION_ENABLED:-"true"} +video_fix_corrupted=${SE_VIDEO_FIX_CORRUPTED:-"true"} ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"} process_name="video.recorder" @@ -187,12 +190,66 @@ function exit_on_max_session_reach() { fi } +function validate_mp4_file() { + local video_file="$1" + local session_id_param="$2" + + if [[ "${video_validation_enabled}" != "true" ]]; then + return 0 + fi + + if [[ ! -f "${video_file}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file does not exist: ${video_file}" + return 1 + fi + + # Check if file has moov atom using ffmpeg probe + if ffmpeg -v error -i "${video_file}" -f null - 2>&1 | grep -q "moov atom not found"; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file missing moov atom: ${video_file}" + return 1 + fi + + # Quick validation: check if ffmpeg can read the file + if ! ffmpeg -v error -i "${video_file}" -t 0.1 -f null - 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file validation failed: ${video_file}" + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file validation passed: ${video_file}" + return 0 +} + +function attempt_fix_mp4_file() { + local video_file="$1" + local session_id_param="$2" + + if [[ "${video_fix_corrupted}" != "true" ]]; then + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Attempting to fix corrupted MP4 file: ${video_file}" + + local temp_file="${video_file}.temp.mp4" + + # Try to recover the file by remuxing with faststart + if ffmpeg -v warning -i "${video_file}" -c copy -movflags +faststart "${temp_file}" 2>/dev/null; then + mv "${temp_file}" "${video_file}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Successfully fixed MP4 file: ${video_file}" + return 0 + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Failed to fix MP4 file: ${video_file}" + rm -f "${temp_file}" + return 1 + fi +} + function stop_ffmpeg() { while true; do FFMPEG_PID=$(pgrep -f "ffmpeg -hide_banner" | tr '\n' ' ') if [ -n "$FFMPEG_PID" ]; then - kill -SIGINT $FFMPEG_PID - wait $FFMPEG_PID + # Single SIGTERM for graceful shutdown - allows FFmpeg to write moov atom + kill -SIGTERM $FFMPEG_PID + wait $FFMPEG_PID 2>/dev/null fi if ! pgrep -f "ffmpeg -hide_banner" >/dev/null; then break @@ -210,11 +267,12 @@ function stop_ffmpeg_graceful_async() { local session_id_param="$6" ( - # Send SIGINT to FFmpeg + # Send single SIGTERM to FFmpeg for graceful shutdown + # This allows FFmpeg to properly write the moov atom (MP4 metadata) FFMPEG_PID=$(pgrep -f "ffmpeg -hide_banner" | tr '\n' ' ') if [ -n "$FFMPEG_PID" ]; then - echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Sending SIGINT to FFmpeg PID: $FFMPEG_PID for file: ${video_file_name_param}" - kill -SIGINT $FFMPEG_PID + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Sending SIGTERM to FFmpeg PID: $FFMPEG_PID for file: ${video_file_name_param}" + kill -SIGTERM $FFMPEG_PID # Wait for FFmpeg to finish writing wait $FFMPEG_PID 2>/dev/null @@ -223,15 +281,35 @@ function stop_ffmpeg_graceful_async() { echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay} seconds for video metadata finalization" sleep ${graceful_stop_delay} - # Verify file integrity after grace period + # Verify file exists if [[ -f "${video_file_to_finalize}" ]]; then echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file finalized: ${video_file_to_finalize}" - # Trigger upload AFTER file is finalized - if [[ "${should_upload}" = "true" ]] && [[ -n "${upload_dest_prefix}" ]] && [[ -n "${upload_pipe_file}" ]]; then - upload_destination="${upload_dest_prefix}/${video_file_name_param}" - echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Add to pipe a signal Uploading video to ${upload_destination}" - echo "${video_file_to_finalize} ${upload_dest_prefix}" >>"${upload_pipe_file}" + # Validate MP4 file integrity + if validate_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then + # File is valid, proceed with upload + if [[ "${should_upload}" = "true" ]] && [[ -n "${upload_dest_prefix}" ]] && [[ -n "${upload_pipe_file}" ]]; then + upload_destination="${upload_dest_prefix}/${video_file_name_param}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Add to pipe a signal Uploading video to ${upload_destination}" + echo "${video_file_to_finalize} ${upload_dest_prefix}" >>"${upload_pipe_file}" + fi + else + # Validation failed, attempt to fix + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video validation failed, attempting recovery" + if attempt_fix_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then + # Fixed successfully, re-validate and upload + if validate_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then + if [[ "${should_upload}" = "true" ]] && [[ -n "${upload_dest_prefix}" ]] && [[ -n "${upload_pipe_file}" ]]; then + upload_destination="${upload_dest_prefix}/${video_file_name_param}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Add to pipe a signal Uploading recovered video to ${upload_destination}" + echo "${video_file_to_finalize} ${upload_dest_prefix}" >>"${upload_pipe_file}" + fi + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file still corrupted after recovery attempt, skipping upload" + fi + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file recovery failed, skipping upload" + fi fi else echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: Video file not found after finalization: ${video_file_to_finalize}" diff --git a/tests/get_started.py b/tests/get_started.py index 488a22bf64..15bfc9545f 100644 --- a/tests/get_started.py +++ b/tests/get_started.py @@ -41,7 +41,7 @@ def run_browser_instance(browser, grid_url): print(f"Session created: {driver.session_id} ({browser})") driver.get('https://www.google.com/') print(driver.title) - time.sleep(100) + time.sleep(10) driver.quit() From 69e75a7206824e3ab167aca07009c7dfb6e4b8e0 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 11 Dec 2025 16:38:42 +0700 Subject: [PATCH 3/3] Fix file recording broken Signed-off-by: Viet Nguyen Duc --- Video/upload.sh | 77 ++++++++++++++++++++++++++++++++++++++++++++----- Video/video.sh | 64 +++++++++++++++++++++++++++++++++++----- 2 files changed, 126 insertions(+), 15 deletions(-) diff --git a/Video/upload.sh b/Video/upload.sh index 7e0c442b4e..8127ca1b40 100755 --- a/Video/upload.sh +++ b/Video/upload.sh @@ -10,8 +10,10 @@ VIDEO_INTERNAL_UPLOAD=${VIDEO_INTERNAL_UPLOAD:-$SE_VIDEO_INTERNAL_UPLOAD} VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} UPLOAD_RETRY_MAX_ATTEMPTS=${SE_UPLOAD_RETRY_MAX_ATTEMPTS:-3} UPLOAD_RETRY_DELAY=${SE_UPLOAD_RETRY_DELAY:-5} -UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-1} +UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-3} +UPLOAD_FILE_STABILITY_RETRIES=${SE_UPLOAD_FILE_STABILITY_RETRIES:-5} UPLOAD_VERIFY_CHECKSUM=${SE_UPLOAD_VERIFY_CHECKSUM:-"true"} +UPLOAD_VALIDATE_MP4=${SE_UPLOAD_VALIDATE_MP4:-"true"} ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"} process_name="video.uploader" @@ -49,6 +51,36 @@ upload_success_count=0 upload_failed_count=0 graceful_exit_called=false +function validate_mp4_moov_atom() { + local file=$1 + + if [[ "${UPLOAD_VALIDATE_MP4}" != "true" ]]; then + return 0 + fi + + # Only validate MP4/M4V files + if [[ ! "${file}" =~ \.(mp4|m4v)$ ]]; then + return 0 + fi + + # Check if file has moov atom using ffmpeg probe + local error_output=$(ffmpeg -v error -i "${file}" -f null - 2>&1) + + if echo "${error_output}" | grep -q "moov atom not found"; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: MP4 file missing moov atom: ${file}" + return 1 + fi + + # Quick validation: check if ffmpeg can read the file + if ! ffmpeg -v error -i "${file}" -t 0.1 -f null - 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: MP4 file validation failed: ${file}" + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - MP4 validation passed: ${file}" + return 0 +} + function verify_file_ready() { local file=$1 @@ -71,13 +103,44 @@ function verify_file_ready() { return 1 fi - # Wait for file to be stable (no longer being written) - local initial_size=${file_size} - sleep ${UPLOAD_FILE_READY_WAIT} - local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + # Wait for file to be stable (no longer being written) with retries + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for file to stabilize: ${file}" + local retry=0 + local stable=false + + while [ ${retry} -lt ${UPLOAD_FILE_STABILITY_RETRIES} ]; do + local initial_size=${file_size} + sleep ${UPLOAD_FILE_READY_WAIT} + local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + + if [ "${initial_size}" = "${final_size}" ]; then + # Size is stable, force filesystem sync and verify again + sync + sleep 1 + local verify_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + + if [ "${final_size}" = "${verify_size}" ]; then + stable=true + echo "$(date -u +"${ts_format}") [${process_name}] - File size stable at ${final_size} bytes after ${retry} retries" + break + fi + fi + + retry=$((retry + 1)) + file_size=${final_size} + + if [ ${retry} -lt ${UPLOAD_FILE_STABILITY_RETRIES} ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - File size changed (${initial_size} -> ${final_size}), waiting more (retry ${retry}/${UPLOAD_FILE_STABILITY_RETRIES})" + fi + done + + if [ "${stable}" != "true" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File did not stabilize after ${UPLOAD_FILE_STABILITY_RETRIES} retries: ${file}" + return 1 + fi - if [ "${initial_size}" != "${final_size}" ]; then - echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File is still being written: ${file} (size changed from ${initial_size} to ${final_size})" + # Validate MP4 moov atom if enabled + if ! validate_mp4_moov_atom "${file}"; then return 1 fi diff --git a/Video/video.sh b/Video/video.sh index 7e82f8d8cd..0ef25009e4 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -19,6 +19,8 @@ max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50} file_ready_max_attempts=${SE_VIDEO_FILE_READY_WAIT_ATTEMPTS:-5} wait_uploader_shutdown_max_attempts=${SE_VIDEO_WAIT_UPLOADER_SHUTDOWN_ATTEMPTS:-5} graceful_stop_delay=${SE_VIDEO_GRACEFUL_STOP_DELAY:-5} +ffmpeg_stop_timeout=${SE_VIDEO_FFMPEG_STOP_TIMEOUT:-10} +file_stability_wait=${SE_VIDEO_FILE_STABILITY_WAIT:-2} min_recording_duration=${SE_VIDEO_MIN_RECORDING_DURATION:-5} video_validation_enabled=${SE_VIDEO_VALIDATION_ENABLED:-"true"} video_fix_corrupted=${SE_VIDEO_FIX_CORRUPTED:-"true"} @@ -190,6 +192,32 @@ function exit_on_max_session_reach() { fi } +function wait_for_file_stability() { + local video_file="$1" + local session_id_param="$2" + + if [[ ! -f "${video_file}" ]]; then + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${file_stability_wait}s for file to stabilize: ${video_file}" + + local prev_size=$(stat -f%z "${video_file}" 2>/dev/null || stat -c%s "${video_file}" 2>/dev/null) + sleep ${file_stability_wait} + local curr_size=$(stat -f%z "${video_file}" 2>/dev/null || stat -c%s "${video_file}" 2>/dev/null) + + if [[ "${prev_size}" != "${curr_size}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: File size changed during stability check (${prev_size} -> ${curr_size}), waiting longer" + sleep ${file_stability_wait} + fi + + # Force filesystem sync + sync + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] File stable at size: ${curr_size} bytes" + return 0 +} + function validate_mp4_file() { local video_file="$1" local session_id_param="$2" @@ -274,17 +302,33 @@ function stop_ffmpeg_graceful_async() { echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Sending SIGTERM to FFmpeg PID: $FFMPEG_PID for file: ${video_file_name_param}" kill -SIGTERM $FFMPEG_PID - # Wait for FFmpeg to finish writing + # Wait for FFmpeg to finish writing with timeout + local wait_count=0 + while kill -0 $FFMPEG_PID 2>/dev/null && [ $wait_count -lt ${ffmpeg_stop_timeout} ]; do + sleep 1 + wait_count=$((wait_count + 1)) + done + + if kill -0 $FFMPEG_PID 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: FFmpeg still running after ${ffmpeg_stop_timeout}s, forcing kill" + kill -SIGKILL $FFMPEG_PID 2>/dev/null + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] FFmpeg exited gracefully after ${wait_count}s" + fi + wait $FFMPEG_PID 2>/dev/null - # Grace period for metadata finalization - echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay} seconds for video metadata finalization" + # Grace period for metadata finalization and filesystem sync + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay}s for video metadata finalization" sleep ${graceful_stop_delay} - # Verify file exists + # Verify file exists and wait for stability if [[ -f "${video_file_to_finalize}" ]]; then echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file finalized: ${video_file_to_finalize}" + # Wait for file to stabilize (no more writes) + wait_for_file_stability "${video_file_to_finalize}" "${session_id_param}" + # Validate MP4 file integrity if validate_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then # File is valid, proceed with upload @@ -436,8 +480,10 @@ if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] && [[ "${VIDEO_FILE_NAME}" != "auto ffmpeg -hide_banner -loglevel warning -threads ${SE_FFMPEG_THREADS:-1} -thread_queue_size 512 \ -probesize 32M -analyzeduration 0 -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} \ -i ${DISPLAY} ${SE_AUDIO_SOURCE} -codec:v ${CODEC} ${PRESET:-"-preset veryfast"} \ - -tune zerolatency -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ - -pix_fmt yuv420p -movflags +faststart "$video_file" & + -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ + -pix_fmt yuv420p -g ${SE_VIDEO_GOP_SIZE:-60} -keyint_min ${SE_VIDEO_KEYINT_MIN:-30} \ + -sc_threshold 0 -force_key_frames "expr:gte(t,n_forced*2)" \ + -movflags +faststart+frag_keyframe+empty_moov+default_base_moof "$video_file" & FFMPEG_PID=$! if ps -p $FFMPEG_PID >/dev/null; then wait $FFMPEG_PID @@ -474,8 +520,10 @@ else ffmpeg -hide_banner -loglevel warning -threads ${SE_FFMPEG_THREADS:-1} -thread_queue_size 512 \ -probesize 32M -analyzeduration 0 -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} \ -i ${DISPLAY} ${SE_AUDIO_SOURCE} -codec:v ${CODEC} ${PRESET:-"-preset veryfast"} \ - -tune zerolatency -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ - -pix_fmt yuv420p -movflags +faststart "$video_file" & + -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ + -pix_fmt yuv420p -g ${SE_VIDEO_GOP_SIZE:-60} -keyint_min ${SE_VIDEO_KEYINT_MIN:-30} \ + -sc_threshold 0 -force_key_frames "expr:gte(t,n_forced*2)" \ + -movflags +faststart+frag_keyframe+empty_moov+default_base_moof "$video_file" & FFMPEG_PID=$! if ps -p $FFMPEG_PID >/dev/null; then recording_started="true"