diff --git a/Video/recorder.conf b/Video/recorder.conf index 9bbef29f6..790c43ff1 100755 --- a/Video/recorder.conf +++ b/Video/recorder.conf @@ -6,7 +6,6 @@ autostart=%(ENV_SE_RECORD_VIDEO)s startsecs=0 autorestart=%(ENV_SE_RECORD_VIDEO)s stopsignal=TERM -stopwaitsecs=30 ;Logs (all activity redirected to stdout so it can be seen through "docker logs" redirect_stderr=true diff --git a/Video/upload.sh b/Video/upload.sh index 1ea39dddb..8127ca1b4 100755 --- a/Video/upload.sh +++ b/Video/upload.sh @@ -8,6 +8,12 @@ UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"} UPLOAD_PIPE_FILE_NAME=${SE_UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} VIDEO_INTERNAL_UPLOAD=${VIDEO_INTERNAL_UPLOAD:-$SE_VIDEO_INTERNAL_UPLOAD} VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} +UPLOAD_RETRY_MAX_ATTEMPTS=${SE_UPLOAD_RETRY_MAX_ATTEMPTS:-3} +UPLOAD_RETRY_DELAY=${SE_UPLOAD_RETRY_DELAY:-5} +UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-3} +UPLOAD_FILE_STABILITY_RETRIES=${SE_UPLOAD_FILE_STABILITY_RETRIES:-5} +UPLOAD_VERIFY_CHECKSUM=${SE_UPLOAD_VERIFY_CHECKSUM:-"true"} +UPLOAD_VALIDATE_MP4=${SE_UPLOAD_VALIDATE_MP4:-"true"} ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"} process_name="video.uploader" @@ -41,6 +47,118 @@ function rename_rclone_env() { } list_rclone_pid=() +upload_success_count=0 +upload_failed_count=0 +graceful_exit_called=false + +function validate_mp4_moov_atom() { + local file=$1 + + if [[ "${UPLOAD_VALIDATE_MP4}" != "true" ]]; then + return 0 + fi + + # Only validate MP4/M4V files + if [[ ! "${file}" =~ \.(mp4|m4v)$ ]]; then + return 0 + fi + + # Check if file has moov atom using ffmpeg probe + local error_output=$(ffmpeg -v error -i "${file}" -f null - 2>&1) + + if echo "${error_output}" | grep -q "moov atom not found"; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: MP4 file missing moov atom: ${file}" + return 1 + fi + + # Quick validation: check if ffmpeg can read the file + if ! ffmpeg -v error -i "${file}" -t 0.1 -f null - 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: MP4 file validation failed: ${file}" + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - MP4 validation passed: ${file}" + return 0 +} + +function verify_file_ready() { + local file=$1 + + # Check if file exists + if [ ! -f "${file}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File does not exist: ${file}" + return 1 + fi + + # Check if file is readable + if [ ! -r "${file}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File is not readable: ${file}" + return 1 + fi + + # Check if file has non-zero size + local file_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + if [ -z "${file_size}" ] || [ "${file_size}" -eq 0 ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File is empty: ${file}" + return 1 + fi + + # Wait for file to be stable (no longer being written) with retries + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for file to stabilize: ${file}" + local retry=0 + local stable=false + + while [ ${retry} -lt ${UPLOAD_FILE_STABILITY_RETRIES} ]; do + local initial_size=${file_size} + sleep ${UPLOAD_FILE_READY_WAIT} + local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + + if [ "${initial_size}" = "${final_size}" ]; then + # Size is stable, force filesystem sync and verify again + sync + sleep 1 + local verify_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null) + + if [ "${final_size}" = "${verify_size}" ]; then + stable=true + echo "$(date -u +"${ts_format}") [${process_name}] - File size stable at ${final_size} bytes after ${retry} retries" + break + fi + fi + + retry=$((retry + 1)) + file_size=${final_size} + + if [ ${retry} -lt ${UPLOAD_FILE_STABILITY_RETRIES} ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - File size changed (${initial_size} -> ${final_size}), waiting more (retry ${retry}/${UPLOAD_FILE_STABILITY_RETRIES})" + fi + done + + if [ "${stable}" != "true" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File did not stabilize after ${UPLOAD_FILE_STABILITY_RETRIES} retries: ${file}" + return 1 + fi + + # Validate MP4 moov atom if enabled + if ! validate_mp4_moov_atom "${file}"; then + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - File verified ready for upload: ${file} (size: ${final_size} bytes)" + return 0 +} + +function calculate_checksum() { + local file=$1 + # Use md5sum for checksum (available on most systems) + if command -v md5sum >/dev/null 2>&1; then + md5sum "${file}" | awk '{print $1}' + elif command -v md5 >/dev/null 2>&1; then + md5 -q "${file}" + else + echo "" + fi +} function check_and_clear_background() { # Wait for a batch rclone processes to finish if [ ${#list_rclone_pid[@]} -eq ${VIDEO_UPLOAD_BATCH_CHECK} ]; then @@ -52,11 +170,89 @@ function check_and_clear_background() { } +function rclone_upload_with_retry() { + local source=$1 + local target=$2 + local attempt=1 + local source_checksum="" + + # Verify file is ready before attempting upload + if ! verify_file_ready "${source}"; then + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: File verification failed, skipping upload: ${source}" + upload_failed_count=$((upload_failed_count + 1)) + return 1 + fi + + # Calculate checksum if verification is enabled + if [ "${UPLOAD_VERIFY_CHECKSUM}" = "true" ]; then + source_checksum=$(calculate_checksum "${source}") + if [ -n "${source_checksum}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Source file checksum: ${source_checksum}" + fi + fi + + # Retry loop + while [ ${attempt} -le ${UPLOAD_RETRY_MAX_ATTEMPTS} ]; do + echo "$(date -u +"${ts_format}") [${process_name}] - Upload attempt ${attempt}/${UPLOAD_RETRY_MAX_ATTEMPTS}: ${source} to ${target}" + + # Execute rclone command + if rclone --config ${RCLONE_CONFIG} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${source}" "${target}"; then + echo "$(date -u +"${ts_format}") [${process_name}] - SUCCESS: Upload completed: ${source} to ${target}" + + # Verify checksum if enabled and using copy command + if [ "${UPLOAD_VERIFY_CHECKSUM}" = "true" ] && [ -n "${source_checksum}" ] && [ "${UPLOAD_COMMAND}" = "copy" ]; then + # For copy command, verify source file still has same checksum + local post_upload_checksum=$(calculate_checksum "${source}") + if [ "${source_checksum}" = "${post_upload_checksum}" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Checksum verification passed: ${source_checksum}" + else + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: Checksum mismatch after upload (before: ${source_checksum}, after: ${post_upload_checksum})" + fi + fi + + # Update statistics file (for cross-process tracking) + ( + flock -x 200 + local current_success=$(grep -oP 'upload_success_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_success_count=$((current_success + 1))" >/tmp/upload_stats.txt.tmp + local current_failed=$(grep -oP 'upload_failed_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_failed_count=${current_failed}" >>/tmp/upload_stats.txt.tmp + mv /tmp/upload_stats.txt.tmp /tmp/upload_stats.txt + ) 200>/tmp/upload_stats.lock + + return 0 + else + local exit_code=$? + echo "$(date -u +"${ts_format}") [${process_name}] - FAILED: Upload attempt ${attempt} failed with exit code ${exit_code}: ${source}" + + if [ ${attempt} -lt ${UPLOAD_RETRY_MAX_ATTEMPTS} ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Retrying in ${UPLOAD_RETRY_DELAY} seconds..." + sleep ${UPLOAD_RETRY_DELAY} + fi + + attempt=$((attempt + 1)) + fi + done + + echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: All upload attempts failed for: ${source}" + + # Update statistics file (for cross-process tracking) + ( + flock -x 200 + local current_success=$(grep -oP 'upload_success_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_success_count=${current_success}" >/tmp/upload_stats.txt.tmp + local current_failed=$(grep -oP 'upload_failed_count=\K\d+' /tmp/upload_stats.txt 2>/dev/null || echo 0) + echo "upload_failed_count=$((current_failed + 1))" >>/tmp/upload_stats.txt.tmp + mv /tmp/upload_stats.txt.tmp /tmp/upload_stats.txt + ) 200>/tmp/upload_stats.lock + + return 1 +} + function rclone_upload() { local source=$1 local target=$2 - echo "$(date -u +"${ts_format}") [${process_name}] - Uploading ${source} to ${target}" - rclone --config ${RCLONE_CONFIG} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${source}" "${target}" & + rclone_upload_with_retry "${source}" "${target}" & list_rclone_pid+=($!) check_and_clear_background } @@ -99,8 +295,30 @@ function wait_until_pipefile_exists() { done } +function wait_for_active_uploads() { + if [ ${#list_rclone_pid[@]} -gt 0 ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${#list_rclone_pid[@]} active upload process(es) to complete" + for pid in "${list_rclone_pid[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for upload process (PID: $pid)" + wait "$pid" 2>/dev/null || true + fi + done + echo "$(date -u +"${ts_format}") [${process_name}] - All active upload processes completed" + list_rclone_pid=() + fi +} + function graceful_exit() { + # Prevent duplicate execution (trap catches both SIGTERM and EXIT) + if [ "$graceful_exit_called" = "true" ]; then + return 0 + fi + graceful_exit_called=true + echo "$(date -u +"${ts_format}") [${process_name}] - Trapped SIGTERM/SIGINT/x so shutting down uploader" + + # Signal the pipe consumer to stop accepting new files if ! check_if_pid_alive "${UPLOAD_PID}"; then consume_pipe_file_in_background & UPLOAD_PID=$! @@ -108,7 +326,22 @@ function graceful_exit() { echo "exit" >>"${UPLOAD_PIPE_FILE}" & wait "${UPLOAD_PID}" echo "$(date -u +"${ts_format}") [${process_name}] - Uploader consumed all files in the pipe" + + # Wait for all active background rclone uploads to complete + wait_for_active_uploads + + # Log upload statistics from file (written by background processes) + if [ -f "/tmp/upload_stats.txt" ]; then + source "/tmp/upload_stats.txt" 2>/dev/null || true + fi + local total_uploads=$((upload_success_count + upload_failed_count)) + echo "$(date -u +"${ts_format}") [${process_name}] - Upload Statistics:" + echo "$(date -u +"${ts_format}") [${process_name}] - Total uploads attempted: ${total_uploads}" + echo "$(date -u +"${ts_format}") [${process_name}] - Successful uploads: ${upload_success_count}" + echo "$(date -u +"${ts_format}") [${process_name}] - Failed uploads: ${upload_failed_count}" + rm -rf "${FORCE_EXIT_FILE}" + rm -rf "/tmp/upload_stats.txt" echo "$(date -u +"${ts_format}") [${process_name}] - Uploader is ready to shutdown" exit 0 } @@ -117,12 +350,28 @@ rename_rclone_env trap graceful_exit SIGTERM SIGINT EXIT while true; do + # Exit main loop if graceful shutdown is in progress + if [ "$graceful_exit_called" = "true" ]; then + break + fi + wait_until_pipefile_exists + + # Exit main loop if graceful shutdown is in progress + if [ "$graceful_exit_called" = "true" ]; then + break + fi + if ! check_if_pid_alive "${UPLOAD_PID}"; then consume_pipe_file_in_background & UPLOAD_PID=$! fi + while check_if_pid_alive "${UPLOAD_PID}"; do + # Exit main loop if graceful shutdown is in progress + if [ "$graceful_exit_called" = "true" ]; then + break 2 # Break out of both while loops + fi sleep 1 done done diff --git a/Video/uploader.conf b/Video/uploader.conf index 0a3162073..bd63fbfe2 100644 --- a/Video/uploader.conf +++ b/Video/uploader.conf @@ -6,7 +6,6 @@ autostart=%(ENV_SE_VIDEO_UPLOAD_ENABLED)s startsecs=0 autorestart=%(ENV_SE_VIDEO_UPLOAD_ENABLED)s stopsignal=TERM -stopwaitsecs=30 ;Logs (all activity redirected to stdout so it can be seen through "docker logs" redirect_stderr=true diff --git a/Video/video.sh b/Video/video.sh index 22c683ccd..0ef25009e 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -18,6 +18,12 @@ poll_interval=${SE_VIDEO_POLL_INTERVAL:-2} max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50} file_ready_max_attempts=${SE_VIDEO_FILE_READY_WAIT_ATTEMPTS:-5} wait_uploader_shutdown_max_attempts=${SE_VIDEO_WAIT_UPLOADER_SHUTDOWN_ATTEMPTS:-5} +graceful_stop_delay=${SE_VIDEO_GRACEFUL_STOP_DELAY:-5} +ffmpeg_stop_timeout=${SE_VIDEO_FFMPEG_STOP_TIMEOUT:-10} +file_stability_wait=${SE_VIDEO_FILE_STABILITY_WAIT:-2} +min_recording_duration=${SE_VIDEO_MIN_RECORDING_DURATION:-5} +video_validation_enabled=${SE_VIDEO_VALIDATION_ENABLED:-"true"} +video_fix_corrupted=${SE_VIDEO_FIX_CORRUPTED:-"true"} ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"} process_name="video.recorder" @@ -33,6 +39,7 @@ else NODE_STATUS_ENDPOINT="${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status" fi +BASIC_AUTH="Authorization: Basic YWRtaW46YWRtaW4=" if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then BASIC_AUTH="$(echo -en "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64 -w0)" BASIC_AUTH="Authorization: Basic ${BASIC_AUTH}" @@ -125,6 +132,51 @@ function wait_util_uploader_shutdown() { fi } +function wait_for_pipe_to_drain() { + if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] || [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then + return 0 + fi + + local wait_count=0 + local max_drain_wait=${SE_VIDEO_WAIT_PIPE_DRAIN:-10} + + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for upload pipe to drain before shutdown" + + # Give uploader time to process queued files + while [[ ${wait_count} -lt ${max_drain_wait} ]]; do + local upload_active=false + + # Check for active rclone/upload processes + if [[ "${VIDEO_INTERNAL_UPLOAD}" = "true" ]]; then + # For internal upload, check rclone processes + local rclone_count=$(pgrep rclone 2>/dev/null | wc -l) + if [[ ${rclone_count} -gt 0 ]]; then + upload_active=true + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${rclone_count} active rclone upload(s) to complete (${wait_count}/${max_drain_wait})" + fi + else + # For external upload, just give some time for pipe to be consumed + # We can't check external container processes, so use a fixed wait + if [[ ${wait_count} -lt 3 ]]; then + upload_active=true + echo "$(date -u +"${ts_format}") [${process_name}] - Giving external uploader time to process queue (${wait_count}/3)" + fi + fi + + if [[ "${upload_active}" = "false" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Upload pipe drained, no active uploads" + break + fi + + sleep ${poll_interval} + wait_count=$((wait_count + 1)) + done + + if [[ ${wait_count} -ge ${max_drain_wait} ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: Pipe drain timeout reached, some uploads may not complete" + fi +} + function send_exit_signal_to_uploader() { if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then echo "$(date -u +"${ts_format}") [${process_name}] - Sending a signal to force exit the uploader" @@ -140,12 +192,92 @@ function exit_on_max_session_reach() { fi } +function wait_for_file_stability() { + local video_file="$1" + local session_id_param="$2" + + if [[ ! -f "${video_file}" ]]; then + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${file_stability_wait}s for file to stabilize: ${video_file}" + + local prev_size=$(stat -f%z "${video_file}" 2>/dev/null || stat -c%s "${video_file}" 2>/dev/null) + sleep ${file_stability_wait} + local curr_size=$(stat -f%z "${video_file}" 2>/dev/null || stat -c%s "${video_file}" 2>/dev/null) + + if [[ "${prev_size}" != "${curr_size}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: File size changed during stability check (${prev_size} -> ${curr_size}), waiting longer" + sleep ${file_stability_wait} + fi + + # Force filesystem sync + sync + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] File stable at size: ${curr_size} bytes" + return 0 +} + +function validate_mp4_file() { + local video_file="$1" + local session_id_param="$2" + + if [[ "${video_validation_enabled}" != "true" ]]; then + return 0 + fi + + if [[ ! -f "${video_file}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file does not exist: ${video_file}" + return 1 + fi + + # Check if file has moov atom using ffmpeg probe + if ffmpeg -v error -i "${video_file}" -f null - 2>&1 | grep -q "moov atom not found"; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file missing moov atom: ${video_file}" + return 1 + fi + + # Quick validation: check if ffmpeg can read the file + if ! ffmpeg -v error -i "${video_file}" -t 0.1 -f null - 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file validation failed: ${video_file}" + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file validation passed: ${video_file}" + return 0 +} + +function attempt_fix_mp4_file() { + local video_file="$1" + local session_id_param="$2" + + if [[ "${video_fix_corrupted}" != "true" ]]; then + return 1 + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Attempting to fix corrupted MP4 file: ${video_file}" + + local temp_file="${video_file}.temp.mp4" + + # Try to recover the file by remuxing with faststart + if ffmpeg -v warning -i "${video_file}" -c copy -movflags +faststart "${temp_file}" 2>/dev/null; then + mv "${temp_file}" "${video_file}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Successfully fixed MP4 file: ${video_file}" + return 0 + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Failed to fix MP4 file: ${video_file}" + rm -f "${temp_file}" + return 1 + fi +} + function stop_ffmpeg() { while true; do FFMPEG_PID=$(pgrep -f "ffmpeg -hide_banner" | tr '\n' ' ') if [ -n "$FFMPEG_PID" ]; then + # Single SIGTERM for graceful shutdown - allows FFmpeg to write moov atom kill -SIGTERM $FFMPEG_PID - wait $FFMPEG_PID + wait $FFMPEG_PID 2>/dev/null fi if ! pgrep -f "ffmpeg -hide_banner" >/dev/null; then break @@ -154,18 +286,118 @@ function stop_ffmpeg() { done } +function stop_ffmpeg_graceful_async() { + local video_file_to_finalize="$1" + local video_file_name_param="$2" + local upload_dest_prefix="$3" + local upload_pipe_file="$4" + local should_upload="$5" + local session_id_param="$6" + + ( + # Send single SIGTERM to FFmpeg for graceful shutdown + # This allows FFmpeg to properly write the moov atom (MP4 metadata) + FFMPEG_PID=$(pgrep -f "ffmpeg -hide_banner" | tr '\n' ' ') + if [ -n "$FFMPEG_PID" ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Sending SIGTERM to FFmpeg PID: $FFMPEG_PID for file: ${video_file_name_param}" + kill -SIGTERM $FFMPEG_PID + + # Wait for FFmpeg to finish writing with timeout + local wait_count=0 + while kill -0 $FFMPEG_PID 2>/dev/null && [ $wait_count -lt ${ffmpeg_stop_timeout} ]; do + sleep 1 + wait_count=$((wait_count + 1)) + done + + if kill -0 $FFMPEG_PID 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: FFmpeg still running after ${ffmpeg_stop_timeout}s, forcing kill" + kill -SIGKILL $FFMPEG_PID 2>/dev/null + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] FFmpeg exited gracefully after ${wait_count}s" + fi + + wait $FFMPEG_PID 2>/dev/null + + # Grace period for metadata finalization and filesystem sync + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay}s for video metadata finalization" + sleep ${graceful_stop_delay} + + # Verify file exists and wait for stability + if [[ -f "${video_file_to_finalize}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file finalized: ${video_file_to_finalize}" + + # Wait for file to stabilize (no more writes) + wait_for_file_stability "${video_file_to_finalize}" "${session_id_param}" + + # Validate MP4 file integrity + if validate_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then + # File is valid, proceed with upload + if [[ "${should_upload}" = "true" ]] && [[ -n "${upload_dest_prefix}" ]] && [[ -n "${upload_pipe_file}" ]]; then + upload_destination="${upload_dest_prefix}/${video_file_name_param}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Add to pipe a signal Uploading video to ${upload_destination}" + echo "${video_file_to_finalize} ${upload_dest_prefix}" >>"${upload_pipe_file}" + fi + else + # Validation failed, attempt to fix + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video validation failed, attempting recovery" + if attempt_fix_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then + # Fixed successfully, re-validate and upload + if validate_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then + if [[ "${should_upload}" = "true" ]] && [[ -n "${upload_dest_prefix}" ]] && [[ -n "${upload_pipe_file}" ]]; then + upload_destination="${upload_dest_prefix}/${video_file_name_param}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Add to pipe a signal Uploading recovered video to ${upload_destination}" + echo "${video_file_to_finalize} ${upload_dest_prefix}" >>"${upload_pipe_file}" + fi + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file still corrupted after recovery attempt, skipping upload" + fi + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] ERROR: Video file recovery failed, skipping upload" + fi + fi + else + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: Video file not found after finalization: ${video_file_to_finalize}" + fi + fi + ) & + local bg_pid=$! + background_finalization_pids+=($bg_pid) + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] FFmpeg graceful stop initiated in background (PID: ${bg_pid}) for file: ${video_file_name_param}" +} + function stop_recording() { - stop_ffmpeg - echo "$(date -u +"${ts_format}") [${process_name}] - Video recording stopped" + local use_async=${1:-true} + + if [[ "${use_async}" = "true" ]]; then + # Async stop - allows immediate start of new recording + # Upload will be triggered AFTER FFmpeg finalization in the background + local should_upload="false" + if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then + should_upload="true" + elif [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." + fi + + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Stopping video recording for file: ${video_file_name} (async mode)" + stop_ffmpeg_graceful_async "${video_file}" "${video_file_name}" "${UPLOAD_DESTINATION_PREFIX}" "${UPLOAD_PIPE_FILE}" "${should_upload}" "${prev_session_id}" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Video recording stopped (async mode - new session can start immediately)" + else + # Sync stop - wait for FFmpeg to fully stop, then upload immediately + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Stopping video recording for file: ${video_file_name} (sync mode)" + stop_ffmpeg + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Video recording stopped (sync mode)" + + if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then + upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Add to pipe a signal Uploading video to $upload_destination" + echo "$video_file ${UPLOAD_DESTINATION_PREFIX}" >>${UPLOAD_PIPE_FILE} & + elif [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." + fi + fi + recorded_count=$((recorded_count + 1)) recording_started="false" - if [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then - upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} - echo "$(date -u +"${ts_format}") [${process_name}] - Add to pipe a signal Uploading video to $upload_destination" - echo "$video_file ${UPLOAD_DESTINATION_PREFIX}" >>${UPLOAD_PIPE_FILE} & - elif [[ "${VIDEO_UPLOAD_ENABLED}" = "true" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." - fi } function check_if_ffmpeg_running() { @@ -193,8 +425,9 @@ function wait_for_file_integrity() { } function stop_if_recording_inprogress() { + local use_async=${1:-false} if [[ "$recording_started" = "true" ]] || check_if_ffmpeg_running; then - stop_recording + stop_recording "${use_async}" fi } @@ -204,9 +437,24 @@ function log_node_response() { fi } +function wait_for_background_finalization() { + if [ ${#background_finalization_pids[@]} -gt 0 ]; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for ${#background_finalization_pids[@]} background finalization process(es) to complete" + for pid in "${background_finalization_pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for background finalization process (PID: $pid)" + wait "$pid" 2>/dev/null || true + fi + done + echo "$(date -u +"${ts_format}") [${process_name}] - All background finalization processes completed" + fi +} + function graceful_exit() { echo "$(date -u +"${ts_format}") [${process_name}] - Trapped SIGTERM/SIGINT/x so shutting down recorder" stop_if_recording_inprogress + wait_for_background_finalization + wait_for_pipe_to_drain send_exit_signal_to_uploader wait_util_uploader_shutdown } @@ -232,8 +480,10 @@ if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] && [[ "${VIDEO_FILE_NAME}" != "auto ffmpeg -hide_banner -loglevel warning -threads ${SE_FFMPEG_THREADS:-1} -thread_queue_size 512 \ -probesize 32M -analyzeduration 0 -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} \ -i ${DISPLAY} ${SE_AUDIO_SOURCE} -codec:v ${CODEC} ${PRESET:-"-preset veryfast"} \ - -tune zerolatency -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ - -pix_fmt yuv420p -movflags +faststart "$video_file" & + -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ + -pix_fmt yuv420p -g ${SE_VIDEO_GOP_SIZE:-60} -keyint_min ${SE_VIDEO_KEYINT_MIN:-30} \ + -sc_threshold 0 -force_key_frames "expr:gte(t,n_forced*2)" \ + -movflags +faststart+frag_keyframe+empty_moov+default_base_moof "$video_file" & FFMPEG_PID=$! if ps -p $FFMPEG_PID >/dev/null; then wait $FFMPEG_PID @@ -250,32 +500,36 @@ else attempts=0 max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} recorded_count=0 + # Track background finalization processes + declare -a background_finalization_pids wait_for_api_respond while curl --noproxy "*" -H "${BASIC_AUTH}" -sk --request GET ${NODE_STATUS_ENDPOINT} >"/tmp/status.json"; do session_id="$(jq -r "${JQ_SESSION_ID_QUERY}" "/tmp/status.json")" if [[ "$session_id" != "null" && "$session_id" != "" && "$session_id" != "reserved" && "$recording_started" = "false" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Session: $session_id is created" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] New session detected" session_capabilities="$(jq -r "${JQ_SESSION_CAPABILITIES_QUERY}" "/tmp/status.json")" return_list=($(python3 "${VIDEO_CONFIG_DIRECTORY}/video_nodeQuery.py" "${session_id}" "${session_capabilities}")) caps_se_video_record="${return_list[0]}" video_file_name="${return_list[1]}.mp4" if [[ "$caps_se_video_record" = "true" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Start recording: $caps_se_video_record, video file name: $video_file_name" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] Start recording enabled, video file: ${video_file_name}" log_node_response video_file="${VIDEO_FOLDER}/$video_file_name" - echo "$(date -u +"${ts_format}") [${process_name}] - Starting to record video" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] Starting FFmpeg to record video" ffmpeg -hide_banner -loglevel warning -threads ${SE_FFMPEG_THREADS:-1} -thread_queue_size 512 \ -probesize 32M -analyzeduration 0 -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} \ -i ${DISPLAY} ${SE_AUDIO_SOURCE} -codec:v ${CODEC} ${PRESET:-"-preset veryfast"} \ - -tune zerolatency -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ - -pix_fmt yuv420p -movflags +faststart "$video_file" & + -crf ${SE_VIDEO_CRF:-28} -maxrate ${SE_VIDEO_MAXRATE:-1000k} -bufsize ${SE_VIDEO_BUFSIZE:-2000k} \ + -pix_fmt yuv420p -g ${SE_VIDEO_GOP_SIZE:-60} -keyint_min ${SE_VIDEO_KEYINT_MIN:-30} \ + -sc_threshold 0 -force_key_frames "expr:gte(t,n_forced*2)" \ + -movflags +faststart+frag_keyframe+empty_moov+default_base_moof "$video_file" & FFMPEG_PID=$! if ps -p $FFMPEG_PID >/dev/null; then recording_started="true" prev_session_id=$session_id fi - echo "$(date -u +"${ts_format}") [${process_name}] - Video recording started" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id}] Video recording started (FFmpeg PID: ${FFMPEG_PID})" sleep ${poll_interval} fi elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]]; then @@ -285,7 +539,7 @@ else exit fi elif [[ $recording_started = "true" ]]; then - echo "$(date -u +"${ts_format}") [${process_name}] - Video recording in progress" + echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${prev_session_id}] Video recording in progress" sleep ${poll_interval} else sleep ${poll_interval} diff --git a/docker-compose-v3-video-upload-standalone-arm64.yml b/docker-compose-v3-video-upload-standalone-arm64.yml new file mode 100644 index 000000000..f918866eb --- /dev/null +++ b/docker-compose-v3-video-upload-standalone-arm64.yml @@ -0,0 +1,79 @@ +# To execute this docker compose yml file use `docker compose -f docker-compose-v3-video-upload-standalone-arm64.yml up` +# Add the `-d` flag at the end for detached execution +# To stop the execution, hit Ctrl+C, and then `docker compose -f docker-compose-v3-video-upload-standalone-arm64.yml down` +# ${variable_pattern} get value from .env in the same directory +services: + # Start a local FTP server to demonstrate video upload with RCLONE (https://github.com/delfer/docker-alpine-ftp-server) + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + volumes: + # Mount the local directory `/home/${USER}/Videos/upload` to the FTP server's `/ftp/seluser` directory to check out the uploaded videos + - /tmp/upload:/ftp/seluser + command: ["/bin/sh", "-c", "/sbin/tini -- /bin/start_vsftpd.sh && tail -f /dev/null"] + stop_grace_period: 30s + + # File browser to manage the uploaded videos from the FTP server + file_browser: + image: filebrowser/filebrowser:latest + container_name: file_browser + restart: always + ports: + - "8081:80" + volumes: + # Mount the local directory `/tmp/upload` to file browser's `/srv` directory to check out the uploaded videos + - /tmp/upload:/srv + environment: + - FB_NOAUTH=true + + standalone_chrome: + image: selenium/standalone-chromium:4.38.0-20251101 + shm_size: 2gb + ports: + - "4444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_RECORD_VIDEO=true + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s + + standalone_firefox: + image: selenium/standalone-firefox:4.38.0-20251101 + shm_size: 2gb + ports: + - "5444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_RECORD_VIDEO=true + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s diff --git a/tests/get_started.py b/tests/get_started.py index 488a22bf6..15bfc9545 100644 --- a/tests/get_started.py +++ b/tests/get_started.py @@ -41,7 +41,7 @@ def run_browser_instance(browser, grid_url): print(f"Session created: {driver.session_id} ({browser})") driver.get('https://www.google.com/') print(driver.title) - time.sleep(100) + time.sleep(10) driver.quit()