Skip to content

Commit 1278a2a

Browse files
VietND96VietND96
committed
Fix file recording broken
Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent 6a788e2 commit 1278a2a

File tree

2 files changed

+118
-11
lines changed

2 files changed

+118
-11
lines changed

‎Video/upload.sh‎

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ VIDEO_INTERNAL_UPLOAD=${VIDEO_INTERNAL_UPLOAD:-$SE_VIDEO_INTERNAL_UPLOAD}
1010
VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"}
1111
UPLOAD_RETRY_MAX_ATTEMPTS=${SE_UPLOAD_RETRY_MAX_ATTEMPTS:-3}
1212
UPLOAD_RETRY_DELAY=${SE_UPLOAD_RETRY_DELAY:-5}
13-
UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-1}
13+
UPLOAD_FILE_READY_WAIT=${SE_UPLOAD_FILE_READY_WAIT:-3}
14+
UPLOAD_FILE_STABILITY_RETRIES=${SE_UPLOAD_FILE_STABILITY_RETRIES:-5}
1415
UPLOAD_VERIFY_CHECKSUM=${SE_UPLOAD_VERIFY_CHECKSUM:-"true"}
16+
UPLOAD_VALIDATE_MP4=${SE_UPLOAD_VALIDATE_MP4:-"true"}
1517
ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"%Y-%m-%d %H:%M:%S,%3N"}
1618
process_name="video.uploader"
1719

@@ -49,6 +51,36 @@ upload_success_count=0
4951
upload_failed_count=0
5052
graceful_exit_called=false
5153

54+
function validate_mp4_moov_atom() {
55+
local file=$1
56+
57+
if [[ "${UPLOAD_VALIDATE_MP4}" != "true" ]]; then
58+
return 0
59+
fi
60+
61+
# Only validate MP4/M4V files
62+
if [[ ! "${file}" =~ \.(mp4|m4v)$ ]]; then
63+
return 0
64+
fi
65+
66+
# Check if file has moov atom using ffmpeg probe
67+
local error_output=$(ffmpeg -v error -i "${file}" -f null - 2>&1)
68+
69+
if echo "${error_output}" | grep -q "moov atom not found"; then
70+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: MP4 file missing moov atom: ${file}"
71+
return 1
72+
fi
73+
74+
# Quick validation: check if ffmpeg can read the file
75+
if ! ffmpeg -v error -i "${file}" -t 0.1 -f null - 2>/dev/null; then
76+
echo "$(date -u +"${ts_format}") [${process_name}] - ERROR: MP4 file validation failed: ${file}"
77+
return 1
78+
fi
79+
80+
echo "$(date -u +"${ts_format}") [${process_name}] - MP4 validation passed: ${file}"
81+
return 0
82+
}
83+
5284
function verify_file_ready() {
5385
local file=$1
5486

@@ -71,13 +103,44 @@ function verify_file_ready() {
71103
return 1
72104
fi
73105

74-
# Wait for file to be stable (no longer being written)
75-
local initial_size=${file_size}
76-
sleep ${UPLOAD_FILE_READY_WAIT}
77-
local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null)
106+
# Wait for file to be stable (no longer being written) with retries
107+
echo "$(date -u +"${ts_format}") [${process_name}] - Waiting for file to stabilize: ${file}"
108+
local retry=0
109+
local stable=false
110+
111+
while [ ${retry} -lt ${UPLOAD_FILE_STABILITY_RETRIES} ]; do
112+
local initial_size=${file_size}
113+
sleep ${UPLOAD_FILE_READY_WAIT}
114+
local final_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null)
115+
116+
if [ "${initial_size}" = "${final_size}" ]; then
117+
# Size is stable, force filesystem sync and verify again
118+
sync
119+
sleep 1
120+
local verify_size=$(stat -f%z "${file}" 2>/dev/null || stat -c%s "${file}" 2>/dev/null)
121+
122+
if [ "${final_size}" = "${verify_size}" ]; then
123+
stable=true
124+
echo "$(date -u +"${ts_format}") [${process_name}] - File size stable at ${final_size} bytes after ${retry} retries"
125+
break
126+
fi
127+
fi
128+
129+
retry=$((retry + 1))
130+
file_size=${final_size}
131+
132+
if [ ${retry} -lt ${UPLOAD_FILE_STABILITY_RETRIES} ]; then
133+
echo "$(date -u +"${ts_format}") [${process_name}] - File size changed (${initial_size} -> ${final_size}), waiting more (retry ${retry}/${UPLOAD_FILE_STABILITY_RETRIES})"
134+
fi
135+
done
136+
137+
if [ "${stable}" != "true" ]; then
138+
echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File did not stabilize after ${UPLOAD_FILE_STABILITY_RETRIES} retries: ${file}"
139+
return 1
140+
fi
78141

79-
if [ "${initial_size}" != "${final_size}" ]; then
80-
echo "$(date -u +"${ts_format}") [${process_name}] - WARNING: File is still being written: ${file} (size changed from ${initial_size} to ${final_size})"
142+
# Validate MP4 moov atom if enabled
143+
if ! validate_mp4_moov_atom "${file}"; then
81144
return 1
82145
fi
83146

‎Video/video.sh‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50}
1919
file_ready_max_attempts=${SE_VIDEO_FILE_READY_WAIT_ATTEMPTS:-5}
2020
wait_uploader_shutdown_max_attempts=${SE_VIDEO_WAIT_UPLOADER_SHUTDOWN_ATTEMPTS:-5}
2121
graceful_stop_delay=${SE_VIDEO_GRACEFUL_STOP_DELAY:-5}
22+
ffmpeg_stop_timeout=${SE_VIDEO_FFMPEG_STOP_TIMEOUT:-10}
23+
file_stability_wait=${SE_VIDEO_FILE_STABILITY_WAIT:-2}
2224
min_recording_duration=${SE_VIDEO_MIN_RECORDING_DURATION:-5}
2325
video_validation_enabled=${SE_VIDEO_VALIDATION_ENABLED:-"true"}
2426
video_fix_corrupted=${SE_VIDEO_FIX_CORRUPTED:-"true"}
@@ -190,6 +192,32 @@ function exit_on_max_session_reach() {
190192
fi
191193
}
192194

195+
function wait_for_file_stability() {
196+
local video_file="$1"
197+
local session_id_param="$2"
198+
199+
if [[ ! -f "${video_file}" ]]; then
200+
return 1
201+
fi
202+
203+
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${file_stability_wait}s for file to stabilize: ${video_file}"
204+
205+
local prev_size=$(stat -f%z "${video_file}" 2>/dev/null || stat -c%s "${video_file}" 2>/dev/null)
206+
sleep ${file_stability_wait}
207+
local curr_size=$(stat -f%z "${video_file}" 2>/dev/null || stat -c%s "${video_file}" 2>/dev/null)
208+
209+
if [[ "${prev_size}" != "${curr_size}" ]]; then
210+
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: File size changed during stability check (${prev_size} -> ${curr_size}), waiting longer"
211+
sleep ${file_stability_wait}
212+
fi
213+
214+
# Force filesystem sync
215+
sync
216+
217+
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] File stable at size: ${curr_size} bytes"
218+
return 0
219+
}
220+
193221
function validate_mp4_file() {
194222
local video_file="$1"
195223
local session_id_param="$2"
@@ -274,17 +302,33 @@ function stop_ffmpeg_graceful_async() {
274302
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Sending SIGTERM to FFmpeg PID: $FFMPEG_PID for file: ${video_file_name_param}"
275303
kill -SIGTERM $FFMPEG_PID
276304

277-
# Wait for FFmpeg to finish writing
305+
# Wait for FFmpeg to finish writing with timeout
306+
local wait_count=0
307+
while kill -0 $FFMPEG_PID 2>/dev/null && [ $wait_count -lt ${ffmpeg_stop_timeout} ]; do
308+
sleep 1
309+
wait_count=$((wait_count + 1))
310+
done
311+
312+
if kill -0 $FFMPEG_PID 2>/dev/null; then
313+
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] WARNING: FFmpeg still running after ${ffmpeg_stop_timeout}s, forcing kill"
314+
kill -SIGKILL $FFMPEG_PID 2>/dev/null
315+
else
316+
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] FFmpeg exited gracefully after ${wait_count}s"
317+
fi
318+
278319
wait $FFMPEG_PID 2>/dev/null
279320

280-
# Grace period for metadata finalization
281-
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay} seconds for video metadata finalization"
321+
# Grace period for metadata finalization and filesystem sync
322+
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Waiting ${graceful_stop_delay}s for video metadata finalization"
282323
sleep ${graceful_stop_delay}
283324

284-
# Verify file exists
325+
# Verify file exists and wait for stability
285326
if [[ -f "${video_file_to_finalize}" ]]; then
286327
echo "$(date -u +"${ts_format}") [${process_name}] - [Session: ${session_id_param}] Video file finalized: ${video_file_to_finalize}"
287328

329+
# Wait for file to stabilize (no more writes)
330+
wait_for_file_stability "${video_file_to_finalize}" "${session_id_param}"
331+
288332
# Validate MP4 file integrity
289333
if validate_mp4_file "${video_file_to_finalize}" "${session_id_param}"; then
290334
# File is valid, proceed with upload

0 commit comments

Comments
 (0)