From ffe91b48dcbb33545841eb00e6c300589c2563da Mon Sep 17 00:00:00 2001 From: invario <67800603+invario@users.noreply.github.com> Date: Fri, 5 Sep 2025 11:04:05 -0400 Subject: [PATCH 1/3] feat(previews): allow ffmpeg to connect direct for AWS S3 buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Côme Chilliet <91878298+come-nc@users.noreply.github.com> Signed-off-by: invario <67800603+invario@users.noreply.github.com> --- .../lib/Lib/Storage/AmazonS3.php | 31 ++++ lib/private/Preview/Movie.php | 148 +++++++++++------- 2 files changed, 121 insertions(+), 58 deletions(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index 1e834811bd18f..e6896096a9eab 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -760,4 +760,35 @@ public function writeStream(string $path, $stream, ?int $size = null): int { return $size; } + + /** + * Generates and returns a presigned URL that expires after set duration + * + */ + public function getDirectDownload(string $path): array|false { + $command = $this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $path, + ]); + $duration = '+10 minutes'; + $expiration = new \DateTime(); + $expiration->modify($duration); + + // generate a presigned URL that expires after $duration time + $request = $this->getConnection()->createPresignedRequest($command, $duration, []); + try { + $presignedUrl = (string)$request->getUri(); + } catch (S3Exception $exception) { + $this->logger->error($exception->getMessage(), [ + 'app' => 'files_external', + 'exception' => $exception, + ]); + } + $result = [ + 'url' => $presignedUrl, + 'presigned' => true, + 'expiration' => $expiration, + ]; + return $result; + } } diff --git a/lib/private/Preview/Movie.php b/lib/private/Preview/Movie.php index 91857effbe1e0..d9d4135dff16e 100644 --- a/lib/private/Preview/Movie.php +++ b/lib/private/Preview/Movie.php @@ -42,6 +42,26 @@ public function isAvailable(FileInfo $file): bool { return is_string($this->binary); } + private function connectDirect(File $file): string|false { + if (stream_get_meta_data($file->fopen('r'))['seekable'] !== true) { + return false; + } + + // Checks for availability to access the video file directly via HTTP/HTTPS. + // Returns a string containing URL if available. Only implemented and tested + // with Amazon S3 currently. In all other cases, return false. ffmpeg + // supports other protocols so this function may expand in the future. + $gddValues = $file->getStorage()->getDirectDownload($file->getName()); + + if (is_array($gddValues)) { + if (array_key_exists('url', $gddValues) && array_key_exists('presigned', $gddValues)) { + $directUrl = (str_starts_with($gddValues['url'], 'http') && ($gddValues['presigned'] === true)) ? $gddValues['url'] : false; + return $directUrl; + } + } + return false; + } + /** * {@inheritDoc} */ @@ -54,74 +74,87 @@ public function getThumbnail(File $file, int $maxX, int $maxY): ?IImage { $result = null; + $connectDirect = $this->connectDirect($file); + // Timestamps to make attempts to generate a still $timeAttempts = [5, 1, 0]; - // By default, download $sizeAttempts from the file along with - // the 'moov' atom. - // Example bitrates in the higher range: - // 4K HDR H265 60 FPS = 75 Mbps = 9 MB per second needed for a still - // 1080p H265 30 FPS = 10 Mbps = 1.25 MB per second needed for a still - // 1080p H264 30 FPS = 16 Mbps = 2 MB per second needed for a still - $sizeAttempts = [1024 * 1024 * 10]; - - if ($this->useTempFile($file)) { - if ($file->getStorage()->isLocal()) { - // Temp file required but file is local, so retrieve $sizeAttempt bytes first, - // and if it doesn't work, retrieve the entire file. - $sizeAttempts[] = null; + // If HTTP/HTTPS direct connect is not available or if the file is encrypted, + // process normally + if (($connectDirect === false) || $file->isEncrypted()) { + // By default, download $sizeAttempts from the file along with + // the 'moov' atom. + // Example bitrates in the higher range: + // 4K HDR H265 60 FPS = 75 Mbps = 9 MB per second needed for a still + // 1080p H265 30 FPS = 10 Mbps = 1.25 MB per second needed for a still + // 1080p H264 30 FPS = 16 Mbps = 2 MB per second needed for a still + $sizeAttempts = [1024 * 1024 * 10]; + + if ($this->useTempFile($file)) { + if ($file->getStorage()->isLocal()) { + // Temp file required but file is local, so retrieve $sizeAttempt bytes first, + // and if it doesn't work, retrieve the entire file. + $sizeAttempts[] = null; + } + } else { + // Temp file is not required and file is local so retrieve entire file. + $sizeAttempts = [null]; } - } else { - // Temp file is not required and file is local so retrieve entire file. - $sizeAttempts = [null]; - } - foreach ($sizeAttempts as $size) { - $absPath = false; - // File is remote, generate a sparse file - if (!$file->getStorage()->isLocal()) { - $absPath = $this->getSparseFile($file, $size); - } - // Defaults to existing routine if generating sparse file fails - if ($absPath === false) { - $absPath = $this->getLocalFile($file, $size); - } - if ($absPath === false) { - Server::get(LoggerInterface::class)->error( - 'Failed to get local file to generate thumbnail for: ' . $file->getPath(), - ['app' => 'core'] - ); - return null; - } + foreach ($sizeAttempts as $size) { + $absPath = false; + // File is remote, generate a sparse file + if (!$file->getStorage()->isLocal()) { + $absPath = $this->getSparseFile($file, $size); + } + // Defaults to existing routine if generating sparse file fails + if ($absPath === false) { + $absPath = $this->getLocalFile($file, $size); + } + if ($absPath === false) { + Server::get(LoggerInterface::class)->error( + 'Failed to get local file to generate thumbnail for: ' . $file->getPath(), + ['app' => 'core'] + ); + return null; + } + + // Attempt still image grabs from selected timestamps + foreach ($timeAttempts as $timeStamp) { + $result = $this->generateThumbNail($maxX, $maxY, $absPath, $timeStamp); + if ($result !== null) { + break; + } + Server::get(LoggerInterface::class)->debug( + 'Movie preview generation attempt failed' + . ', file=' . $file->getPath() + . ', time=' . $timeStamp + . ', size=' . ($size ?? 'entire file'), + ['app' => 'core'] + ); + } + + $this->cleanTmpFiles(); - // Attempt still image grabs from selected timestamps - foreach ($timeAttempts as $timeStamp) { - $result = $this->generateThumbNail($maxX, $maxY, $absPath, $timeStamp); if ($result !== null) { + Server::get(LoggerInterface::class)->debug( + 'Movie preview generation attempt success' + . ', file=' . $file->getPath() + . ', time=' . $timeStamp + . ', size=' . ($size ?? 'entire file'), + ['app' => 'core'] + ); break; } - Server::get(LoggerInterface::class)->debug( - 'Movie preview generation attempt failed' - . ', file=' . $file->getPath() - . ', time=' . $timeStamp - . ', size=' . ($size ?? 'entire file'), - ['app' => 'core'] - ); } - - $this->cleanTmpFiles(); - - if ($result !== null) { - Server::get(LoggerInterface::class)->debug( - 'Movie preview generation attempt success' - . ', file=' . $file->getPath() - . ', time=' . $timeStamp - . ', size=' . ($size ?? 'entire file'), - ['app' => 'core'] - ); - break; + } else { + // HTTP/HTTPS direct connect is available so pass the URL directly to ffmpeg + foreach ($timeAttempts as $timeStamp) { + $result = $this->generateThumbNail($maxX, $maxY, $connectDirect, $timeStamp); + if ($result !== null) { + break; + } } - } if ($result === null) { Server::get(LoggerInterface::class)->error( @@ -330,7 +363,6 @@ private function generateThumbNail(int $maxX, int $maxY, string $absPath, int $s } } - unlink($tmpPath); return null; } From 2ea1bd4cdc5d61706ad76f594aa817bd1b6b129f Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 2 Jan 2026 11:05:10 +0100 Subject: [PATCH 2/3] feat(movie-preview): Use getDirectDownloadById for generating preview Allow to speed-up considerably the creation of previews for movies stored on S3. Signed-off-by: Carl Schwan --- .../lib/Lib/Storage/AmazonS3.php | 38 ++++++++++++------- .../Files/ObjectStore/S3ObjectTrait.php | 8 ++-- lib/private/Preview/Movie.php | 15 +++----- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index e6896096a9eab..fe7e31a13565d 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -23,6 +23,7 @@ use OCP\ICacheFactory; use OCP\ITempManager; use OCP\Server; +use Override; use Psr\Log\LoggerInterface; class AmazonS3 extends Common { @@ -761,34 +762,43 @@ public function writeStream(string $path, $stream, ?int $size = null): int { return $size; } - /** - * Generates and returns a presigned URL that expires after set duration - * - */ + #[Override] public function getDirectDownload(string $path): array|false { + if (!$this->isUsePresignedUrl()) { + return false; + } + $command = $this->getConnection()->getCommand('GetObject', [ 'Bucket' => $this->bucket, 'Key' => $path, ]); - $duration = '+10 minutes'; - $expiration = new \DateTime(); - $expiration->modify($duration); + $expiration = new \DateTimeImmutable('+60 minutes'); - // generate a presigned URL that expires after $duration time - $request = $this->getConnection()->createPresignedRequest($command, $duration, []); try { - $presignedUrl = (string)$request->getUri(); + // generate a presigned URL that expires after $expiration time + $presignedUrl = (string)$this->getConnection()->createPresignedRequest($command, $expiration, [ + 'signPayload' => true, + ])->getUri(); } catch (S3Exception $exception) { $this->logger->error($exception->getMessage(), [ 'app' => 'files_external', 'exception' => $exception, ]); + return false; } - $result = [ + return [ 'url' => $presignedUrl, - 'presigned' => true, - 'expiration' => $expiration, + 'expiration' => $expiration->getTimestamp(), ]; - return $result; + } + + #[Override] + public function getDirectDownloadById(string $fileId): array|false { + if (!$this->isUsePresignedUrl()) { + return false; + } + + $entry = $this->getCache()->get((int)$fileId); + return $this->getDirectDownload($entry->getPath()); } } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 91eff90babb88..b6b55c746b2c9 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -298,15 +298,15 @@ public function copyObject($from, $to, array $options = []) { } public function preSignedUrl(string $urn, \DateTimeInterface $expiration): ?string { + if (!$this->isUsePresignedUrl()) { + return null; + } + $command = $this->getConnection()->getCommand('GetObject', [ 'Bucket' => $this->getBucket(), 'Key' => $urn, ]); - if (!$this->isUsePresignedUrl()) { - return null; - } - try { return (string)$this->getConnection()->createPresignedRequest($command, $expiration, [ 'signPayload' => true, diff --git a/lib/private/Preview/Movie.php b/lib/private/Preview/Movie.php index d9d4135dff16e..00648e7526e27 100644 --- a/lib/private/Preview/Movie.php +++ b/lib/private/Preview/Movie.php @@ -43,21 +43,18 @@ public function isAvailable(FileInfo $file): bool { } private function connectDirect(File $file): string|false { - if (stream_get_meta_data($file->fopen('r'))['seekable'] !== true) { + if ($file->isEncrypted()) { return false; } // Checks for availability to access the video file directly via HTTP/HTTPS. // Returns a string containing URL if available. Only implemented and tested - // with Amazon S3 currently. In all other cases, return false. ffmpeg + // with Amazon S3 currently. In all other cases, return false. ffmpeg // supports other protocols so this function may expand in the future. - $gddValues = $file->getStorage()->getDirectDownload($file->getName()); + $gddValues = $file->getStorage()->getDirectDownloadById((string)$file->getId()); - if (is_array($gddValues)) { - if (array_key_exists('url', $gddValues) && array_key_exists('presigned', $gddValues)) { - $directUrl = (str_starts_with($gddValues['url'], 'http') && ($gddValues['presigned'] === true)) ? $gddValues['url'] : false; - return $directUrl; - } + if (is_array($gddValues) && array_key_exists('url', $gddValues)) { + return str_starts_with($gddValues['url'], 'http') ? $gddValues['url'] : false; } return false; } @@ -81,7 +78,7 @@ public function getThumbnail(File $file, int $maxX, int $maxY): ?IImage { // If HTTP/HTTPS direct connect is not available or if the file is encrypted, // process normally - if (($connectDirect === false) || $file->isEncrypted()) { + if ($connectDirect === false) { // By default, download $sizeAttempts from the file along with // the 'moov' atom. // Example bitrates in the higher range: From 98192fc17c458e78009aab95a493bf14ef9d622b Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 2 Jan 2026 11:46:54 +0100 Subject: [PATCH 3/3] feat(external-s3): Expose config option to enable direct s3 preview Signed-off-by: Carl Schwan --- apps/files_external/lib/Lib/Backend/AmazonS3.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/files_external/lib/Lib/Backend/AmazonS3.php b/apps/files_external/lib/Lib/Backend/AmazonS3.php index 0da93ad0d509e..9eb479e03f010 100644 --- a/apps/files_external/lib/Lib/Backend/AmazonS3.php +++ b/apps/files_external/lib/Lib/Backend/AmazonS3.php @@ -28,6 +28,8 @@ public function __construct(IL10N $l, AccessKey $legacyAuth) { ->setFlag(DefinitionParameter::FLAG_OPTIONAL), (new DefinitionParameter('port', $l->t('Port'))) ->setFlag(DefinitionParameter::FLAG_OPTIONAL), + (new DefinitionParameter('proxy', $l->t('Proxy'))) + ->setFlag(DefinitionParameter::FLAG_OPTIONAL), (new DefinitionParameter('region', $l->t('Region'))) ->setFlag(DefinitionParameter::FLAG_OPTIONAL), (new DefinitionParameter('storageClass', $l->t('Storage Class'))) @@ -42,6 +44,9 @@ public function __construct(IL10N $l, AccessKey $legacyAuth) { (new DefinitionParameter('useMultipartCopy', $l->t('Enable multipart copy'))) ->setType(DefinitionParameter::VALUE_BOOLEAN) ->setDefaultValue(true), + (new DefinitionParameter('use_presigned_url', $l->t('Use presigned S3 url'))) + ->setType(DefinitionParameter::VALUE_BOOLEAN) + ->setDefaultValue(false), (new DefinitionParameter('sse_c_key', $l->t('SSE-C encryption key'))) ->setType(DefinitionParameter::VALUE_PASSWORD) ->setFlag(DefinitionParameter::FLAG_OPTIONAL),