From 0a9757b9bcf986f1da11e472515ea2ec751760e7 Mon Sep 17 00:00:00 2001 From: mszulik <69617961+mszulik@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:04:30 +0200 Subject: [PATCH] fix an issue where trying to upload large transcoded videos to the cloud would fail (#68) * remove obsolete CloudStorage helper * create a pr image when labeling with pullpreview * add debug logging for video transcoding --- .github/workflows/docker.yml | 2 + .phpstorm.meta.php | 11 --- _ide_helper.php | 99 ++++++++++++------- app/Classes/MediaHandler/VideoHandler.php | 2 + app/Classes/Transcode.php | 1 + app/Facades/CloudStorageFacade.php | 18 ---- .../PhpFfmpegVideoStreaming/S3Helper.php | 63 ------------ .../Controllers/V1/UploadSlotController.php | 2 + app/Interfaces/CloudStorageInterface.php | 25 ----- app/Jobs/TranscodeVideo.php | 88 +++++++++++------ app/Providers/CloudStorageServiceProvider.php | 29 ------ config/app.php | 2 - config/transmorpher.php | 10 -- 13 files changed, 130 insertions(+), 222 deletions(-) delete mode 100644 app/Facades/CloudStorageFacade.php delete mode 100644 app/Helpers/PhpFfmpegVideoStreaming/S3Helper.php delete mode 100644 app/Interfaces/CloudStorageInterface.php delete mode 100644 app/Providers/CloudStorageServiceProvider.php diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 87802124..ba1d4928 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,6 +3,8 @@ name: Docker Image CI on: release: types: [ published ] + pull_request: + types: [ labeled, synchronize, closed ] jobs: build-push-docker-image: diff --git a/.phpstorm.meta.php b/.phpstorm.meta.php index b46c7636..59bc2a7e 100644 --- a/.phpstorm.meta.php +++ b/.phpstorm.meta.php @@ -176,7 +176,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -394,7 +393,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -612,7 +610,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -830,7 +827,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -1048,7 +1044,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -1266,7 +1261,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -1484,7 +1478,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -1702,7 +1695,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -1920,7 +1912,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -2138,7 +2129,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, @@ -2356,7 +2346,6 @@ 'cache.psr6' => \Symfony\Component\Cache\Adapter\Psr16Adapter::class, 'cache.store' => \Illuminate\Cache\Repository::class, 'cdn' => \App\Helpers\CloudFrontHelper::class, - 'cloud.storage' => \App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, 'command.ide-helper.eloquent' => \Barryvdh\LaravelIdeHelper\Console\EloquentCommand::class, 'command.ide-helper.generate' => \Barryvdh\LaravelIdeHelper\Console\GeneratorCommand::class, 'command.ide-helper.meta' => \Barryvdh\LaravelIdeHelper\Console\MetaCommand::class, diff --git a/_ide_helper.php b/_ide_helper.php index 1c360ae7..ac6286e6 100644 --- a/_ide_helper.php +++ b/_ide_helper.php @@ -5,7 +5,7 @@ /** * A helper file for Laravel, to provide autocomplete information to your IDE - * Generated for Laravel 11.5.0. + * Generated for Laravel 11.7.0. * * This file should not be included in your code, only analyzed by your IDE! * @@ -18036,34 +18036,6 @@ /** * * - */ class CloudStorageFacade { - /** - * Returns the configuration for opening data from the cloud storage. - * - * @param string $key - * @return array - * @static - */ public static function getOpenConfiguration($key) - { - /** @var \App\Helpers\PhpFfmpegVideoStreaming\S3Helper $instance */ - return $instance->getOpenConfiguration($key); - } - /** - * Returns the configuration for saving data to the cloud storage. - * - * @param string $destinationPath - * @param string $fileName - * @return array - * @static - */ public static function getSaveConfiguration($destinationPath, $fileName) - { - /** @var \App\Helpers\PhpFfmpegVideoStreaming\S3Helper $instance */ - return $instance->getSaveConfiguration($destinationPath, $fileName); - } - } - /** - * - * */ class TranscodeFacade { /** * Returns the class which handles the actual transcoding. @@ -18853,10 +18825,10 @@ * * * @static - */ public static function registerErrorHandler() + */ public static function registerErrorHandler($errorLevels = null) { /** @var \Spatie\FlareClient\Flare $instance */ - return $instance->registerErrorHandler(); + return $instance->registerErrorHandler($errorLevels); } /** * @@ -18924,10 +18896,19 @@ * * * @static - */ public static function report($throwable, $callback = null, $report = null) + */ public static function report($throwable, $callback = null, $report = null, $handled = null) + { + /** @var \Spatie\FlareClient\Flare $instance */ + return $instance->report($throwable, $callback, $report, $handled); + } + /** + * + * + * @static + */ public static function reportHandled($throwable) { /** @var \Spatie\FlareClient\Flare $instance */ - return $instance->report($throwable, $callback, $report); + return $instance->reportHandled($throwable); } /** * @@ -21581,6 +21562,57 @@ class Eloquent extends \Illuminate\Database\Eloquent\Model { { /** @var \Illuminate\Database\Query\Builder $instance */ return $instance->orWhereJsonDoesntContain($column, $value); + } + /** + * Add a "where JSON overlaps" clause to the query. + * + * @param string $column + * @param mixed $value + * @param string $boolean + * @param bool $not + * @return \Illuminate\Database\Query\Builder + * @static + */ public static function whereJsonOverlaps($column, $value, $boolean = 'and', $not = false) + { + /** @var \Illuminate\Database\Query\Builder $instance */ + return $instance->whereJsonOverlaps($column, $value, $boolean, $not); + } + /** + * Add an "or where JSON overlaps" clause to the query. + * + * @param string $column + * @param mixed $value + * @return \Illuminate\Database\Query\Builder + * @static + */ public static function orWhereJsonOverlaps($column, $value) + { + /** @var \Illuminate\Database\Query\Builder $instance */ + return $instance->orWhereJsonOverlaps($column, $value); + } + /** + * Add a "where JSON not overlap" clause to the query. + * + * @param string $column + * @param mixed $value + * @param string $boolean + * @return \Illuminate\Database\Query\Builder + * @static + */ public static function whereJsonDoesntOverlap($column, $value, $boolean = 'and') + { + /** @var \Illuminate\Database\Query\Builder $instance */ + return $instance->whereJsonDoesntOverlap($column, $value, $boolean); + } + /** + * Add an "or where JSON not overlap" clause to the query. + * + * @param string $column + * @param mixed $value + * @return \Illuminate\Database\Query\Builder + * @static + */ public static function orWhereJsonDoesntOverlap($column, $value) + { + /** @var \Illuminate\Database\Query\Builder $instance */ + return $instance->orWhereJsonDoesntOverlap($column, $value); } /** * Add a clause that determines if a JSON path exists to the query. @@ -22717,7 +22749,6 @@ class Validator extends \Illuminate\Support\Facades\Validator {} class View extends \Illuminate\Support\Facades\View {} class Vite extends \Illuminate\Support\Facades\Vite {} class CdnHelper extends \App\Facades\CdnHelperFacade {} - class CloudStorage extends \App\Facades\CloudStorageFacade {} class InterventionImage extends \Intervention\Image\Facades\Image {} class Transcode extends \App\Facades\TranscodeFacade {} class Transform extends \App\Facades\TransformFacade {} diff --git a/app/Classes/MediaHandler/VideoHandler.php b/app/Classes/MediaHandler/VideoHandler.php index a93efb39..81a54341 100644 --- a/app/Classes/MediaHandler/VideoHandler.php +++ b/app/Classes/MediaHandler/VideoHandler.php @@ -27,7 +27,9 @@ class VideoHandler implements MediaHandlerInterface */ public function handleSavedFile(string $basePath, UploadSlot $uploadSlot, Version $version): ResponseState { + \Log::info(sprintf('Dispatching transcoding job for media %s and version %s.', $version->Media->identifier, $version->getKey())); $success = Transcode::createJob($version, $uploadSlot); + \Log::info(sprintf('Transcoding job dispatched with result %s for media %s and version %s.', $success, $version->Media->identifier, $version->getKey())); return $success ? ResponseState::VIDEO_UPLOAD_SUCCESSFUL : ResponseState::TRANSCODING_JOB_DISPATCH_FAILED; } diff --git a/app/Classes/Transcode.php b/app/Classes/Transcode.php index 49712199..03d2a63c 100644 --- a/app/Classes/Transcode.php +++ b/app/Classes/Transcode.php @@ -96,6 +96,7 @@ public function callback(ResponseState $responseState, string $uploadToken, Medi $signedNotification = SodiumHelper::sign(json_encode($notification)); + \Log::info(sprintf('Sending signed notification with state %s to client package for media %s and version %s, message: %s.', $responseState->getState()->value, $media->identifier, Version::whereNumber($versionNumber)->first()?->getKey(), $responseState->getMessage())); Http::post($media->User->api_url, ['signed_notification' => $signedNotification]); } } diff --git a/app/Facades/CloudStorageFacade.php b/app/Facades/CloudStorageFacade.php deleted file mode 100644 index d4a0e8a8..00000000 --- a/app/Facades/CloudStorageFacade.php +++ /dev/null @@ -1,18 +0,0 @@ - $this->getS3(), - 'options' => [ - 'Bucket' => config(sprintf('filesystems.disks.%s.bucket', config('transmorpher.disks.originals'))), - 'Key' => $key, - ], - ]; - } - - /** - * Returns the configuration for saving data to the cloud storage. - * - * @param string $destinationPath - * @param string $fileName - * - * @return array - */ - public function getSaveConfiguration(string $destinationPath, string $fileName): array - { - return [ - 'cloud' => $this->getS3(), - 'options' => [ - 'dest' => sprintf('s3://%s/%s', - config(sprintf('filesystems.disks.%s.bucket', config('transmorpher.disks.videoDerivatives'))), - $destinationPath - ), - 'filename' => $fileName, - ], - ]; - } - - protected function getS3(): S3 - { - return new S3( - [ - 'version' => 'latest', - 'region' => config('transmorpher.aws.region'), - 'credentials' => [ - 'key' => config('transmorpher.aws.key'), - 'secret' => config('transmorpher.aws.secret'), - ], - ] - ); - } -} diff --git a/app/Http/Controllers/V1/UploadSlotController.php b/app/Http/Controllers/V1/UploadSlotController.php index af31198e..8383e4bd 100644 --- a/app/Http/Controllers/V1/UploadSlotController.php +++ b/app/Http/Controllers/V1/UploadSlotController.php @@ -102,8 +102,10 @@ protected function saveFile(UploadedFile $uploadedFile, UploadSlot $uploadSlot, $version->update(['filename' => $version->createOriginalFileName($uploadedFile->getClientOriginalName())]); if (MediaStorage::ORIGINALS->getDisk()->putFileAs($basePath, $uploadedFile, $version->filename)) { + \Log::info(sprintf('File for media %s and version %s saved successfully.', $media->identifier, $version->number)); $responseState = $type->handler()->handleSavedFile($basePath, $uploadSlot, $version); } else { + \Log::error(sprintf('Could not write file for media %s and version %s.', $media->identifier, $version->number)); $responseState = ResponseState::WRITE_FAILED; } diff --git a/app/Interfaces/CloudStorageInterface.php b/app/Interfaces/CloudStorageInterface.php deleted file mode 100644 index bb95b8d6..00000000 --- a/app/Interfaces/CloudStorageInterface.php +++ /dev/null @@ -1,25 +0,0 @@ -onQueue('video-transcoding'); + \Log::info(sprintf('Constructing job for media %s and version %s with uploadToken %s.', $version->Media->identifier, $version->getKey(), $uploadSlot->token)); $this->originalFilePath = $version->originalFilePath(); $this->uploadToken = $this->uploadSlot->token; } @@ -83,12 +78,12 @@ public function __construct( */ public function handle(): void { + \Log::info(sprintf('Transcoding video for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); // Check for newer versions and validity of upload slot. if ($this->isMostRecentVersion()) { $this->originalsDisk = MediaStorage::ORIGINALS->getDisk(); $this->derivativesDisk = MediaStorage::VIDEO_DERIVATIVES->getDisk(); $this->localDisk = Storage::disk('local'); - $this->tempMp4Filename = $this->getTempMp4Filename(); $this->tempOriginalFilename = $this->getTempOriginalFilename(); $this->transcodeVideo(); @@ -96,6 +91,7 @@ public function handle(): void $this->responseState = ResponseState::TRANSCODING_ABORTED; } + \Log::info(sprintf('Transcoding finished for media %s and version %s with response state %s.', $this->version->Media->identifier, $this->version->getKey(), $this->responseState->value)); match ($this->responseState) { ResponseState::TRANSCODING_SUCCESSFUL => Transcode::callback($this->responseState, $this->uploadToken, $this->version->Media, $this->version->number), ResponseState::TRANSCODING_ABORTED => $this->failed(null), @@ -111,14 +107,16 @@ public function handle(): void public function failed(?Throwable $exception): void { // All properties have not yet been initialized, because failed jobs use a new instance. + \Log::error(sprintf('Transcoding video for media %s and version %s failed. Exception: %s', $this->version->Media->identifier, $this->version->getKey(), $exception?->getMessage())); - $tempDerivativesDirectoryPath = $this->getTempVideoDerivativesDirectoryPath(); $localDisk = Storage::disk('local'); - MediaStorage::VIDEO_DERIVATIVES->getDisk()->deleteDirectory($tempDerivativesDirectoryPath); + MediaStorage::VIDEO_DERIVATIVES->getDisk()->deleteDirectory($this->getTempDerivativesDirectoryPath()); $localDisk->delete($this->getTempMp4Filename()); $localDisk->delete($this->getTempOriginalFilename()); $localDisk->deleteDirectory($this->getFfmpegTempDirectory()); + // This directory stores local temp derivatives in case cloud storage is used. + $localDisk->deleteDirectory($this->getTempDerivativesDirectoryPath()); if (!$this->oldVersionNumber) { // A failed upload must not create a version. @@ -145,11 +143,13 @@ protected function transcodeVideo(): void 'temporary_directory' => $this->localDisk->path($this->getFfmpegTempDirectory()) ]); + \Log::info(sprintf('Downloading video for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); $video = $this->loadVideo($ffmpeg); // Set the necessary file path information. $this->setFilePaths(); + // Generate MP4. $this->generateMp4($video); // Generate HLS @@ -159,6 +159,7 @@ protected function transcodeVideo(): void $this->localDisk->delete($this->tempOriginalFilename); $this->localDisk->deleteDirectory($this->getFfmpegTempDirectory()); + $this->localDisk->deleteDirectory($this->getTempDerivativesDirectoryPath()); // Derivatives are generated at this point of time and located in the temporary folder. $this->moveDerivativesToDestinationPath(); @@ -211,7 +212,6 @@ protected function openFromCloud(FFMpeg $ffmpeg): StreamingMedia protected function setFilePaths(): void { $this->derivativesDestinationPath = $this->version->Media->baseDirectory(); - $this->tempDerivativesDirectoryPath = $this->getTempVideoDerivativesDirectoryPath(); } /** @@ -224,14 +224,28 @@ protected function setFilePaths(): void */ protected function saveVideo(Streaming $video, string $format): void { + $tempDerivativeFilePath = $this->getTempDerivativeFilePath($format); + + \Log::info(sprintf('Generating %s for media %s and version %s.', $format, $this->version->Media->identifier, $this->version->getKey())); // Save to temporary folder first, to prevent race conditions when multiple versions are uploaded simultaneously. - $this->isLocalFilesystem($this->derivativesDisk) ? - $video->save($this->derivativesDisk->path($this->getTempVideoDerivativeFilePath($format))) - : $video->save(null, - CloudStorage::getSaveConfiguration( - sprintf('%s/%s', $this->derivativesDisk->path($this->tempDerivativesDirectoryPath), $format), 'video' - ) - ); + if ($this->isLocalFilesystem($this->derivativesDisk)) { + $video->save($this->derivativesDisk->path($tempDerivativeFilePath)); + } else { + // When using cloud storage, we save to local storage first and then upload manually, + // because the php-ffmpeg-video-streaming package direct upload functionality led to S3 disconnects for large files. + $video->save($this->localDisk->path($tempDerivativeFilePath)); + + $tempDerivativesFormatDirectoryPath = $this->getTempDerivativesFormatDirectoryPath($format); + + \Log::info(sprintf('Writing %s to S3 for media %s and version %s.', $format, $this->version->Media->identifier, $this->version->getKey())); + foreach ($this->localDisk->allFiles($tempDerivativesFormatDirectoryPath) as $filePath) { + $this->derivativesDisk->writeStream( + $filePath, + $this->localDisk->readStream($filePath)); + } + + $this->localDisk->deleteDirectory($tempDerivativesFormatDirectoryPath); + } } /** @@ -245,15 +259,17 @@ protected function saveVideo(Streaming $video, string $format): void */ protected function generateMp4(StreamingMedia $video): void { - $video->save((new X264())->setAdditionalParameters(config('transmorpher.additional_transcoding_parameters')), $this->localDisk->path($this->tempMp4Filename)); + $tempMp4Filename = $this->getTempMp4Filename(); + \Log::info(sprintf('Generating MP4 for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); + $video->save((new X264())->setAdditionalParameters(config('transmorpher.additional_transcoding_parameters')), $this->localDisk->path($tempMp4Filename)); - $derivativePath = $this->getTempVideoDerivativeFilePath('mp4'); + \Log::info(sprintf('Writing MP4 to S3 for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); $this->derivativesDisk->writeStream( - sprintf('%s.%s', $derivativePath, 'mp4'), - $this->localDisk->readStream($this->tempMp4Filename) + sprintf('%s.%s', $this->getTempDerivativeFilePath('mp4'), 'mp4'), + $this->localDisk->readStream($tempMp4Filename) ); - $this->localDisk->delete($this->tempMp4Filename); + $this->localDisk->delete($tempMp4Filename); } /** @@ -267,17 +283,21 @@ protected function moveDerivativesToDestinationPath(): void if ($this->isMostRecentVersion()) { // This will make sure we can invalidate the cache before the current derivative gets deleted. // If this fails, the job will stop and cleanup will be done in the 'failed()'-method. + \Log::info(sprintf('Invalidating CDN cache for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); $this->invalidateCdnCache(); $this->derivativesDisk->deleteDirectory($this->derivativesDestinationPath); + \Log::info(sprintf('Moving video derivatives for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); $this->moveFromTempDirectory(); // Invalidate the cache again for the newly generated derivative. + \Log::info(sprintf('Invalidating CDN cache after moving for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); $this->invalidateCdnCache(); $this->version->update(['processed' => true]); $this->responseState = ResponseState::TRANSCODING_SUCCESSFUL; } else { + \Log::info(sprintf('Transcoding aborted since not latest version for media %s and version %s.', $this->version->Media->identifier, $this->version->getKey())); $this->responseState = ResponseState::TRANSCODING_ABORTED; } } @@ -290,7 +310,7 @@ protected function moveDerivativesToDestinationPath(): void protected function moveFromTempDirectory(): void { if ($this->isLocalFilesystem($this->derivativesDisk)) { - $this->derivativesDisk->move($this->tempDerivativesDirectoryPath, $this->derivativesDestinationPath); + $this->derivativesDisk->move($this->getTempDerivativesDirectoryPath(), $this->derivativesDestinationPath); } else { $this->moveFromCloudTempDirectory(); } @@ -304,8 +324,8 @@ protected function moveFromTempDirectory(): void */ protected function moveFromCloudTempDirectory(): void { - $hlsFiles = $this->derivativesDisk->allFiles(sprintf('%s/%s/', $this->tempDerivativesDirectoryPath, StreamingFormat::HLS->value)); - $dashFiles = $this->derivativesDisk->allFiles(sprintf('%s/%s/', $this->tempDerivativesDirectoryPath, StreamingFormat::DASH->value)); + $hlsFiles = $this->derivativesDisk->allFiles($this->getTempDerivativesFormatDirectoryPath(StreamingFormat::HLS->value)); + $dashFiles = $this->derivativesDisk->allFiles($this->getTempDerivativesFormatDirectoryPath(StreamingFormat::DASH->value)); foreach ($hlsFiles as $file) { $this->derivativesDisk->move($file, $this->version->Media->videoDerivativeFilePath(StreamingFormat::HLS->value, basename($file))); @@ -315,10 +335,9 @@ protected function moveFromCloudTempDirectory(): void $this->derivativesDisk->move($file, $this->version->Media->videoDerivativeFilePath(StreamingFormat::DASH->value, basename($file))); } - $tempDerivativePath = $this->getTempVideoDerivativeFilePath('mp4'); // Move MP4 file. $this->derivativesDisk->move( - sprintf('%s.mp4', $tempDerivativePath), + sprintf('%s.mp4', $this->getTempDerivativeFilePath('mp4')), sprintf('%s.mp4', $this->version->Media->videoDerivativeFilePath('mp4')) ); } @@ -376,9 +395,9 @@ protected function isMostRecentVersion(): bool * @param string $format * @return string */ - protected function getTempVideoDerivativeFilePath(string $format): string + protected function getTempDerivativeFilePath(string $format): string { - return sprintf('%s/%s/%s', $this->getTempVideoDerivativesDirectoryPath(), $format, 'video'); + return sprintf('%s/%s', $this->getTempDerivativesFormatDirectoryPath($format), 'video'); } /** @@ -387,8 +406,17 @@ protected function getTempVideoDerivativeFilePath(string $format): string * * @return string */ - protected function getTempVideoDerivativesDirectoryPath(): string + protected function getTempDerivativesDirectoryPath(): string { return sprintf('%s-%s-temp', $this->version->Media->baseDirectory(), $this->version->getKey()); } + + /** + * Get the path to the temporary video derivatives directory for a format. + * Path structure: {username}/{identifier}-{versionKey}-temp/{format} + */ + protected function getTempDerivativesFormatDirectoryPath(string $format): string + { + return sprintf('%s/%s', $this->getTempDerivativesDirectoryPath(), $format); + } } diff --git a/app/Providers/CloudStorageServiceProvider.php b/app/Providers/CloudStorageServiceProvider.php deleted file mode 100644 index 38a43b97..00000000 --- a/app/Providers/CloudStorageServiceProvider.php +++ /dev/null @@ -1,29 +0,0 @@ -app->singleton('cloud.storage', fn(): CloudStorageInterface => new (config('transmorpher.cloud_storage_helper'))); - } - - /** - * Bootstrap services. - * - * @return void - */ - public function boot() - { - // - } -} diff --git a/config/app.php b/config/app.php index 717ef2ba..1ffc0a50 100644 --- a/config/app.php +++ b/config/app.php @@ -175,7 +175,6 @@ // App\Providers\BroadcastServiceProvider::class, App\Providers\EventServiceProvider::class, App\Providers\CdnHelperServiceProvider::class, - App\Providers\CloudStorageServiceProvider::class, App\Providers\SqsFifoServiceProvider::class, App\Providers\TranscodeServiceProvider::class, App\Providers\TransformServiceProvider::class, @@ -194,7 +193,6 @@ 'aliases' => Facade::defaultAliases()->merge([ 'CdnHelper' => App\Facades\CdnHelperFacade::class, - 'CloudStorage' => App\Facades\CloudStorageFacade::class, 'InterventionImage' => Intervention\Image\Facades\Image::class, 'Transcode' => App\Facades\TranscodeFacade::class, 'Transform' => App\Facades\TransformFacade::class, diff --git a/config/transmorpher.php b/config/transmorpher.php index 20a88396..8be799b1 100644 --- a/config/transmorpher.php +++ b/config/transmorpher.php @@ -125,16 +125,6 @@ '-dn', '-map', '-0:t?', '-sn' ], - /* - |-------------------------------------------------------------------------- - | Cloud Storage Helper - |-------------------------------------------------------------------------- - | - | Helper for managing cloud storage access when transcoding videos. - | - */ - 'cloud_storage_helper' => App\Helpers\PhpFfmpegVideoStreaming\S3Helper::class, - /* |-------------------------------------------------------------------------- | CDN Helper