From d9b42096296fa30b31930a0545119a2610da3f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Robin?= Date: Wed, 31 Jan 2024 16:44:56 +0100 Subject: [PATCH] Closes #6394: Reintroduce limit preload at job level (#6409) --- inc/Engine/Preload/Database/Queries/Cache.php | 12 +++- .../Cron/Subscriber/processPendingUrls.php | 16 ++--- .../Database/Queries/Cache/getPendingJobs.php | 25 ++++++++ .../Database/Queries/Cache/getPendingJobs.php | 61 ++++++++++++++----- 4 files changed, 90 insertions(+), 24 deletions(-) diff --git a/inc/Engine/Preload/Database/Queries/Cache.php b/inc/Engine/Preload/Database/Queries/Cache.php index 2feb6f6520..82d06fb935 100644 --- a/inc/Engine/Preload/Database/Queries/Cache.php +++ b/inc/Engine/Preload/Database/Queries/Cache.php @@ -338,8 +338,16 @@ public function remove_all_not_accessed_rows( float $delay = 1, string $unit = ' * @return array */ public function get_pending_jobs( int $total = 45 ) { + $inprogress_count = $this->query( + [ + 'count' => true, + 'status' => 'in-progress', + 'is_locked' => false, + ], + false + ); - if ( $total <= 0 ) { + if ( $total <= 0 || $inprogress_count >= $total ) { return []; } @@ -358,7 +366,7 @@ public function get_pending_jobs( int $total = 45 ) { return $this->query( [ - 'number' => $total, + 'number' => ( $total - $inprogress_count ), 'status' => 'pending', 'fields' => [ 'id', diff --git a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php index a4e7a3e1fa..69225f2e07 100644 --- a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php +++ b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php @@ -1,7 +1,7 @@ [ - 'config' => [ + 'pendingShouldAddToTheQueue' => [ + 'config' => [ 'rocket_preload_cache_pending_jobs_cron_rows_count' => 10, 'manual_preload' => true, 'rocket_preload_outdated' => -1, @@ -18,7 +18,7 @@ 'actions' => [ ] - ], + ], 'expected' => [ 'rows' => [ [ @@ -47,7 +47,7 @@ ], ] ] - ], + ], 'InProgressAndInQueueShouldNotFail' => [ 'config' => [ 'rocket_preload_cache_pending_jobs_cron_rows_count' => 10, @@ -120,7 +120,7 @@ ], [ 'url' => 'http://example.org/test3', - 'status' => 'in-progress' + 'status' => 'pending' ], [ 'url' => 'http://example.org/test4', @@ -147,11 +147,11 @@ ], [ 'url' => 'http://example.org/test3', - 'status' => 'in-progress' + 'status' => 'pending' ], [ 'url' => 'http://example.org/test4', - 'status' => 'in-progress' + 'status' => 'pending' ], [ 'url' => 'http://example.org/test5', @@ -181,7 +181,7 @@ ], ], [ - 'exists' => true, + 'exists' => false, 'args' => [ 'hook' => 'rocket_preload_job_preload_url', 'args' => ['http://example.org/test4'] diff --git a/tests/Fixtures/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php b/tests/Fixtures/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php index 93529a3404..1c46cff0fe 100644 --- a/tests/Fixtures/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php +++ b/tests/Fixtures/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php @@ -8,6 +8,7 @@ 'shouldReturnPendingRowsWhenInProgressLessThanTotal' => [ 'config' => [ 'total' => 45, + 'in-progress' => 1, 'results' => [ $cache, ] @@ -16,9 +17,33 @@ $cache ] ], + 'shouldNotReturnEmptyPendingRowsWhenInProgressNotEqualsTotal' => [ + 'config' => [ + 'total' => 45, + 'in-progress' => 44, + 'results' => [ + $cache, + ], + ], + 'expected' => [ + $cache + ] + ], + 'shouldReturnEmptyPendingRowsWhenInProgressEqualsTotal' => [ + 'config' => [ + 'total' => 45, + 'in-progress' => 45, + 'results' => [ + $cache, + ], + ], + 'expected' => [ + ] + ], 'negativeCountShouldReturnEmpty' => [ 'config' => [ 'total' => -10, + 'in-progress' => 35, 'results' => [ $cache, ] diff --git a/tests/Unit/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php b/tests/Unit/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php index a7d8dee2ad..11ccf2c23f 100644 --- a/tests/Unit/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php +++ b/tests/Unit/inc/Engine/Preload/Database/Queries/Cache/getPendingJobs.php @@ -24,22 +24,55 @@ protected function setUp(): void * @dataProvider configTestData */ public function testShouldReturnPending($config, $expected) { + $queue = new SplQueue(); - if($config['total'] > 0) { + if($config['total'] > 0 && $config['in-progress'] < $config['total'] ) { + $queue->enqueue( + [ + 'params' => + [ + 'count' => true, + 'status' => 'in-progress', + 'is_locked' => false, + ], + 'return' => $config['in-progress'] + ] + ); + + $queue->enqueue( + [ + 'params' => + [ + 'number' => $config['total']-$config['in-progress'], + 'status' => 'pending', + 'fields' => [ + 'id', + 'url', + ], + 'job_id__not_in' => [ + 'not_in' => '', + ], + 'orderby' => Filters\applied( 'rocket_preload_order' ) > 0 ? 'id' : 'modified', + 'order' => 'asc', + 'is_locked' => false + ], + 'return' => $config['results'] + ] + ); + + $this->query->expects(self::exactly(2))->method('query') + ->willReturnCallback(function($params) use ($queue) { + $dequeue = $queue->dequeue(); + $this->assertEquals($dequeue['params'], $params); + return $dequeue['return']; + }); + } else { $this->query->expects(self::once())->method('query')->with([ - 'number' => $config['total'], - 'status' => 'pending', - 'fields' => [ - 'id', - 'url', - ], - 'job_id__not_in' => [ - 'not_in' => '', - ], - 'orderby' => Filters\applied( 'rocket_preload_order' ) > 0 ? 'id' : 'modified', - 'order' => 'asc', - 'is_locked' => false - ])->willReturn($config['results']); + 'count' => true, + 'status' => 'in-progress', + 'is_locked' => false, + ])->willReturn($config['in-progress']); + } $this->assertSame($expected, $this->query->get_pending_jobs($config['total']));