From 234e3259ccad590361dc57587ce1b52a5f4d5168 Mon Sep 17 00:00:00 2001 From: Mior Muhammad Zaki Date: Thu, 30 Nov 2023 12:25:25 +0800 Subject: [PATCH 1/3] [10.x] Fixes retrying failed jobs causes PHP memory exhaustion errors when dealing with thousands of failed jobs fixes laravel/framework#49185 Signed-off-by: Mior Muhammad Zaki --- src/Illuminate/Queue/Console/RetryCommand.php | 18 +++++++++++++----- .../Queue/Failed/DatabaseFailedJobProvider.php | 15 +++++++++++++++ .../Failed/DatabaseUuidFailedJobProvider.php | 16 ++++++++++++++++ .../Queue/Failed/DynamoDbFailedJobProvider.php | 14 ++++++++++++++ .../Failed/FailedJobProviderInterface.php | 3 +++ .../Queue/Failed/FileFailedJobProvider.php | 14 ++++++++++++++ .../Queue/Failed/NullFailedJobProvider.php | 11 +++++++++++ 7 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/Illuminate/Queue/Console/RetryCommand.php b/src/Illuminate/Queue/Console/RetryCommand.php index 37827dd01683..8fc6f8bc9278 100644 --- a/src/Illuminate/Queue/Console/RetryCommand.php +++ b/src/Illuminate/Queue/Console/RetryCommand.php @@ -70,7 +70,11 @@ protected function getJobIds() $ids = (array) $this->argument('id'); if (count($ids) === 1 && $ids[0] === 'all') { - return Arr::pluck($this->laravel['queue.failer']->all(), 'id'); + $failer = $this->laravel['queue.failer']; + + return method_exists($failer, 'keys') + ? $failer->keys() + : Arr::pluck($failer->all(), 'id'); } if ($queue = $this->option('queue')) { @@ -92,10 +96,14 @@ protected function getJobIds() */ protected function getJobIdsByQueue($queue) { - $ids = collect($this->laravel['queue.failer']->all()) - ->where('queue', $queue) - ->pluck('id') - ->toArray(); + $failer = $this->laravel['queue.failer']; + + $ids = method_exists($failer, 'keys') + ? $failer->keys($queue) + : collect($failer->all()) + ->where('queue', $queue) + ->pluck('id') + ->toArray(); if (count($ids) === 0) { $this->components->error("Unable to find failed jobs for queue [{$queue}]."); diff --git a/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php b/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php index 4da8c0337c2f..5b6e21acb562 100644 --- a/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php @@ -64,6 +64,21 @@ public function log($connection, $queue, $payload, $exception) )); } + /** + * Get a list of all of IDs the failed jobs. + * + * @param string|null $queue + * @return array + */ + public function keys($queue = null) + { + return $this->getTable() + ->when(! is_null($queue), fn ($query) => $query->where('queue', $queue)) + ->orderBy('id', 'desc') + ->pluck('id') + ->all(); + } + /** * Get a list of all of the failed jobs. * diff --git a/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php b/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php index f51c46f571b0..98da430439e4 100644 --- a/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php @@ -67,6 +67,22 @@ public function log($connection, $queue, $payload, $exception) return $uuid; } + + /** + * Get a list of all of IDs the failed jobs. + * + * @param string|null $queue + * @return array + */ + public function keys($queue = null) + { + return $this->getTable() + ->when(! is_null($queue), fn ($query) => $query->where('queue', $queue)) + ->orderBy('id', 'desc') + ->pluck('uuid') + ->all(); + } + /** * Get a list of all of the failed jobs. * diff --git a/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php b/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php index 2feeaa98a29f..c72f7bccbf38 100644 --- a/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php @@ -78,6 +78,20 @@ public function log($connection, $queue, $payload, $exception) return $id; } + /** + * Get a list of all of IDs the failed jobs. + * + * @param string|null $queue + * @return array + */ + public function keys($queue = null) + { + return collect($this->all()) + ->when(! is_null($queue), fn ($collect) => $collect->where('queue', $queue)) + ->pluck('id') + ->all(); + } + /** * Get a list of all of the failed jobs. * diff --git a/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php b/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php index 26dd583b9496..37b13bd492e2 100644 --- a/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php +++ b/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php @@ -2,6 +2,9 @@ namespace Illuminate\Queue\Failed; +/** + * @method array keys(string $queue = null) + */ interface FailedJobProviderInterface { /** diff --git a/src/Illuminate/Queue/Failed/FileFailedJobProvider.php b/src/Illuminate/Queue/Failed/FileFailedJobProvider.php index f79efc05f39f..a7fb0d79fb5a 100644 --- a/src/Illuminate/Queue/Failed/FileFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/FileFailedJobProvider.php @@ -78,6 +78,20 @@ public function log($connection, $queue, $payload, $exception) }); } + /** + * Get a list of all of IDs the failed jobs. + * + * @param string|null $queue + * @return array + */ + public function keys($queue = null) + { + return collect($this->all()) + ->when(! is_null($queue), fn ($collect) => $collect->where('queue', $queue)) + ->pluck('id') + ->all(); + } + /** * Get a list of all of the failed jobs. * diff --git a/src/Illuminate/Queue/Failed/NullFailedJobProvider.php b/src/Illuminate/Queue/Failed/NullFailedJobProvider.php index a086134c98f3..a8cbda8dbc92 100644 --- a/src/Illuminate/Queue/Failed/NullFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/NullFailedJobProvider.php @@ -18,6 +18,17 @@ public function log($connection, $queue, $payload, $exception) // } + /** + * Get a list of all of IDs the failed jobs. + * + * @param string|null $queue + * @return array + */ + public function keys($queue = null) + { + return []; + } + /** * Get a list of all of the failed jobs. * From 9f3fb21672ea519fa02eb9548f9348464e1f7c57 Mon Sep 17 00:00:00 2001 From: StyleCI Bot Date: Thu, 30 Nov 2023 04:57:03 +0000 Subject: [PATCH 2/3] Apply fixes from StyleCI --- src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php b/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php index 98da430439e4..f3d044786a62 100644 --- a/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php @@ -67,7 +67,6 @@ public function log($connection, $queue, $payload, $exception) return $uuid; } - /** * Get a list of all of IDs the failed jobs. * From 403c708b506b2341fed9c63dcc30eb4a88136ef5 Mon Sep 17 00:00:00 2001 From: Taylor Otwell Date: Thu, 30 Nov 2023 11:45:39 -0600 Subject: [PATCH 3/3] formatting --- src/Illuminate/Queue/Console/RetryCommand.php | 8 ++++---- src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php | 4 ++-- .../Queue/Failed/DatabaseUuidFailedJobProvider.php | 4 ++-- src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php | 4 ++-- .../Queue/Failed/FailedJobProviderInterface.php | 2 +- src/Illuminate/Queue/Failed/FileFailedJobProvider.php | 4 ++-- src/Illuminate/Queue/Failed/NullFailedJobProvider.php | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Illuminate/Queue/Console/RetryCommand.php b/src/Illuminate/Queue/Console/RetryCommand.php index 8fc6f8bc9278..13640feb4efb 100644 --- a/src/Illuminate/Queue/Console/RetryCommand.php +++ b/src/Illuminate/Queue/Console/RetryCommand.php @@ -72,8 +72,8 @@ protected function getJobIds() if (count($ids) === 1 && $ids[0] === 'all') { $failer = $this->laravel['queue.failer']; - return method_exists($failer, 'keys') - ? $failer->keys() + return method_exists($failer, 'ids') + ? $failer->ids() : Arr::pluck($failer->all(), 'id'); } @@ -98,8 +98,8 @@ protected function getJobIdsByQueue($queue) { $failer = $this->laravel['queue.failer']; - $ids = method_exists($failer, 'keys') - ? $failer->keys($queue) + $ids = method_exists($failer, 'ids') + ? $failer->ids($queue) : collect($failer->all()) ->where('queue', $queue) ->pluck('id') diff --git a/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php b/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php index 5b6e21acb562..49cb3b98ae9a 100644 --- a/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DatabaseFailedJobProvider.php @@ -65,12 +65,12 @@ public function log($connection, $queue, $payload, $exception) } /** - * Get a list of all of IDs the failed jobs. + * Get the IDs of all of the failed jobs. * * @param string|null $queue * @return array */ - public function keys($queue = null) + public function ids($queue = null) { return $this->getTable() ->when(! is_null($queue), fn ($query) => $query->where('queue', $queue)) diff --git a/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php b/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php index f3d044786a62..b3192f246beb 100644 --- a/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DatabaseUuidFailedJobProvider.php @@ -68,12 +68,12 @@ public function log($connection, $queue, $payload, $exception) } /** - * Get a list of all of IDs the failed jobs. + * Get the IDs of all of the failed jobs. * * @param string|null $queue * @return array */ - public function keys($queue = null) + public function ids($queue = null) { return $this->getTable() ->when(! is_null($queue), fn ($query) => $query->where('queue', $queue)) diff --git a/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php b/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php index c72f7bccbf38..226c752e94ef 100644 --- a/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/DynamoDbFailedJobProvider.php @@ -79,12 +79,12 @@ public function log($connection, $queue, $payload, $exception) } /** - * Get a list of all of IDs the failed jobs. + * Get the IDs of all of the failed jobs. * * @param string|null $queue * @return array */ - public function keys($queue = null) + public function ids($queue = null) { return collect($this->all()) ->when(! is_null($queue), fn ($collect) => $collect->where('queue', $queue)) diff --git a/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php b/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php index 37b13bd492e2..bb52d1749ada 100644 --- a/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php +++ b/src/Illuminate/Queue/Failed/FailedJobProviderInterface.php @@ -3,7 +3,7 @@ namespace Illuminate\Queue\Failed; /** - * @method array keys(string $queue = null) + * @method array ids(string $queue = null) */ interface FailedJobProviderInterface { diff --git a/src/Illuminate/Queue/Failed/FileFailedJobProvider.php b/src/Illuminate/Queue/Failed/FileFailedJobProvider.php index a7fb0d79fb5a..43f43248deb1 100644 --- a/src/Illuminate/Queue/Failed/FileFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/FileFailedJobProvider.php @@ -79,12 +79,12 @@ public function log($connection, $queue, $payload, $exception) } /** - * Get a list of all of IDs the failed jobs. + * Get the IDs of all of the failed jobs. * * @param string|null $queue * @return array */ - public function keys($queue = null) + public function ids($queue = null) { return collect($this->all()) ->when(! is_null($queue), fn ($collect) => $collect->where('queue', $queue)) diff --git a/src/Illuminate/Queue/Failed/NullFailedJobProvider.php b/src/Illuminate/Queue/Failed/NullFailedJobProvider.php index a8cbda8dbc92..f92c59eba658 100644 --- a/src/Illuminate/Queue/Failed/NullFailedJobProvider.php +++ b/src/Illuminate/Queue/Failed/NullFailedJobProvider.php @@ -19,12 +19,12 @@ public function log($connection, $queue, $payload, $exception) } /** - * Get a list of all of IDs the failed jobs. + * Get the IDs of all of the failed jobs. * * @param string|null $queue * @return array */ - public function keys($queue = null) + public function ids($queue = null) { return []; }