Skip to content

Commit

Permalink
Updated env config back to original.
Browse files Browse the repository at this point in the history
Trackable-Job package added to composer as a suggestion.
styleci.
  • Loading branch information
Davis Puciriuss committed May 10, 2024
1 parent cde973a commit 26a7973
Show file tree
Hide file tree
Showing 21 changed files with 71 additions and 63 deletions.
2 changes: 0 additions & 2 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ APP_ENV=development
XDEBUG_CONFIG=remote_port=9001 remote_host=172.17.0.1 remote_autostart=1 remote_log=/home/user/xdebug.log
PHP_IDE_CONFIG=serverName=Scout
ELASTICSEARCH_HOST=elasticsearch:9200
ELASTICSEARCH_USER=elasticsearch
ELASTICSEARCH_PASSWORD=password
DB_HOST=db
DB_PORT=3306
DB_DATABASE=my_database
Expand Down
4 changes: 3 additions & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
"elasticsearch/elasticsearch": "^8.0",
"handcraftedinthealps/elasticsearch-dsl": "^8.0",
"laravel/scout": "^8.0|^9.0|^10.0",
"mateusjunges/laravel-trackable-jobs": "^1.6",
"roave/better-reflection": "^4.3|^5.0|^6.18"
},
"require-dev": {
Expand Down Expand Up @@ -50,6 +49,9 @@
]
}
},
"suggest": {
"mateusjunges/laravel-trackable-jobs": "Required when using --parallel option in ImportCommand."
},
"config": {
"sort-packages": true,
"preferred-install": "dist"
Expand Down
12 changes: 6 additions & 6 deletions src/Console/Commands/ImportCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public function handle(): void
{
$parallel = false;

if($this->option('parallel')) {
if ($this->option('parallel')) {
$parallel = true;
}

Expand All @@ -42,7 +42,7 @@ public function handle(): void
}

/**
* @param array<string> $argument
* @param array<string> $argument
*
* @return Collection<int, string>
*/
Expand All @@ -56,8 +56,8 @@ private function searchableList(array $argument): Collection
}

/**
* @param string $searchable
* @param bool $parallel
* @param string $searchable
* @param bool $parallel
*
* @return void
*/
Expand All @@ -68,7 +68,7 @@ private function import(string $searchable, bool $parallel): void
$job = new Import($source);
/** @var int|null $queueTimeout */
$queueTimeout = Config::queueTimeout();
if($queueTimeout !== null) {
if ($queueTimeout !== null) {
$job->timeout = (int) $queueTimeout;
}
$job->parallel = $parallel;
Expand All @@ -77,7 +77,7 @@ private function import(string $searchable, bool $parallel): void
$job = (new QueueableJob())->chain([$job]);
/** @var int|null $queueTimeout */
$queueTimeout = Config::queueTimeout();
if($queueTimeout !== null) {
if ($queueTimeout !== null) {
$job->timeout = (int) $queueTimeout;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/ElasticSearch/Config/Storage.php
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public function queueTimeout(): ?int
if (is_numeric($queueTimeoutConfig)) {
return intval($queueTimeoutConfig);
}

return null;
}

Expand Down
4 changes: 2 additions & 2 deletions src/ElasticSearch/EloquentHitsIteratorAggregate.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

namespace Matchish\ScoutElasticSearch\ElasticSearch;

use Illuminate\Database\Eloquent\Model;
use IteratorAggregate;
use Laravel\Scout\Builder;
use Traversable;
use Illuminate\Database\Eloquent\Model;
use Laravel\Scout\Searchable;
use Traversable;

/**
* @internal
Expand Down
4 changes: 2 additions & 2 deletions src/ElasticSearch/HitsIteratorAggregate.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
interface HitsIteratorAggregate extends \IteratorAggregate
{
/**
* @param array $results
* @param callable|null $callback
* @param array $results
* @param callable|null $callback
*/
public function __construct(array $results, callable $callback = null);

Expand Down
13 changes: 7 additions & 6 deletions src/Engines/ElasticSearchEngine.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@
namespace Matchish\ScoutElasticSearch\Engines;

use Elastic\Elasticsearch\Client;
use Elastic\Elasticsearch\Response\Elasticsearch;
use Elastic\Elasticsearch\Exception\ServerResponseException;
use Elastic\Elasticsearch\Response\Elasticsearch;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\LazyCollection;
use Laravel\Scout\Builder;
use Laravel\Scout\Builder as BaseBuilder;
use Laravel\Scout\Engines\Engine;
use Laravel\Scout\Searchable;
use Matchish\ScoutElasticSearch\ElasticSearch\HitsIteratorAggregate;
use Matchish\ScoutElasticSearch\ElasticSearch\Params\Bulk;
use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Refresh;
use Matchish\ScoutElasticSearch\ElasticSearch\Params\Search as SearchParams;
use Matchish\ScoutElasticSearch\ElasticSearch\SearchFactory;
use Matchish\ScoutElasticSearch\ElasticSearch\SearchResults;
use Illuminate\Database\Eloquent\Model;
use Laravel\Scout\Searchable;
use ONGR\ElasticsearchDSL\Query\MatchAllQuery;
use ONGR\ElasticsearchDSL\Search;

Expand All @@ -42,7 +42,7 @@ public function __construct(Client $elasticsearch)
}

/**
* @param Collection<int, Model|Searchable> $models
* @param Collection<int, Model|Searchable> $models
*/
public function update($models)
{
Expand All @@ -64,7 +64,7 @@ public function update($models)
}

/**
* @param Collection<int, Model|Searchable> $models
* @param Collection<int, Model|Searchable> $models
*/
public function delete($models)
{
Expand All @@ -74,7 +74,7 @@ public function delete($models)
}

/**
* @param Model|Searchable $model
* @param Model|Searchable $model
*/
public function flush($model)
{
Expand Down Expand Up @@ -111,6 +111,7 @@ public function paginate(BaseBuilder $builder, $perPage, $page)

/**
* {@inheritdoc}
*
* @return Collection<int, int>
*/
public function mapIds($results)
Expand Down
6 changes: 3 additions & 3 deletions src/Jobs/Import.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ final class Import
*/
private $source;
/**
* @var boolean
* @var bool
*/
public $parallel = false;

Expand Down Expand Up @@ -52,8 +52,8 @@ public function handle(Client $elasticsearch): void
$this->progressBar()->setMessage($currentStage->title());
$currentStage->handle($elasticsearch);
$this->progressBar()->advance($currentStage->advance());
if($currentStage->completed()) {
if($stages->isEmpty()) {
if ($currentStage->completed()) {
if ($stages->isEmpty()) {
/** @var null $currentStage */
$currentStage = null;
} else {
Expand Down
4 changes: 2 additions & 2 deletions src/Jobs/ImportStages.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
namespace Matchish\ScoutElasticSearch\Jobs;

use Illuminate\Support\Collection;
use Matchish\ScoutElasticSearch\Jobs\Stages\StageInterface;
use Matchish\ScoutElasticSearch\ElasticSearch\Index;
use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp;
use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex;
use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource;
use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSourceParallel;
use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex;
use Matchish\ScoutElasticSearch\Jobs\Stages\StageInterface;
use Matchish\ScoutElasticSearch\Jobs\Stages\SwitchToNewAndRemoveOldIndex;
use Matchish\ScoutElasticSearch\Searchable\ImportSource;

Expand All @@ -28,7 +28,7 @@ public static function fromSource(ImportSource $source, bool $parallel = false)
{
$index = Index::fromSource($source);

if($parallel) {
if ($parallel && class_exists(\Junges\TrackableJobs\Providers\TrackableJobsServiceProvider::class)) {
return (new self([
new CleanUp($source),
new CreateWriteIndex($source, $index),
Expand Down
9 changes: 4 additions & 5 deletions src/Jobs/ProcessSearchable.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,14 @@

namespace Matchish\ScoutElasticSearch\Jobs;


use Illuminate\Support\Collection;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Junges\TrackableJobs\Concerns\Trackable;
use Illuminate\Database\Eloquent\Model;
use Laravel\Scout\Searchable;

class ProcessSearchable implements ShouldQueue
Expand All @@ -27,7 +26,7 @@ class ProcessSearchable implements ShouldQueue
private $data;

/**
* @param Collection<int, Model|Searchable> $data
* @param Collection<int, Model|Searchable> $data
*/
public function __construct(Collection $data)
{
Expand All @@ -37,7 +36,7 @@ public function __construct(Collection $data)

/**
* Handles the job execution.
*
*
* @return void
*/
public function handle(): void
Expand Down
2 changes: 1 addition & 1 deletion src/Jobs/Stages/CleanUp.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
namespace Matchish\ScoutElasticSearch\Jobs\Stages;

use Elastic\Elasticsearch\Client;
use Elastic\Elasticsearch\Response\Elasticsearch;
use Elastic\Elasticsearch\Exception\ClientResponseException;
use Elastic\Elasticsearch\Response\Elasticsearch;
use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Get as GetAliasParams;
use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Delete as DeleteIndexParams;
use Matchish\ScoutElasticSearch\Searchable\ImportSource;
Expand Down
2 changes: 1 addition & 1 deletion src/Jobs/Stages/PullFromSource.php
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public function completed(): bool
public static function chunked(ImportSource $source): ?PullFromSource
{
$source = $source->chunked();
if($source === null) {
if ($source === null) {
return null;
}
return new static($source);
Expand Down
26 changes: 14 additions & 12 deletions src/Jobs/Stages/PullFromSourceParallel.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@

use Elastic\Elasticsearch\Client;
use Junges\TrackableJobs\Models\TrackedJob;
use Matchish\ScoutElasticSearch\Database\Scopes\PageScope;
use Matchish\ScoutElasticSearch\Database\Scopes\FromScope;
use Matchish\ScoutElasticSearch\Database\Scopes\PageScope;
use Matchish\ScoutElasticSearch\Jobs\ProcessSearchable;
use Matchish\ScoutElasticSearch\Searchable\ImportSource;

Expand Down Expand Up @@ -33,7 +33,7 @@ final class PullFromSourceParallel implements StageInterface
* @var array
*/
private $dispatchedJobIds = [];

/**
* @var int
*/
Expand All @@ -52,7 +52,7 @@ final class PullFromSourceParallel implements StageInterface
public function __construct(ImportSource $source)
{
$this->source = $source;
$this->queues = collect(config('scout.chunk.handlers', self::DEFAULT_HANDLER_COUNT))->map(function($i) {
$this->queues = collect(config('scout.chunk.handlers', self::DEFAULT_HANDLER_COUNT))->map(function ($i) {
return 'import_'.$i;
})->toArray();
}
Expand All @@ -65,6 +65,7 @@ private function getNextQueue(): string
/** @var string $queue */
$queue = array_shift($this->queues);
$this->queues[] = $queue;

return $queue;
}

Expand All @@ -73,22 +74,22 @@ private function getNextQueue(): string
*/
public function handle(Client $elasticsearch = null): void
{
if(count($this->dispatchedJobIds) > 0) {
if (count($this->dispatchedJobIds) > 0) {
$jobs = TrackedJob::findMany($this->dispatchedJobIds);
$finishedJobs = $jobs->filter(function($job) {
$finishedJobs = $jobs->filter(function ($job) {
return $job->status === TrackedJob::STATUS_FINISHED;
});
$this->handledJobs = array_merge($this->handledJobs, $finishedJobs->pluck('id')->toArray());
$this->advanceBy += $finishedJobs->count();
$this->dispatchedJobIds = $jobs->filter(function($job) {
$this->dispatchedJobIds = $jobs->filter(function ($job) {
return $job->status !== TrackedJob::STATUS_FINISHED;
})->pluck('id')->toArray();
}
if(count($this->dispatchedJobIds) >= $this->source->getTotalChunks()) {

if (count($this->dispatchedJobIds) >= $this->source->getTotalChunks()) {
return;
}

$results = $this->source->get()->filter->shouldBeSearchable();

if (! $results->isEmpty()) {
Expand All @@ -98,7 +99,7 @@ public function handle(Client $elasticsearch = null): void
if ($results->first()->getKeyType() !== 'int') {
$this->source->setChunkScope(
new PageScope(
count($this->handledJobs) + count($this->dispatchedJobIds),
count($this->handledJobs) + count($this->dispatchedJobIds),
$this->source->getChunkSize())
);
} else {
Expand All @@ -122,6 +123,7 @@ public function advance(): int
{
$advance = $this->advanceBy;
$this->advanceBy = 0;

return $advance;
}

Expand Down Expand Up @@ -149,10 +151,10 @@ public static function chunked(ImportSource $source): PullFromSourceParallel|nul
{
$source = $source->chunked();

if($source === null) {
if ($source === null) {
return null;
}

return new static($source);
}
}
2 changes: 1 addition & 1 deletion src/Jobs/Stages/StageInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public function advance(): int;
public function completed(): bool;

/**
* @param Client $elasticsearch
* @param Client $elasticsearch
* @return void
*/
public function handle(Client $elasticsearch): void;
Expand Down
Loading

0 comments on commit 26a7973

Please sign in to comment.