Merge pull request #275 from DavisPuciriuss/improve-import-speed

Parallel import with trackable jobs.

.github/workflows/test-application.yaml

@@ -65,6 +65,7 @@ jobs:
                 run: |
                     composer validate --strict
                     composer install --no-interaction --prefer-dist
+                    composer require mateusjunges/laravel-trackable-jobs --no-interaction
 
             -   name: Run tests
                 run: vendor/bin/phpunit --coverage-clover=coverage.xml

.gitignore

@@ -3,4 +3,5 @@
 composer.lock
 .env
 .phpunit.result.cache
-/vendor/
+/vendor/
+/build/

CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/)
 
 ## [Unreleased]
 
+## [8.0.0-alpha.1] - 2024-05-29
+### Added
+- fromScope, uses forPageAfterId.
+- a new option "--parallel" for ImportCommand, can only be used with [Trackable-Jobs](https://github.com/mateusjunges/trackable-jobs-for-laravel) package currently.
+
+## Changed
+- StageInterface now has two more functions. Both are used to make PullFromSource stage an iterable one.
+- ImportSource interface now has three more functions. Chunk scope can now be set from a stage.
+- When possible (model without a custom key), by default fromScope is used instead of pageScope.
+
 ## [7.6.1] - 2024-05-14
 ### Fixed
 - fix for [parser incompatibility](https://github.com/matchish/laravel-scout-elasticsearch/issues/273)

Makefile

@@ -27,24 +27,28 @@ up: ## Start all containers (in background) for development
 	$(docker_compose_bin) up -d
 
 down: ## Stop all started for development containers
-	$(docker_compose_bin) down
+	$(docker_compose_bin) down -v
 
 restart: up ## Restart all started for development containers
 	$(docker_compose_bin) restart
 
+rebuild: down
+	$(docker_compose_bin) up --build -d
+
 shell: up ## Start shell into application container
 	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" /bin/sh
 
 install: up ## Install application dependencies into application container
 	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" composer install --no-interaction --ansi
 
-test: up ## Execute application tests
+larastan: up
 	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" ./vendor/bin/phpstan analyze --memory-limit=4000M
+
+test: up ## Execute application tests 
 	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" ./vendor/bin/phpunit --testdox --stop-on-failure
 
 test-coverage: up ## Execute application tests and generate report
-	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" ./vendor/bin/phpstan analyze
 	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" ./vendor/bin/phpunit  --coverage-html build/coverage-report
 
 test-filter:
-	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" ./vendor/bin/phpunit --filter=$(filter) --testdox
+	$(docker_compose_bin) exec "$(APP_CONTAINER_NAME)" ./vendor/bin/phpunit --filter=$(filter) --testdox

README.md

@@ -185,7 +185,11 @@ While working in production, to keep your existing search experience available w
 
 `php artisan scout:import`
 
-The command create new temporary index, import all models to it, and then switch to the index and remove old index.
+The command creates new temporary index, imports all models to it, and then switches to the index and remove old index.
+
+### Parallel import
+When importing massive ammounts of data, you can use the option `--parallel`, to speed up the import process.
+This however requires you to set-up the suggested trackable-jobs package and queue workers.
 
 ### Search
 

composer.json

@@ -23,8 +23,8 @@
         "roave/better-reflection": "^4.3|^5.0|^6.18|^6.36"
     },
     "require-dev": {
+        "larastan/larastan": "^2.9",
         "laravel/legacy-factories": "^1.0",
-        "nunomaduro/larastan": "^2.4",
         "orchestra/testbench": "^6.17|^7.0|^8.0",
         "php-http/guzzle7-adapter": "^1.0",
         "phpunit/phpunit": "^9.4.0"
@@ -49,6 +49,9 @@
             ]
         }
     },
+    "suggest": {
+        "mateusjunges/laravel-trackable-jobs": "Required when using --parallel option in ImportCommand."
+    },
     "config": {
         "sort-packages": true,
         "preferred-install": "dist"

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 services:
   app:
     build:
@@ -50,6 +48,7 @@ services:
       - bootstrap.memory_lock=true
       - action.destructive_requires_name=false
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - xpack.security.enabled=false
     ulimits:
       memlock:
         soft: -1

docker/app/Dockerfile

@@ -41,9 +41,10 @@ RUN echo "xdebug.remote_connect_back=0" >> /usr/local/etc/php/conf.d/docker-php-
 RUN echo "xdebug.idekey=\"PHPSTORM\"" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
 RUN echo "xdebug.remote_port=9000" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
 RUN echo "xdebug.remote_autostart=1" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
+RUN echo "xdebug.mode=debug,coverage" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
 
 RUN apt-get update && apt-get install -y procps
-RUN apt-get update && apt-get install -y netcat
+RUN apt-get update && apt-get install -y netcat-traditional
 RUN apt-get update && apt-get install -y net-tools
 
 FROM php as php-testing

phpstan.neon.dist

@@ -1,5 +1,5 @@
 includes:
-    - vendor/nunomaduro/larastan/extension.neon
+    - vendor/larastan/larastan/extension.neon
 parameters:
     level: max
     paths:

src/Console/Commands/ImportCommand.php

@@ -9,7 +9,6 @@
 use Matchish\ScoutElasticSearch\ElasticSearch\Config\Config;
 use Matchish\ScoutElasticSearch\Jobs\Import;
 use Matchish\ScoutElasticSearch\Jobs\QueueableJob;
-use Matchish\ScoutElasticSearch\Searchable\ImportSource;
 use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory;
 use Matchish\ScoutElasticSearch\Searchable\SearchableListFactory;
 
@@ -18,7 +17,8 @@ final class ImportCommand extends Command
     /**
      * {@inheritdoc}
      */
-    protected $signature = 'scout:import {searchable?* : The name of the searchable}';
+    protected $signature = 'scout:import {searchable?* : The name of the searchable} {--P|parallel : Index items in parallel}';
+
     /**
      * {@inheritdoc}
      */
@@ -29,12 +29,22 @@ final class ImportCommand extends Command
      */
     public function handle(): void
     {
+        $parallel = false;
+
+        if ($this->option('parallel')) {
+            $parallel = true;
+        }
+
         $this->searchableList((array) $this->argument('searchable'))
-        ->each(function ($searchable) {
-            $this->import($searchable);
+        ->each(function (string $searchable) use ($parallel) {
+            $this->import($searchable, $parallel);
         });
     }
 
+    /**
+     * @param  array<string>  $argument
+     * @return Collection<int, string>
+     */
     private function searchableList(array $argument): Collection
     {
         return collect($argument)->whenEmpty(function () {
@@ -44,18 +54,31 @@ private function searchableList(array $argument): Collection
         });
     }
 
-    private function import(string $searchable): void
+    /**
+     * @param  string  $searchable
+     * @param  bool  $parallel
+     * @return void
+     */
+    private function import(string $searchable, bool $parallel): void
     {
         $sourceFactory = app(ImportSourceFactory::class);
         $source = $sourceFactory::from($searchable);
         $job = new Import($source);
-        $job->timeout = Config::queueTimeout();
+        /** @var int|null $queueTimeout */
+        $queueTimeout = Config::queueTimeout();
+        if ($queueTimeout !== null) {
+            $job->timeout = (int) $queueTimeout;
+        }
+        $job->parallel = $parallel;
 
         if (config('scout.queue')) {
             $job = (new QueueableJob())->chain([$job]);
-            $job->timeout = Config::queueTimeout();
+            /** @var int|null $queueTimeout */
+            $queueTimeout = Config::queueTimeout();
+            if ($queueTimeout !== null) {
+                $job->timeout = (int) $queueTimeout;
+            }
         }
-
         $bar = (new ProgressBarFactory($this->output))->create();
         $job->withProgressReport($bar);
 

src/Database/Scopes/FromScope.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Database\Scopes;
+
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Scope;
+use Laravel\Scout\Searchable;
+
+class FromScope implements Scope
+{
+    /**
+     * @var int
+     */
+    private $lastId;
+    /**
+     * @var int
+     */
+    private $perPage;
+
+    /**
+     * PageScope constructor.
+     *
+     * @param  int  $lastId
+     * @param  int  $perPage
+     */
+    public function __construct(int $lastId, int $perPage)
+    {
+        $this->lastId = $lastId;
+        $this->perPage = $perPage;
+    }
+
+    /**
+     * Apply the scope to a given Eloquent query builder.
+     *
+     * @param  Builder<Model|Searchable>  $builder
+     * @param  Model  $model
+     * @return void
+     */
+    public function apply(Builder $builder, Model $model)
+    {
+        $builder->forPageAfterId($this->perPage, $this->lastId);
+    }
+}

src/Database/Scopes/PageScope.php

@@ -5,6 +5,7 @@
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\Scope;
+use Laravel\Scout\Searchable;
 
 class PageScope implements Scope
 {
@@ -32,8 +33,8 @@ public function __construct(int $page, int $perPage)
     /**
      * Apply the scope to a given Eloquent query builder.
      *
-     * @param  \Illuminate\Database\Eloquent\Builder  $builder
-     * @param  \Illuminate\Database\Eloquent\Model  $model
+     * @param  Builder<Model|Searchable>  $builder
+     * @param  Model  $model
      * @return void
      */
     public function apply(Builder $builder, Model $model)