Skip to content

🐛 Fixed issues with incremental indexer command #561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pushrbx wants to merge 4 commits into from
Closed

🐛 Fixed issues with incremental indexer command #561

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
12d3189
fixed issues with incremental indexer command
pushrbx Nov 13, 2024
e213318
changed exit code and refactored cancellable sleep function
pushrbx Nov 13, 2024
5e0d0fd
added PromptsForMissingInput contract to IncrementalIndexer
pushrbx Nov 13, 2024
a318f24
refactor
pushrbx Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 39 additions & 15 deletions app/Console/Commands/Indexer/IncrementalIndexer.php
View file
Open in desktop
Comment thread
irfan-dahir marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,17 @@
namespace App\Console\Commands\Indexer;

use Illuminate\Console\Command;
use Illuminate\Contracts\Console\PromptsForMissingInput;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Validator;

class IncrementalIndexer extends Command
class IncrementalIndexer extends Command implements PromptsForMissingInput
{
/**
* @var bool
*/
private bool $cancelled = false;
private int $receivedSignal = 0;

/**
* The name and signature of the console command.
Expand Down Expand Up @@ -54,10 +56,10 @@ private function getIdsToFetch(string $mediaType): array
return [];
}

$this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json...");
$newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json");
$newIdsHash = sha1($newIdsRaw);

/** @noinspection PhpConditionAlreadyCheckedInspection */
if ($this->cancelled)
{
return [];
Expand Down Expand Up @@ -91,19 +93,20 @@ private function getFailedIdsToFetch(string $mediaType): array
return json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json"));
}

private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): void
private function fetchIds(string $mediaType, array $idsToFetch, int $delay, bool $resume): void
{
$index = 0;
$success = [];
$failedIds = [];
$idCount = count($idsToFetch);

if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save"))
{
$index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save");
$this->info("Resuming from index: $index");
}

$ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']);
$idCount = count($ids);

if ($index > 0 && !isset($ids[$index]))
{
Expand All @@ -119,10 +122,11 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v
{
if ($this->cancelled)
{
$this->info("Cancelling...");
return;
}

$id = $ids[$index];
$id = $ids[$i];

$url = env('APP_URL') . "/v4/$mediaType/$id";
$this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]");
Expand All @@ -142,6 +146,16 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v
$this->warn("[SKIPPED] Failed to fetch $url");
$failedIds[] = $id;
Storage::put("indexer/incremental/$mediaType.failed", json_encode($failedIds));
continue;
}
finally
{
cancellable_sleep($delay * 1000, fn() => $this->cancelled);
if ($this->cancelled)
{
$this->info("Cancelling...");
return;
}
}

$success[] = $id;
Expand All @@ -168,27 +182,35 @@ public function handle(): int
[
'mediaType' => $this->argument('mediaType'),
'delay' => $this->option('delay'),
'resume' => $this->option('resume') ?? false,
'failed' => $this->option('failed') ?? false
'resume' => $this->option('resume'),
'failed' => $this->option('failed')
],
[
'mediaType' => 'required|in:anime,manga',
'mediaType' => 'required|array',
'mediaType.*' => 'in:anime,manga',
'delay' => 'integer|min:1',
'resume' => 'bool|prohibited_with:failed',
'failed' => 'bool|prohibited_with:resume'
'resume' => 'bool',
'failed' => 'bool'
]
);

if ($validator->fails()) {
if ($validator->fails())
{
$this->error($validator->errors()->toJson());
return 1;
}

// we want to handle signals from the OS
$this->trap([SIGTERM, SIGQUIT, SIGINT], fn () => $this->cancelled = true);
$this->trap([SIGTERM, SIGQUIT, SIGINT], function (int $signal) {
$this->cancelled = true;
$this->receivedSignal = $signal;
});

$this->info("Info: IncrementalIndexer uses purarue/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n");

$resume = $this->option('resume') ?? false;
$onlyFailed = $this->option('failed') ?? false;
$delay = $this->option('delay') ?? 3;

/**
* @var $mediaTypes array
Expand All @@ -211,18 +233,20 @@ public function handle(): int

if ($this->cancelled)
{
return 127;
$this->info("Cancelling...");
return 128 + $this->receivedSignal;
}

$idCount = count($idsToFetch);
if ($idCount === 0)
{
$this->info("No $mediaType entries to index");
continue;
}

$this->fetchIds($mediaType, $idsToFetch, $resume);
$this->fetchIds($mediaType, $idsToFetch, $delay, $resume);
}

return 0;
return $this->cancelled && $this->receivedSignal > 0 ? 128 + $this->receivedSignal : 0;
}
}
9 changes: 8 additions & 1 deletion app/Providers/AppServiceProvider.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@
use App\Support\DefaultMediator;
use App\Support\JikanConfig;
use App\Support\JikanUnitOfWork;
use Illuminate\Console\Signals;
use Illuminate\Contracts\Container\BindingResolutionException;
use Illuminate\Http\JsonResponse;
use Laravel\Lumen\Application;
use Illuminate\Http\Response;
use Illuminate\Support\Env;
use Illuminate\Support\ServiceProvider;
use Illuminate\Support\Collection;
use Jikan\MyAnimeList\MalClient;
use Laravel\Scout\Builder as ScoutBuilder;
use Typesense\LaravelTypesense\Typesense;
use App\Features;
Expand Down Expand Up @@ -96,6 +96,13 @@ public function register(): void
$this->app->singleton(\App\Services\TypesenseCollectionDescriptor::class);
}
$this->registerModelRepositories();

// lumen hack for signal handling in artisan commands
Signals::resolveAvailabilityUsing(function () {
return $this->app->runningInConsole()
&& ! $this->app->runningUnitTests()
&& extension_loaded('pcntl');
Comment thread
irfan-dahir marked this conversation as resolved.
});
}

private function getSearchService(Repository $repository): SearchService
Expand Down
18 changes: 18 additions & 0 deletions app/Support/helpers.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,3 +123,21 @@ function ensureEnumPrimitiveValue(int|string|bool|float|null|\Spatie\Enum\Larave
return $value;
}
}

if (!function_exists("cancellable_sleep")) {
function cancellable_sleep(int $milliseconds, callable $isCancelled): void {
$interval = 100; // check every 100 ms
$elapsed = 0;

while ($elapsed < $milliseconds)
{
if ($isCancelled())
{
return;
}

usleep($interval * 1000);
$elapsed += $interval;
}
}
}