You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
46 lines
1.1 KiB
46 lines
1.1 KiB
<?php
|
|
|
|
namespace App\Services\Chunkers;
|
|
|
|
use App\Interfaces\ChunkerInterface;
|
|
use App\Services\Decoder\JsonDecoder;
|
|
use Illuminate\Support\Facades\Redis;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use JsonMachine\Exception\InvalidArgumentException;
|
|
use JsonMachine\Items;
|
|
use JsonMachine\JsonDecoder\ExtJsonDecoder;
|
|
|
|
class JsonChunker implements ChunkerInterface
|
|
{
|
|
/**
|
|
* @throws InvalidArgumentException
|
|
*/
|
|
public function handle(string $uuid, string $path): \Generator
|
|
{
|
|
$chunk = [];
|
|
|
|
$latest = Redis::get($uuid . ":latest") ?? 0;
|
|
|
|
foreach (Items::fromFile(Storage::path($path), ['decoder' => new ExtJsonDecoder(true)]) as $key => $value) {
|
|
if ($key < $latest) {
|
|
continue;
|
|
}
|
|
|
|
// for cut off test you have to uncomment below codes
|
|
// if($key == 900){
|
|
// throw new \Exception('cut off');
|
|
// }
|
|
|
|
$chunk[] = $value;
|
|
|
|
if (count($chunk) == 500) {
|
|
yield $key => $chunk;
|
|
$chunk = [];
|
|
}
|
|
}
|
|
|
|
if (count($chunk) > 0) {
|
|
yield $key => $chunk;
|
|
}
|
|
}
|
|
}
|
|
|