feat(convert json): finish

app/Http/Controllers/V1/ConvertController.php

@@ -6,6 +6,7 @@ use App\Enums\FileFormatEnum;
 use App\Http\Controllers\Controller;
 use App\Jobs\ChunkFile;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
 

app/Interfaces/ConverterInterface.php

@@ -1,8 +0,0 @@
-<?php
-
-namespace App\Interfaces;
-
-interface ConverterInterface
-{
-    public function handle(): void;
-}

app/Jobs/ChunkFile.php

@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Facades\Storage;
 
 class ChunkFile implements ShouldQueue
@@ -21,8 +22,8 @@ class ChunkFile implements ShouldQueue
      * @return void
      */
     public function __construct(
-        public string $uuid,
+        public string         $uuid,
-        public string $path,
+        public string         $path,
         public FileFormatEnum $format
     )
     {
@@ -37,9 +38,11 @@ class ChunkFile implements ShouldQueue
     public function handle()
     {
         $chunks = $this->format->chunker($this->uuid, $this->path);
-        foreach($chunks as $chunk){
+        foreach ($chunks as $key => $chunk) {
             StoreChunk::dispatch($chunk);
+            Redis::set($this->uuid . ":latest", $key);
         }
         Storage::delete($this->path);
+        Redis::del($this->uuid . ":latest");
     }
 }

app/Jobs/StoreChunk.php

@@ -2,25 +2,35 @@
 
 namespace App\Jobs;
 
+use App\Models\User;
+use App\Pipes\StoreChunk\AgeFilterPipe;
+use App\Pipes\StoreChunk\CreditCardFilter;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Pipeline\Pipeline;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Collection;
 
 class StoreChunk implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    public Collection $chunk;
+
     /**
      * Create a new job instance.
      *
      * @return void
      */
-    public function __construct()
+    public function __construct(
+        array $chunk
+    )
     {
-        //
+        $this->chunk = collect($chunk);
     }
 
     /**
@@ -30,6 +40,37 @@ class StoreChunk implements ShouldQueue
      */
     public function handle()
     {
-        //
+        $chunk = (new Pipeline(app()))
+            ->through([
+                AgeFilterPipe::class,
+                CreditCardFilter::class,
+            ])
+            ->send($this->changeDateOfBirthToCarbon())
+            ->thenReturn();
+
+        User::insert(
+            $this->prepare($chunk)->toArray()
+        );
+    }
+
+    protected function changeDateOfBirthToCarbon()
+    {
+        return $this->chunk->map(function ($item) {
+            if (!strpos($item['date_of_birth'], '/')) {
+                $item['date_of_birth'] = Carbon::create($item['date_of_birth']);
+            } else {
+                $item['date_of_birth'] = Carbon::createFromFormat('d/m/Y', $item['date_of_birth']);
+            }
+
+            return $item;
+        });
+    }
+
+    protected function prepare($chunk)
+    {
+        return $chunk->map(function ($item) {
+            $item['credit_card'] = json_encode($item['credit_card']);
+            return $item;
+        });
     }
 }

app/Models/User.php

@@ -40,5 +40,10 @@ class User extends Authenticatable
      *
      * @var array<string, string>
      */
-    protected $casts = [];
+    protected $casts = [
+        'date_of_birth' => 'datetime',
+        'credit_card' => 'json',
+    ];
+
+    public $timestamps = false;
 }

app/Pipes/StoreChunk/AgeFilterPipe.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Pipes\StoreChunk;
+
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Collection;
+
+class AgeFilterPipe
+{
+    /**
+     * @param Collection $chunk
+     * @param \Closure $next
+     * @return mixed
+     */
+    public function handle($chunk, \Closure $next)
+    {
+        $chunk = $chunk->filter(function ($item) {
+            $age = $item['date_of_birth']->age;
+            return 18 <= $age && $age <= 65;
+        });
+
+        return $next($chunk);
+    }
+}

app/Pipes/StoreChunk/CreditCardFilter.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Pipes\StoreChunk;
+
+use Illuminate\Support\Collection;
+
+class CreditCardFilter
+{
+    /**
+     * @param Collection $chunk
+     * @param \Closure $next
+     * @return mixed
+     */
+    public function handle($chunk, \Closure $next)
+    {
+        // filter based on credit card number
+
+        return $next($chunk);
+    }
+}

app/Services/Chunkers/JsonChunker.php

@@ -3,26 +3,44 @@
 namespace App\Services\Chunkers;
 
 use App\Interfaces\ChunkerInterface;
+use App\Services\Decoder\JsonDecoder;
+use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Facades\Storage;
+use JsonMachine\Exception\InvalidArgumentException;
 use JsonMachine\Items;
+use JsonMachine\JsonDecoder\ExtJsonDecoder;
 
 class JsonChunker implements ChunkerInterface
 {
+    /**
+     * @throws InvalidArgumentException
+     */
     public function handle(string $uuid, string $path): \Generator
     {
         $chunk = [];
 
-        foreach (Items::fromFile(Storage::path($path)) as $key => $value) {
+        $latest = Redis::get($uuid . ":latest") ?? 0;
+
+        foreach (Items::fromFile(Storage::path($path), ['decoder' => new ExtJsonDecoder(true)]) as $key => $value) {
+            if ($key < $latest) {
+                continue;
+            }
+
+//            for cut off test you have to uncomment below codes
+//            if($key == 900){
+//                throw new \Exception('cut off');
+//            }
+
             $chunk[] = $value;
 
             if (count($chunk) == 500) {
-                yield $chunk;
+                yield $key => $chunk;
                 $chunk = [];
             }
         }
 
-        if(count($chunk) > 0){
+        if (count($chunk) > 0) {
-            yield $chunk;
+            yield $key => $chunk;
         }
     }
 }

app/Services/Converters/JsonToDatabaseConverter.php

@@ -1,10 +0,0 @@
-<?php
-
-namespace App\Services\Converters;
-
-use App\Interfaces\ConverterInterface;
-
-class JsonToDatabaseConverter implements ConverterInterface
-{
-
-}

database/factories/UserFactory.php

@@ -19,10 +19,19 @@ class UserFactory extends Factory
     {
         return [
             'name' => fake()->name(),
+            'account' => fake()->numberBetween(10000,99999),
+            'address' => fake()->address(),
+            'checked' => fake()->boolean(),
             'email' => fake()->unique()->safeEmail(),
-            'email_verified_at' => now(),
+            'description' => fake()->realText(),
-            'password' => '$2y$10$92IXUNpkjO0rOQ5byMi.Ye4oKoEa3Ro9llC/.og/at2.uheWG/igi', // password
+            'interest' => fake()->jobTitle(),
-            'remember_token' => Str::random(10),
+            'date_of_birth' => fake()->dateTimeBetween('-70 years', 'now'),
+            'credit_card' => json_encode([
+                "type" => fake()->creditCardType(),
+                "number" => fake()->creditCardNumber(),
+                "name" => fake()->name(),
+                "expirationDate" => fake()->creditCardExpirationDateString(),
+            ]),
         ];
     }
 
@@ -33,7 +42,7 @@ class UserFactory extends Factory
      */
     public function unverified()
     {
-        return $this->state(fn (array $attributes) => [
+        return $this->state(fn(array $attributes) => [
             'email_verified_at' => null,
         ]);
     }

database/migrations/2014_10_12_000000_create_users_table.php

@@ -15,6 +15,7 @@ return new class extends Migration
     {
         Schema::create('users', function (Blueprint $table) {
             $table->id();
+            $table->foreignId('account');
             $table->string('name');
             $table->text('address');
             $table->boolean('checked');

docker-compose.yml

@@ -38,7 +38,7 @@ services:
             MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
         volumes:
             - 'sail-mariadb:/var/lib/mysql'
-            - './docker/database/mysql/create-testing-database.sh:/docker-entrypoint-initdb.d/10-create-testing-database.sh'
+            - './docker/database/mysql/create-testing-database.sh:/docker-entrypoint-initdb.d'
         networks:
             - sail
         healthcheck:

docker/database/mysql/create-testing-database.sh/testing-database.sql

@@ -0,0 +1,3 @@
+create database testing;
+GRANT ALL ON `testing`.* TO 'sail'@'%';
+FLUSH PRIVILEGES;

tests/Feature/Upload/ConvertJsonTest.php

@@ -16,8 +16,7 @@ class ConvertJsonTest extends TestCase
      * upload json file.
      *
      * @return void
-     * @group upload
+     * @group convert
-     * @group json
      */
     public function test_example()
     {

tests/TestCase.php

@@ -2,9 +2,11 @@
 
 namespace Tests;
 
+use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Foundation\Testing\TestCase as BaseTestCase;
 
 abstract class TestCase extends BaseTestCase
 {
     use CreatesApplication;
+    use RefreshDatabase;
 }

tests/Unit/Jobs/ChunkFileTest.php

@@ -17,6 +17,7 @@ class ChunkFileTest extends TestCase
      * A basic unit test example.
      *
      * @return void
+     * @group convert
      */
     public function test_chunk_json_file()
     {

tests/Unit/Jobs/StoreChunkTest.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace Tests\Unit\Jobs;
+
+use App\Jobs\StoreChunk;
+use App\Models\User;
+use Illuminate\Database\Eloquent\Factories\Sequence;
+use Tests\TestCase;
+
+class StoreChunkTest extends TestCase
+{
+    /**
+     * A basic unit test example.
+     *
+     * @return void
+     * @group convert2
+     */
+    public function test_store_chunk()
+    {
+        (new StoreChunk(
+            User::factory()->count(99)->state(
+                new Sequence(
+                    [
+                        'date_of_birth' => fake()->dateTimeBetween('-70 years', '-66 years'),
+                    ],
+                    [
+                        'date_of_birth' => fake()->dateTimeBetween('-65 years', '-18 years'),
+                    ],
+                    [
+                        'date_of_birth' => fake()->dateTimeBetween('-18 years', 'now'),
+                    ]
+                )
+            )->make()->toArray()
+        ))->handle();
+
+        $this->assertDatabaseCount('users', 33);
+    }
+}