From 35761327f9af3ab4cd57e9b1bb34daf256c2d2ee Mon Sep 17 00:00:00 2001 From: seyedmr Date: Sat, 19 Nov 2022 03:14:33 +0330 Subject: [PATCH] feat(convert json): finish --- app/Http/Controllers/V1/ConvertController.php | 1 + app/Interfaces/ConverterInterface.php | 8 ---- app/Jobs/ChunkFile.php | 9 ++-- app/Jobs/StoreChunk.php | 47 +++++++++++++++++-- app/Models/User.php | 7 ++- app/Pipes/StoreChunk/AgeFilterPipe.php | 24 ++++++++++ app/Pipes/StoreChunk/CreditCardFilter.php | 20 ++++++++ app/Services/Chunkers/JsonChunker.php | 26 ++++++++-- .../Converters/JsonToDatabaseConverter.php | 10 ---- database/factories/UserFactory.php | 17 +++++-- .../2014_10_12_000000_create_users_table.php | 1 + docker-compose.yml | 2 +- .../testing-database.sql | 3 ++ tests/Feature/Upload/ConvertJsonTest.php | 3 +- tests/TestCase.php | 2 + tests/Unit/Jobs/ChunkFileTest.php | 1 + tests/Unit/Jobs/StoreChunkTest.php | 38 +++++++++++++++ 17 files changed, 183 insertions(+), 36 deletions(-) delete mode 100644 app/Interfaces/ConverterInterface.php create mode 100644 app/Pipes/StoreChunk/AgeFilterPipe.php create mode 100644 app/Pipes/StoreChunk/CreditCardFilter.php delete mode 100644 app/Services/Converters/JsonToDatabaseConverter.php create mode 100644 docker/database/mysql/create-testing-database.sh/testing-database.sql create mode 100644 tests/Unit/Jobs/StoreChunkTest.php diff --git a/app/Http/Controllers/V1/ConvertController.php b/app/Http/Controllers/V1/ConvertController.php index b3cbc4c..bf70473 100644 --- a/app/Http/Controllers/V1/ConvertController.php +++ b/app/Http/Controllers/V1/ConvertController.php @@ -6,6 +6,7 @@ use App\Enums\FileFormatEnum; use App\Http\Controllers\Controller; use App\Jobs\ChunkFile; use Illuminate\Http\Request; +use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; diff --git a/app/Interfaces/ConverterInterface.php b/app/Interfaces/ConverterInterface.php deleted file mode 100644 index 6674d2d..0000000 --- a/app/Interfaces/ConverterInterface.php +++ /dev/null @@ -1,8 +0,0 @@ -format->chunker($this->uuid, $this->path); - foreach($chunks as $chunk){ + foreach ($chunks as $key => $chunk) { StoreChunk::dispatch($chunk); + Redis::set($this->uuid . ":latest", $key); } Storage::delete($this->path); + Redis::del($this->uuid . ":latest"); } } diff --git a/app/Jobs/StoreChunk.php b/app/Jobs/StoreChunk.php index 5b59bc1..2a941c9 100644 --- a/app/Jobs/StoreChunk.php +++ b/app/Jobs/StoreChunk.php @@ -2,25 +2,35 @@ namespace App\Jobs; +use App\Models\User; +use App\Pipes\StoreChunk\AgeFilterPipe; +use App\Pipes\StoreChunk\CreditCardFilter; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; +use Illuminate\Pipeline\Pipeline; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Carbon; +use Illuminate\Support\Collection; class StoreChunk implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + public Collection $chunk; + /** * Create a new job instance. * * @return void */ - public function __construct() + public function __construct( + array $chunk + ) { - // + $this->chunk = collect($chunk); } /** @@ -30,6 +40,37 @@ class StoreChunk implements ShouldQueue */ public function handle() { - // + $chunk = (new Pipeline(app())) + ->through([ + AgeFilterPipe::class, + CreditCardFilter::class, + ]) + ->send($this->changeDateOfBirthToCarbon()) + ->thenReturn(); + + User::insert( + $this->prepare($chunk)->toArray() + ); + } + + protected function changeDateOfBirthToCarbon() + { + return $this->chunk->map(function ($item) { + if (!strpos($item['date_of_birth'], '/')) { + $item['date_of_birth'] = Carbon::create($item['date_of_birth']); + } else { + $item['date_of_birth'] = Carbon::createFromFormat('d/m/Y', $item['date_of_birth']); + } + + return $item; + }); + } + + protected function prepare($chunk) + { + return $chunk->map(function ($item) { + $item['credit_card'] = json_encode($item['credit_card']); + return $item; + }); } } diff --git a/app/Models/User.php b/app/Models/User.php index 1dbcba0..460bd50 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -40,5 +40,10 @@ class User extends Authenticatable * * @var array */ - protected $casts = []; + protected $casts = [ + 'date_of_birth' => 'datetime', + 'credit_card' => 'json', + ]; + + public $timestamps = false; } diff --git a/app/Pipes/StoreChunk/AgeFilterPipe.php b/app/Pipes/StoreChunk/AgeFilterPipe.php new file mode 100644 index 0000000..0522209 --- /dev/null +++ b/app/Pipes/StoreChunk/AgeFilterPipe.php @@ -0,0 +1,24 @@ +filter(function ($item) { + $age = $item['date_of_birth']->age; + return 18 <= $age && $age <= 65; + }); + + return $next($chunk); + } +} diff --git a/app/Pipes/StoreChunk/CreditCardFilter.php b/app/Pipes/StoreChunk/CreditCardFilter.php new file mode 100644 index 0000000..87184c1 --- /dev/null +++ b/app/Pipes/StoreChunk/CreditCardFilter.php @@ -0,0 +1,20 @@ + $value) { + $latest = Redis::get($uuid . ":latest") ?? 0; + + foreach (Items::fromFile(Storage::path($path), ['decoder' => new ExtJsonDecoder(true)]) as $key => $value) { + if ($key < $latest) { + continue; + } + +// for cut off test you have to uncomment below codes +// if($key == 900){ +// throw new \Exception('cut off'); +// } + $chunk[] = $value; if (count($chunk) == 500) { - yield $chunk; + yield $key => $chunk; $chunk = []; } } - if(count($chunk) > 0){ - yield $chunk; + if (count($chunk) > 0) { + yield $key => $chunk; } } } diff --git a/app/Services/Converters/JsonToDatabaseConverter.php b/app/Services/Converters/JsonToDatabaseConverter.php deleted file mode 100644 index 32f4853..0000000 --- a/app/Services/Converters/JsonToDatabaseConverter.php +++ /dev/null @@ -1,10 +0,0 @@ - fake()->name(), + 'account' => fake()->numberBetween(10000,99999), + 'address' => fake()->address(), + 'checked' => fake()->boolean(), 'email' => fake()->unique()->safeEmail(), - 'email_verified_at' => now(), - 'password' => '$2y$10$92IXUNpkjO0rOQ5byMi.Ye4oKoEa3Ro9llC/.og/at2.uheWG/igi', // password - 'remember_token' => Str::random(10), + 'description' => fake()->realText(), + 'interest' => fake()->jobTitle(), + 'date_of_birth' => fake()->dateTimeBetween('-70 years', 'now'), + 'credit_card' => json_encode([ + "type" => fake()->creditCardType(), + "number" => fake()->creditCardNumber(), + "name" => fake()->name(), + "expirationDate" => fake()->creditCardExpirationDateString(), + ]), ]; } @@ -33,7 +42,7 @@ class UserFactory extends Factory */ public function unverified() { - return $this->state(fn (array $attributes) => [ + return $this->state(fn(array $attributes) => [ 'email_verified_at' => null, ]); } diff --git a/database/migrations/2014_10_12_000000_create_users_table.php b/database/migrations/2014_10_12_000000_create_users_table.php index ce39a00..822e801 100644 --- a/database/migrations/2014_10_12_000000_create_users_table.php +++ b/database/migrations/2014_10_12_000000_create_users_table.php @@ -15,6 +15,7 @@ return new class extends Migration { Schema::create('users', function (Blueprint $table) { $table->id(); + $table->foreignId('account'); $table->string('name'); $table->text('address'); $table->boolean('checked'); diff --git a/docker-compose.yml b/docker-compose.yml index d0b8a97..66037a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' volumes: - 'sail-mariadb:/var/lib/mysql' - - './docker/database/mysql/create-testing-database.sh:/docker-entrypoint-initdb.d/10-create-testing-database.sh' + - './docker/database/mysql/create-testing-database.sh:/docker-entrypoint-initdb.d' networks: - sail healthcheck: diff --git a/docker/database/mysql/create-testing-database.sh/testing-database.sql b/docker/database/mysql/create-testing-database.sh/testing-database.sql new file mode 100644 index 0000000..8803730 --- /dev/null +++ b/docker/database/mysql/create-testing-database.sh/testing-database.sql @@ -0,0 +1,3 @@ +create database testing; +GRANT ALL ON `testing`.* TO 'sail'@'%'; +FLUSH PRIVILEGES; diff --git a/tests/Feature/Upload/ConvertJsonTest.php b/tests/Feature/Upload/ConvertJsonTest.php index 74877c0..e3fa466 100644 --- a/tests/Feature/Upload/ConvertJsonTest.php +++ b/tests/Feature/Upload/ConvertJsonTest.php @@ -16,8 +16,7 @@ class ConvertJsonTest extends TestCase * upload json file. * * @return void - * @group upload - * @group json + * @group convert */ public function test_example() { diff --git a/tests/TestCase.php b/tests/TestCase.php index 2932d4a..29bd88b 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -2,9 +2,11 @@ namespace Tests; +use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Foundation\Testing\TestCase as BaseTestCase; abstract class TestCase extends BaseTestCase { use CreatesApplication; + use RefreshDatabase; } diff --git a/tests/Unit/Jobs/ChunkFileTest.php b/tests/Unit/Jobs/ChunkFileTest.php index 0a35699..fa62dd6 100644 --- a/tests/Unit/Jobs/ChunkFileTest.php +++ b/tests/Unit/Jobs/ChunkFileTest.php @@ -17,6 +17,7 @@ class ChunkFileTest extends TestCase * A basic unit test example. * * @return void + * @group convert */ public function test_chunk_json_file() { diff --git a/tests/Unit/Jobs/StoreChunkTest.php b/tests/Unit/Jobs/StoreChunkTest.php new file mode 100644 index 0000000..0cbe0c3 --- /dev/null +++ b/tests/Unit/Jobs/StoreChunkTest.php @@ -0,0 +1,38 @@ +count(99)->state( + new Sequence( + [ + 'date_of_birth' => fake()->dateTimeBetween('-70 years', '-66 years'), + ], + [ + 'date_of_birth' => fake()->dateTimeBetween('-65 years', '-18 years'), + ], + [ + 'date_of_birth' => fake()->dateTimeBetween('-18 years', 'now'), + ] + ) + )->make()->toArray() + ))->handle(); + + $this->assertDatabaseCount('users', 33); + } +}