diff --git a/app/Enums/FileFormatEnum.php b/app/Enums/FileFormatEnum.php index 60ab4cb..27deb5f 100644 --- a/app/Enums/FileFormatEnum.php +++ b/app/Enums/FileFormatEnum.php @@ -9,19 +9,19 @@ enum FileFormatEnum: string { case JSON = 'json'; - public function chunker($path): \Generator + public function chunker(string $uuid, string $path): \Generator { $chunker = match ($this) { - self::JSON => JsonChunker::class + self::JSON => new JsonChunker }; - return $chunker->handle($path); + return $chunker->handle($uuid, $path); } public function converter() { return match($this){ - self::JSON => new JsonToDatabaseConverter() + self::JSON => new JsonToDatabaseConverter }; } } diff --git a/app/Interfaces/ChunkerInterface.php b/app/Interfaces/ChunkerInterface.php index 4910f75..eab4ec3 100644 --- a/app/Interfaces/ChunkerInterface.php +++ b/app/Interfaces/ChunkerInterface.php @@ -4,5 +4,5 @@ namespace App\Interfaces; interface ChunkerInterface { - public function handle(): \Generator; + public function handle(string $uuid, string $path): \Generator; } diff --git a/app/Jobs/ChunkFile.php b/app/Jobs/ChunkFile.php index 872d571..d9440af 100644 --- a/app/Jobs/ChunkFile.php +++ b/app/Jobs/ChunkFile.php @@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\Storage; class ChunkFile implements ShouldQueue { @@ -35,6 +36,10 @@ class ChunkFile implements ShouldQueue */ public function handle() { - + $chunks = $this->format->chunker($this->uuid, $this->path); + foreach($chunks as $chunk){ + StoreChunk::dispatch($chunk); + } + Storage::delete($this->path); } } diff --git a/app/Models/User.php b/app/Models/User.php index 23b4063..1dbcba0 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -19,8 +19,13 @@ class User extends Authenticatable */ protected $fillable = [ 'name', + 'address', + 'checked', 'email', - 'password', + 'description', + 'interest', + 'date_of_birth', + 'credit_card', ]; /** @@ -28,17 +33,12 @@ class User extends Authenticatable * * @var array */ - protected $hidden = [ - 'password', - 'remember_token', - ]; + protected $hidden = []; /** * The attributes that should be cast. * * @var array */ - protected $casts = [ - 'email_verified_at' => 'datetime', - ]; + protected $casts = []; } diff --git a/app/Providers/HorizonServiceProvider.php b/app/Providers/HorizonServiceProvider.php new file mode 100644 index 0000000..0888117 --- /dev/null +++ b/app/Providers/HorizonServiceProvider.php @@ -0,0 +1,42 @@ +email, [ + // + ]); + }); + } +} diff --git a/app/Services/Chunkers/JsonChunker.php b/app/Services/Chunkers/JsonChunker.php index 88310c9..a1307d8 100644 --- a/app/Services/Chunkers/JsonChunker.php +++ b/app/Services/Chunkers/JsonChunker.php @@ -3,11 +3,26 @@ namespace App\Services\Chunkers; use App\Interfaces\ChunkerInterface; +use Illuminate\Support\Facades\Storage; +use JsonMachine\Items; class JsonChunker implements ChunkerInterface { - public function handle(): \Generator + public function handle(string $uuid, string $path): \Generator { + $chunk = []; + foreach (Items::fromFile(Storage::path($path)) as $key => $value) { + $chunk[] = $value; + + if (count($chunk) == 500) { + yield $chunk; + $chunk = []; + } + } + + if(count($chunk) > 0){ + yield $chunk; + } } } diff --git a/composer.json b/composer.json index 027ed8b..bca6600 100644 --- a/composer.json +++ b/composer.json @@ -7,6 +7,7 @@ "require": { "php": "^8.0.2", "guzzlehttp/guzzle": "^7.2", + "halaxa/json-machine": "^1.1", "laravel/framework": "^9.19", "laravel/horizon": "^5.10", "laravel/sanctum": "^3.0", diff --git a/composer.lock b/composer.lock index f8002c9..5cce323 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9dcd1c31751916ad055da24bc3ed4c71", + "content-hash": "7b1ad9d92186f7979d5ea9f3a6a9efa6", "packages": [ { "name": "brick/math", @@ -897,6 +897,64 @@ ], "time": "2022-10-26T14:07:24+00:00" }, + { + "name": "halaxa/json-machine", + "version": "1.1.3", + "source": { + "type": "git", + "url": "https://github.com/halaxa/json-machine.git", + "reference": "514025c5ebbdb8a058745b573b4a1e81d685802c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/halaxa/json-machine/zipball/514025c5ebbdb8a058745b573b4a1e81d685802c", + "reference": "514025c5ebbdb8a058745b573b4a1e81d685802c", + "shasum": "" + }, + "require": { + "php": ">=7.0" + }, + "require-dev": { + "ext-json": "*", + "friendsofphp/php-cs-fixer": "^3.0", + "phpunit/phpunit": "^8.0" + }, + "suggest": { + "ext-json": "To run JSON Machine out of the box without custom decoders.", + "guzzlehttp/guzzle": "To run example with GuzzleHttp" + }, + "type": "library", + "autoload": { + "psr-4": { + "JsonMachine\\": "src/" + }, + "exclude-from-classmap": [ + "src/autoloader.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Filip Halaxa", + "email": "filip@halaxa.cz" + } + ], + "description": "Efficient, easy-to-use and fast JSON pull parser", + "support": { + "issues": "https://github.com/halaxa/json-machine/issues", + "source": "https://github.com/halaxa/json-machine/tree/1.1.3" + }, + "funding": [ + { + "url": "https://ko-fi.com/G2G57KTE4", + "type": "other" + } + ], + "time": "2022-10-12T11:40:33+00:00" + }, { "name": "laravel/framework", "version": "v9.40.1", diff --git a/config/app.php b/config/app.php index ef76a7e..e01fa28 100644 --- a/config/app.php +++ b/config/app.php @@ -193,6 +193,7 @@ return [ App\Providers\AuthServiceProvider::class, // App\Providers\BroadcastServiceProvider::class, App\Providers\EventServiceProvider::class, + App\Providers\HorizonServiceProvider::class, App\Providers\RouteServiceProvider::class, ], diff --git a/config/horizon.php b/config/horizon.php new file mode 100644 index 0000000..1e05c1b --- /dev/null +++ b/config/horizon.php @@ -0,0 +1,197 @@ + env('HORIZON_DOMAIN'), + + /* + |-------------------------------------------------------------------------- + | Horizon Path + |-------------------------------------------------------------------------- + | + | This is the URI path where Horizon will be accessible from. Feel free + | to change this path to anything you like. Note that the URI will not + | affect the paths of its internal API that aren't exposed to users. + | + */ + + 'path' => env('HORIZON_PATH', 'horizon'), + + /* + |-------------------------------------------------------------------------- + | Horizon Redis Connection + |-------------------------------------------------------------------------- + | + | This is the name of the Redis connection where Horizon will store the + | meta information required for it to function. It includes the list + | of supervisors, failed jobs, job metrics, and other information. + | + */ + + 'use' => 'default', + + /* + |-------------------------------------------------------------------------- + | Horizon Redis Prefix + |-------------------------------------------------------------------------- + | + | This prefix will be used when storing all Horizon data in Redis. You + | may modify the prefix when you are running multiple installations + | of Horizon on the same server so that they don't have problems. + | + */ + + 'prefix' => env( + 'HORIZON_PREFIX', + Str::slug(env('APP_NAME', 'laravel'), '_').'_horizon:' + ), + + /* + |-------------------------------------------------------------------------- + | Horizon Route Middleware + |-------------------------------------------------------------------------- + | + | These middleware will get attached onto each Horizon route, giving you + | the chance to add your own middleware to this list or change any of + | the existing middleware. Or, you can simply stick with this list. + | + */ + + 'middleware' => ['web'], + + /* + |-------------------------------------------------------------------------- + | Queue Wait Time Thresholds + |-------------------------------------------------------------------------- + | + | This option allows you to configure when the LongWaitDetected event + | will be fired. Every connection / queue combination may have its + | own, unique threshold (in seconds) before this event is fired. + | + */ + + 'waits' => [ + 'redis:default' => 60, + ], + + /* + |-------------------------------------------------------------------------- + | Job Trimming Times + |-------------------------------------------------------------------------- + | + | Here you can configure for how long (in minutes) you desire Horizon to + | persist the recent and failed jobs. Typically, recent jobs are kept + | for one hour while all failed jobs are stored for an entire week. + | + */ + + 'trim' => [ + 'recent' => 60, + 'pending' => 60, + 'completed' => 60, + 'recent_failed' => 10080, + 'failed' => 10080, + 'monitored' => 10080, + ], + + /* + |-------------------------------------------------------------------------- + | Metrics + |-------------------------------------------------------------------------- + | + | Here you can configure how many snapshots should be kept to display in + | the metrics graph. This will get used in combination with Horizon's + | `horizon:snapshot` schedule to define how long to retain metrics. + | + */ + + 'metrics' => [ + 'trim_snapshots' => [ + 'job' => 24, + 'queue' => 24, + ], + ], + + /* + |-------------------------------------------------------------------------- + | Fast Termination + |-------------------------------------------------------------------------- + | + | When this option is enabled, Horizon's "terminate" command will not + | wait on all of the workers to terminate unless the --wait option + | is provided. Fast termination can shorten deployment delay by + | allowing a new instance of Horizon to start while the last + | instance will continue to terminate each of its workers. + | + */ + + 'fast_termination' => false, + + /* + |-------------------------------------------------------------------------- + | Memory Limit (MB) + |-------------------------------------------------------------------------- + | + | This value describes the maximum amount of memory the Horizon master + | supervisor may consume before it is terminated and restarted. For + | configuring these limits on your workers, see the next section. + | + */ + + 'memory_limit' => 64, + + /* + |-------------------------------------------------------------------------- + | Queue Worker Configuration + |-------------------------------------------------------------------------- + | + | Here you may define the queue worker settings used by your application + | in all environments. These supervisors and settings handle all your + | queued jobs and will be provisioned by Horizon during deployment. + | + */ + + 'defaults' => [ + 'supervisor-1' => [ + 'connection' => 'redis', + 'queue' => ['default'], + 'balance' => 'auto', + 'maxProcesses' => 1, + 'maxTime' => 0, + 'maxJobs' => 0, + 'memory' => 128, + 'tries' => 1, + 'timeout' => 60, + 'nice' => 0, + ], + ], + + 'environments' => [ + 'production' => [ + 'supervisor-1' => [ + 'maxProcesses' => 10, + 'balanceMaxShift' => 1, + 'balanceCooldown' => 3, + ], + ], + + 'local' => [ + 'supervisor-1' => [ + 'maxProcesses' => 3, + ], + ], + ], +]; diff --git a/database/migrations/2014_10_12_000000_create_users_table.php b/database/migrations/2014_10_12_000000_create_users_table.php index cf6b776..4b54afd 100644 --- a/database/migrations/2014_10_12_000000_create_users_table.php +++ b/database/migrations/2014_10_12_000000_create_users_table.php @@ -16,10 +16,13 @@ return new class extends Migration Schema::create('users', function (Blueprint $table) { $table->id(); $table->string('name'); - $table->string('email')->unique(); - $table->timestamp('email_verified_at')->nullable(); - $table->string('password'); - $table->rememberToken(); + $table->text('address'); + $table->boolean('checked'); + $table->string('email'); + $table->text('description')->nullable(); + $table->text('interest')->nullable(); + $table->dateTimeTz('date_of_birth'); + $table->json('credit_card'); $table->timestamps(); }); } diff --git a/database/migrations/2014_10_12_100000_create_password_resets_table.php b/database/migrations/2014_10_12_100000_create_password_resets_table.php deleted file mode 100644 index fcacb80..0000000 --- a/database/migrations/2014_10_12_100000_create_password_resets_table.php +++ /dev/null @@ -1,32 +0,0 @@ -string('email')->index(); - $table->string('token'); - $table->timestamp('created_at')->nullable(); - }); - } - - /** - * Reverse the migrations. - * - * @return void - */ - public function down() - { - Schema::dropIfExists('password_resets'); - } -}; diff --git a/database/migrations/2019_12_14_000001_create_personal_access_tokens_table.php b/database/migrations/2019_12_14_000001_create_personal_access_tokens_table.php deleted file mode 100644 index 6c81fd2..0000000 --- a/database/migrations/2019_12_14_000001_create_personal_access_tokens_table.php +++ /dev/null @@ -1,37 +0,0 @@ -id(); - $table->morphs('tokenable'); - $table->string('name'); - $table->string('token', 64)->unique(); - $table->text('abilities')->nullable(); - $table->timestamp('last_used_at')->nullable(); - $table->timestamp('expires_at')->nullable(); - $table->timestamps(); - }); - } - - /** - * Reverse the migrations. - * - * @return void - */ - public function down() - { - Schema::dropIfExists('personal_access_tokens'); - } -};