diff --git a/.env.example b/.env.example index 00b6110..541583b 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ APP_NAME=Laravel APP_ENV=local -APP_KEY= +APP_KEY=base64:XzNXQDrJlhmfWWBMxGSbte6eMyNdFkNddlmPLUg55ZY= APP_DEBUG=true APP_URL=http://localhost @@ -9,22 +9,22 @@ LOG_DEPRECATIONS_CHANNEL=null LOG_LEVEL=debug DB_CONNECTION=mysql -DB_HOST=127.0.0.1 +DB_HOST=mariadb DB_PORT=3306 DB_DATABASE=laravel -DB_USERNAME=root -DB_PASSWORD= +DB_USERNAME=sail +DB_PASSWORD=password BROADCAST_DRIVER=log CACHE_DRIVER=file FILESYSTEM_DISK=local -QUEUE_CONNECTION=sync +QUEUE_CONNECTION=redis SESSION_DRIVER=file SESSION_LIFETIME=120 -MEMCACHED_HOST=127.0.0.1 +MEMCACHED_HOST=memcached -REDIS_HOST=127.0.0.1 +REDIS_HOST=redis REDIS_PASSWORD=null REDIS_PORT=6379 @@ -56,3 +56,6 @@ VITE_PUSHER_HOST="${PUSHER_HOST}" VITE_PUSHER_PORT="${PUSHER_PORT}" VITE_PUSHER_SCHEME="${PUSHER_SCHEME}" VITE_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}" + +WWWGROUP=1000 +WWWUSER=1000 diff --git a/README.md b/README.md index 6b841b5..d8cad62 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,16 @@ ## installation +- ``cp .env.example .env`` +- ``docker-compose up -d`` +- ``docker-compose exec -it laravel.test composer install`` +- ``docker-compose exec -it laravel.test php artisan migrate`` +- ``docker-compose exec -it laravel.test php artisan test`` + + +## Api route + +``POST => http://localhost/api/v1/convert/json`` + +### Params +'file' => your json file diff --git a/app/Enums/FileFormatEnum.php b/app/Enums/FileFormatEnum.php index 27deb5f..002a66c 100644 --- a/app/Enums/FileFormatEnum.php +++ b/app/Enums/FileFormatEnum.php @@ -17,11 +17,4 @@ enum FileFormatEnum: string return $chunker->handle($uuid, $path); } - - public function converter() - { - return match($this){ - self::JSON => new JsonToDatabaseConverter - }; - } } diff --git a/app/Http/Controllers/V1/ConvertController.php b/app/Http/Controllers/V1/ConvertController.php index bf70473..8ffc4d6 100644 --- a/app/Http/Controllers/V1/ConvertController.php +++ b/app/Http/Controllers/V1/ConvertController.php @@ -14,11 +14,13 @@ class ConvertController extends Controller { public function convert(FileFormatEnum $format, Request $request) { + // save file Storage::put( $request->file('file')->getClientOriginalName(), $request->file('file')->getContent(), ); + // push a queue ChunkFile::dispatch( Str::uuid(), $request->file('file')->getClientOriginalName(), diff --git a/app/Jobs/ChunkFile.php b/app/Jobs/ChunkFile.php index 06fd191..7b24869 100644 --- a/app/Jobs/ChunkFile.php +++ b/app/Jobs/ChunkFile.php @@ -37,11 +37,16 @@ class ChunkFile implements ShouldQueue */ public function handle() { + // chunk file (generator) $chunks = $this->format->chunker($this->uuid, $this->path); + foreach ($chunks as $key => $chunk) { StoreChunk::dispatch($chunk); + + // set latest key to redis (cut off handling) Redis::set($this->uuid . ":latest", $key); } + // delete file and clear redis Storage::delete($this->path); Redis::del($this->uuid . ":latest"); } diff --git a/app/Jobs/StoreChunk.php b/app/Jobs/StoreChunk.php index 2a941c9..2b02b0d 100644 --- a/app/Jobs/StoreChunk.php +++ b/app/Jobs/StoreChunk.php @@ -43,7 +43,8 @@ class StoreChunk implements ShouldQueue $chunk = (new Pipeline(app())) ->through([ AgeFilterPipe::class, - CreditCardFilter::class, + // uncomment if you want to filter by credit card! + // CreditCardFilter::class, ]) ->send($this->changeDateOfBirthToCarbon()) ->thenReturn(); diff --git a/app/Services/Chunkers/JsonChunker.php b/app/Services/Chunkers/JsonChunker.php index 4758eb1..12b8283 100644 --- a/app/Services/Chunkers/JsonChunker.php +++ b/app/Services/Chunkers/JsonChunker.php @@ -19,6 +19,7 @@ class JsonChunker implements ChunkerInterface { $chunk = []; + // get latest key for cut off handling $latest = Redis::get($uuid . ":latest") ?? 0; foreach (Items::fromFile(Storage::path($path), ['decoder' => new ExtJsonDecoder(true)]) as $key => $value) { diff --git a/docker/runtimes/8.1/supervisord.conf b/docker/runtimes/8.1/supervisord.conf index 9d28479..59a6574 100644 --- a/docker/runtimes/8.1/supervisord.conf +++ b/docker/runtimes/8.1/supervisord.conf @@ -12,3 +12,12 @@ stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 + +[program:php-supervisor] +command=/usr/bin/php /var/www/html/artisan horizon +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0