From bb469f25201e54d366606a393f2fa1aa406d1e52 Mon Sep 17 00:00:00 2001 From: Peter Papp Date: Sun, 14 Mar 2021 12:42:25 +0100 Subject: [PATCH] added delete_failed_files into SchedulerService.php --- app/Console/Kernel.php | 4 ++ .../FileManager/EditItemsController.php | 34 +++++++-------- .../FileManagerService.php} | 41 ++----------------- app/Services/SchedulerService.php | 36 ++++++++++++++++ tests/Feature/App/SchedulerTest.php | 29 +++++++++++++ 5 files changed, 90 insertions(+), 54 deletions(-) rename app/{Http/Tools/Editor.php => Services/FileManagerService.php} (94%) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 99c6b40d..cfda8496 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -34,6 +34,10 @@ class Kernel extends ConsoleKernel $schedule->call(function () use ($scheduler) { $scheduler->delete_old_zips(); + + if (!is_storage_driver(['local'])) { + $scheduler->delete_failed_files(); + } })->everySixHours(); // Run queue jobs every minute diff --git a/app/Http/Controllers/FileManager/EditItemsController.php b/app/Http/Controllers/FileManager/EditItemsController.php index c68a4415..d96c1787 100644 --- a/app/Http/Controllers/FileManager/EditItemsController.php +++ b/app/Http/Controllers/FileManager/EditItemsController.php @@ -14,7 +14,7 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\Auth; use App\Http\Controllers\Controller; use App\Http\Tools\Guardian; -use App\Http\Tools\Editor; +use App\Http\Tools\FileManagerService; use App\Models\Folder; use App\Models\File; use Exception; @@ -50,7 +50,7 @@ class EditItemsController extends Controller } // Create new folder - return Editor::create_folder($request); + return FileManagerService::create_folder($request); } /** @@ -77,7 +77,7 @@ class EditItemsController extends Controller Guardian::check_item_access($request->parent_id, $shared); // Create folder - return Editor::create_folder($request, $shared); + return FileManagerService::create_folder($request, $shared); } /** @@ -117,11 +117,11 @@ class EditItemsController extends Controller // If request have a change folder icon values set the folder icon if ($request->type === 'folder' && ($request->filled('emoji') || $request->filled('color'))) { - Editor::set_folder_icon($request, $id); + FileManagerService::set_folder_icon($request, $id); } // Rename Item - return Editor::rename_item($request, $id); + return FileManagerService::rename_item($request, $id); } /** @@ -160,11 +160,11 @@ class EditItemsController extends Controller // If request have a change folder icon values set the folder icon if ($request->type === 'folder' && $request->filled('icon')) { - Editor::set_folder_icon($request, $id); + FileManagerService::set_folder_icon($request, $id); } // Rename item - $item = Editor::rename_item($request, $id, $shared); + $item = FileManagerService::rename_item($request, $id, $shared); // Set public url if ($item->type !== 'folder') { @@ -215,7 +215,7 @@ class EditItemsController extends Controller } // Delete item - Editor::delete_item($item, $item['id']); + FileManagerService::delete_item($item, $item['id']); } return response(null, 204); @@ -258,7 +258,7 @@ class EditItemsController extends Controller } // Delete item - Editor::delete_item($file, $file['id'], $shared); + FileManagerService::delete_item($file, $file['id'], $shared); } // Return response return response('Done', 204); @@ -292,7 +292,7 @@ class EditItemsController extends Controller } // Return new uploaded file - return Editor::upload($request); + return FileManagerService::upload($request); } /** @@ -322,7 +322,7 @@ class EditItemsController extends Controller Guardian::check_item_access($request->folder_id, $shared); // Return new uploaded file - $new_file = Editor::upload($request, $shared); + $new_file = FileManagerService::upload($request, $shared); // Set public access url $new_file->setPublicUrl($token); @@ -363,7 +363,7 @@ class EditItemsController extends Controller abort(404, 'Requested folder doesn\'t exists.'); } - $zip = Editor::zip_folder($id); + $zip = FileManagerService::zip_folder($id); // Get file return response([ @@ -396,7 +396,7 @@ class EditItemsController extends Controller abort(404, 'Requested folder doesn\'t exists.'); } - $zip = Editor::zip_folder($id, $shared); + $zip = FileManagerService::zip_folder($id, $shared); // Get file return response([ @@ -440,7 +440,7 @@ class EditItemsController extends Controller ->whereIn('id', $request->input('items')) ->get(); - $zip = Editor::zip_files($files); + $zip = FileManagerService::zip_files($files); // Get file return response([ @@ -475,7 +475,7 @@ class EditItemsController extends Controller ->whereIn('id', $request->items) ->get(); - $zip = Editor::zip_files($files, $shared); + $zip = FileManagerService::zip_files($files, $shared); // Get file return response([ @@ -517,7 +517,7 @@ class EditItemsController extends Controller } // Move item - Editor::move($request, $to_id); + FileManagerService::move($request, $to_id); return response('Done!', 204); } @@ -566,7 +566,7 @@ class EditItemsController extends Controller } } - Editor::move($request, $request->to_id); + FileManagerService::move($request, $request->to_id); return response('Done!', 204); } diff --git a/app/Http/Tools/Editor.php b/app/Services/FileManagerService.php similarity index 94% rename from app/Http/Tools/Editor.php rename to app/Services/FileManagerService.php index 25d476c0..f7caac78 100644 --- a/app/Http/Tools/Editor.php +++ b/app/Services/FileManagerService.php @@ -1,6 +1,6 @@ allFiles('files'), - $local_disk->allFiles('chunks') - ])->collapse(); - - $files->each(function ($file) use ($local_disk) { - - // Get the file's last modification time. - $last_modified = $local_disk->lastModified($file); - - // Get diffInHours - $diff = Carbon::parse($last_modified)->diffInHours(Carbon::now()); - - // Delete if file is in local storage more than 24 hours - if ($diff > 24) { - - Log::info('Failed file or chunk ' . $file . ' deleted.'); - - // Delete file from local storage - $local_disk->delete($file); - } - }); - } - /** * Move file to external storage if is set * @@ -546,6 +511,7 @@ class Editor $client = $adapter->getClient(); // Prepare the upload parameters. + // TODO: replace local files with temp folder $uploader = new MultipartUploader($client, config('filesystems.disks.local.root') . '/files/' . $file, [ 'bucket' => $adapter->getBucket(), 'key' => 'files/' . $file @@ -570,6 +536,7 @@ class Editor } else { // Stream file object to s3 + // TODO: replace local files with temp folder Storage::putFileAs('files', config('filesystems.disks.local.root') . '/files/' . $file, $file, 'private'); } diff --git a/app/Services/SchedulerService.php b/app/Services/SchedulerService.php index 60586a22..a29a7c3e 100644 --- a/app/Services/SchedulerService.php +++ b/app/Services/SchedulerService.php @@ -7,6 +7,8 @@ namespace App\Services; use App\Models\Share; use App\Models\Zip; use Carbon\Carbon; +use Illuminate\Support\Facades\Log; +use Illuminate\Support\Facades\Storage; class SchedulerService { @@ -45,4 +47,38 @@ class SchedulerService } }); } + + /** + * Get and delete failed files older than 24 hours + */ + public function delete_failed_files(): void + { + $local_disk = Storage::disk('local'); + + // Get all files from storage + $files = collect([ + //$local_disk->allFiles('files'), + $local_disk->allFiles('chunks') + ])->collapse(); + + $files->each(function ($file) use ($local_disk) { + + // Get the file's last modification time. + $last_modified = $local_disk + ->lastModified($file); + + // Get diffInHours + $diff = Carbon::parse($last_modified) + ->diffInHours(Carbon::now()); + + // Delete if file is in local storage more than 24 hours + if ($diff >= 24) { + + Log::info("Failed file or chunk $file deleted."); + + // Delete file from local storage + $local_disk->delete($file); + } + }); + } } \ No newline at end of file diff --git a/tests/Feature/App/SchedulerTest.php b/tests/Feature/App/SchedulerTest.php index ccf36305..8808c3c4 100644 --- a/tests/Feature/App/SchedulerTest.php +++ b/tests/Feature/App/SchedulerTest.php @@ -70,4 +70,33 @@ class SchedulerTest extends TestCase Storage::disk('local') ->assertMissing('zip/EHWKcuvKzA4Gv29v-archive.zip'); } + + /** + * @test + */ + public function it_delete_failed_files_older_than_one_day() + { + Storage::fake('local'); + + $this->setup->create_directories(); + + $this->travel(-26)->hours(); + + $file = UploadedFile::fake() + ->create('fake-file.zip', 2000, 'application/zip'); + + collect(['chunks']) + ->each(function ($folder) use ($file){ + Storage::putFileAs($folder, $file, 'fake-file.zip'); + }); + + $this->scheduler->delete_failed_files(); + + collect(['chunks']) + ->each(function ($folder) { + Storage::disk('local') + ->assertMissing("$folder/fake-file.zip"); + }); + + } }