file chunks refactoring

This commit is contained in:
Čarodej
2022-04-19 09:40:31 +02:00
parent 6c3e7801da
commit 19e29e69e0
19 changed files with 240 additions and 204 deletions

View File

@@ -47,19 +47,19 @@ return [
'endpoint' => env('S3_URL'),
],
'ftp' => [
'driver' => 'ftp',
'host' => env('FTP_HOST'),
'driver' => 'ftp',
'host' => env('FTP_HOST'),
'username' => env('FTP_USERNAME'),
'password' => env('FTP_PASSWORD'),
],
'azure' => [
'driver' => 'azure',
'name' => env('AZURE_STORAGE_NAME'),
'key' => env('AZURE_STORAGE_KEY'),
'container' => env('AZURE_STORAGE_CONTAINER'),
'url' => env('AZURE_STORAGE_URL'),
'prefix' => null,
'connection_string' => env('AZURE_STORAGE_CONNECTION_STRING') // optional, will override default endpoint builder
'driver' => 'azure',
'name' => env('AZURE_STORAGE_NAME'),
'key' => env('AZURE_STORAGE_KEY'),
'container' => env('AZURE_STORAGE_CONTAINER'),
'url' => env('AZURE_STORAGE_URL'),
'prefix' => null,
'connection_string' => env('AZURE_STORAGE_CONNECTION_STRING'), // optional, will override default endpoint builder
],
],
];

View File

@@ -1,6 +1,6 @@
{
"/js/main.js": "/js/main.js",
"/chunks/request.js": "/chunks/request.js?id=0dad383d43899f08",
"/chunks/request.js": "/chunks/request.js?id=a255aafa7261e2ac",
"/chunks/request-upload.js": "/chunks/request-upload.js?id=c456f33bfbfd4038",
"/chunks/setup-wizard.js": "/chunks/setup-wizard.js?id=19a0784e59d768ec",
"/chunks/status-check.js": "/chunks/status-check.js?id=f40938d1fb43820f",
@@ -9,13 +9,13 @@
"/chunks/environment.js": "/chunks/environment.js?id=784c2442268b36dc",
"/chunks/app-setup.js": "/chunks/app-setup.js?id=cbe7bfed06400736",
"/chunks/admin-account.js": "/chunks/admin-account.js?id=78d257775f5fc485",
"/chunks/shared.js": "/chunks/shared.js?id=af0396de02d1e02f",
"/chunks/shared.js": "/chunks/shared.js?id=df78268616502614",
"/chunks/shared/browser.js": "/chunks/shared/browser.js?id=d2fff07a2bc7af3f",
"/chunks/shared/single-file.js": "/chunks/shared/single-file.js?id=a6063bed9be75a09",
"/chunks/shared/authenticate.js": "/chunks/shared/authenticate.js?id=b5519d193bce2339",
"/chunks/not-found.js": "/chunks/not-found.js?id=d31bd699138cf828",
"/chunks/temporary-unavailable.js": "/chunks/temporary-unavailable.js?id=26798085f527d955",
"/chunks/admin.js": "/chunks/admin.js?id=5710bcabdcf37f2c",
"/chunks/admin.js": "/chunks/admin.js?id=361574392a095c32",
"/chunks/dashboard.js": "/chunks/dashboard.js?id=5ab55a12214433c8",
"/chunks/invoices.js": "/chunks/invoices.js?id=799928609f57ca10",
"/chunks/subscriptions.js": "/chunks/subscriptions.js?id=a0c4f59d0ec4aee0",
@@ -57,12 +57,12 @@
"/chunks/sign-up.js": "/chunks/sign-up.js?id=80da89f329c514fc",
"/chunks/forgotten-password.js": "/chunks/forgotten-password.js?id=27cda9364b6593d8",
"/chunks/create-new-password.js": "/chunks/create-new-password.js?id=2f0401ee2fc148c4",
"/chunks/settings.js": "/chunks/settings.js?id=25d4749b73a5077f",
"/chunks/settings.js": "/chunks/settings.js?id=a0eae95a131d4fb1",
"/chunks/profile.js": "/chunks/profile.js?id=3e24bb5e1f52d4bb",
"/chunks/settings-password.js": "/chunks/settings-password.js?id=d00bf503d8126dc4",
"/chunks/settings-storage.js": "/chunks/settings-storage.js?id=092e324aad54656b",
"/chunks/billing.js": "/chunks/billing.js?id=115c25478cee576d",
"/chunks/platform.js": "/chunks/platform.js?id=1fb4ee8ee38bcaeb",
"/chunks/platform.js": "/chunks/platform.js?id=b2be2d8a25d580e0",
"/chunks/files.js": "/chunks/files.js?id=aaea9173f7697d6e",
"/chunks/recent-uploads.js": "/chunks/recent-uploads.js?id=4bab41df721a6fc6",
"/chunks/my-shared-items.js": "/chunks/my-shared-items.js?id=c62bc3eb07de20df",

View File

@@ -10,7 +10,7 @@
<div class="flex items-center">
<div class="mr-3 ml-0 flex items-center lg:mx-3">
<span
class="inline-block max-w-[150px] xs:max-w-[230px] md:max-w-[290px] overflow-hidden text-ellipsis whitespace-nowrap text-sm font-bold"
class="inline-block max-w-[150px] xs:max-w-[220px] md:max-w-[240px] overflow-hidden text-ellipsis whitespace-nowrap text-sm font-bold"
>
{{ currentFile.data.attributes.name }}
</span>

View File

@@ -231,7 +231,7 @@ const FunctionHelpers = {
attempts = 0
// Set form data
formData.set('filename', item.file.name)
formData.set('name', item.file.name)
formData.set('file', chunk, source_name)
formData.set('path', item.path)
formData.set('parent_id', item.parent_id)

View File

@@ -12,7 +12,6 @@ class MoveFileToFTPStorageAction
string $file,
string $userId
): void {
// Stream file object to ftp
Storage::putFileAs("files/$userId", config('filesystems.disks.local.root') . "/files/$userId/$file", $file, 'private');

View File

@@ -0,0 +1,98 @@
<?php
namespace Domain\Files\Actions;
use Illuminate\Support\Str;
use Domain\Files\Models\File;
use Domain\Folders\Models\Folder;
use Illuminate\Support\Facades\Storage;
use Domain\Files\Requests\UploadRequest;
use Domain\Traffic\Actions\RecordUploadAction;
class ProcessFileAction
{
public function __construct(
public RecordUploadAction $recordUpload,
public GetFileParentId $getFileParentId,
public StoreFileExifMetadataAction $storeExifMetadata,
public MoveFileToFTPStorageAction $moveFileToFTPStorage,
public ProcessImageThumbnailAction $createImageThumbnail,
public MoveFileToExternalStorageAction $moveFileToExternalStorage,
) {
}
/**
* Upload new file
*/
public function __invoke(
UploadRequest $request,
?string $userId,
string $chunkPath,
) {
// Get local disk instance
$localDisk = Storage::disk('local');
// Get file data
$size = $localDisk->size($chunkPath);
$mimetype = $localDisk->mimeType($chunkPath);
$name = Str::uuid() . '.' . $request->input('extension');
// Get upload limit
$uploadLimit = get_settings('upload_limit');
// File size handling
if ($uploadLimit && $size > format_bytes($uploadLimit)) {
abort(413);
}
// Get user
$user = $request->filled('parent_id')
? Folder::find($request->input('parent_id'))->getLatestParent()->user
: auth()->user();
// Check if user has enough space to upload file
if (! $user->canUpload($size)) {
// Delete file from chunk directory
$localDisk->delete($chunkPath);
// Set up response
$response = response([
'type' => 'error',
'message' => __t('user_action_not_allowed'),
], 401);
// Abort code
abort($response);
}
// Move file to user directory
$localDisk->move($chunkPath, "files/$user->id/$name");
// Create multiple image thumbnails
($this->createImageThumbnail)($name, $user->id);
// Move file to external storage
match (config('filesystems.default')) {
's3' => ($this->moveFileToExternalStorage)($name, $user->id),
'ftp', 'azure' => ($this->moveFileToFTPStorage)($name, $user->id),
default => null
};
// Create new file
$file = File::create([
'mimetype' => $request->input('extension'),
'type' => getFileType($mimetype),
'parent_id' => ($this->getFileParentId)($request, $user->id),
'name' => $request->input('name'),
'basename' => $name,
'filesize' => $size,
'user_id' => $user->id,
'creator_id' => auth()->id(),
]);
// Store file exif data
($this->storeExifMetadata)($file);
// Return new file
return $file;
}
}

View File

@@ -22,28 +22,33 @@ class ProcessImageThumbnailAction
* Create image thumbnail from uploaded image
*/
public function __invoke(
string $fileName,
$file,
string $userId
string $name,
string $userId,
): void {
// Create thumbnail from image
if (in_array($file->getClientMimeType(), $this->availableFormats)) {
// Make copy of file for the thumbnail generation
Storage::disk('local')->copy("files/$userId/{$fileName}", "temp/$userId/{$fileName}");
// Get local disk instance
$disk = Storage::disk('local');
// Create thumbnails instantly
($this->generateImageThumbnail)(
fileName: $fileName,
userId: $userId,
execution: 'immediately'
);
if (! in_array($disk->mimeType("files/$userId/$name"), $this->availableFormats)) {
return;
}
// Create thumbnails later
($this->generateImageThumbnail)->onQueue('high')->execute(
fileName: $fileName,
// Make copy of file for the thumbnail generation
$disk->copy("files/$userId/$name", "temp/$userId/$name");
// Create thumbnails instantly
($this->generateImageThumbnail)(
fileName: $name,
userId: $userId,
execution: 'immediately'
);
// Create thumbnails later
($this->generateImageThumbnail)
->onQueue('high')
->execute(
fileName: $name,
userId: $userId,
execution: 'later'
);
}
}
}

View File

@@ -0,0 +1,33 @@
<?php
namespace Domain\Files\Actions;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Storage;
use Domain\Files\Requests\UploadRequest;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
class StoreFileChunksAction
{
/**
* @throws FileNotFoundException
*/
public function __invoke(UploadRequest $request)
{
// Get uploaded file
$file = $request->file('file');
// Get chunk name
$name = $file->getClientOriginalName();
// Get chunk file path
$path = Storage::disk('local')->path("chunks/$name");
// Build the file
File::append($path, $file->get());
// If last chunk, then return file path
if ($request->boolean('is_last')) {
return "chunks/$name";
}
}
}

View File

@@ -1,18 +1,23 @@
<?php
namespace Domain\Files\Actions;
use Domain\Files\Models\File;
class StoreFileExifMetadataAction
{
public function __invoke($item, $file)
public function __invoke(File $file)
{
// Get exif metadata
$exif_data = get_image_meta_data($file);
$data = readExifData("files/$file->user_id/$file->basename");
if ($exif_data) {
// Convert array to collection
$data = json_decode(json_encode($exif_data));
if (is_null($data)) {
return;
}
$item->exif()->create([
$file
->exif()
->create([
'date_time_original' => $data->DateTimeOriginal ?? null,
'artist' => $data->OwnerName ?? null,
'width' => $data->COMPUTED->Width ?? null,
@@ -33,6 +38,5 @@ class StoreFileExifMetadataAction
'longitude_ref' => $data->GPSLongitudeRef ?? null,
'latitude_ref' => $data->GPSLatitudeRef ?? null,
]);
}
}
}

View File

@@ -1,111 +0,0 @@
<?php
namespace Domain\Files\Actions;
use Illuminate\Support\Str;
use Domain\Folders\Models\Folder;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Storage;
use Domain\Files\Requests\UploadRequest;
use Domain\Files\Models\File as UserFile;
use Domain\Traffic\Actions\RecordUploadAction;
use App\Users\Exceptions\InvalidUserActionException;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
class UploadFileAction
{
public function __construct(
public RecordUploadAction $recordUpload,
public GetFileParentId $getFileParentId,
public StoreFileExifMetadataAction $storeExifMetadata,
public MoveFileToFTPStorageAction $moveFileToFTPStorage,
public ProcessImageThumbnailAction $createImageThumbnail,
public MoveFileToExternalStorageAction $moveFileToExternalStorage,
) {
}
/**
* Upload new file
*
* @throws InvalidUserActionException|FileNotFoundException
*/
public function __invoke(
UploadRequest $request,
?string $userId = null,
) {
$file = $request->file('file');
$chunkName = $file->getClientOriginalName();
// File Path
$filePath = Storage::disk('local')->path('chunks/' . $chunkName);
// Generate file
File::append($filePath, $file->get());
// Size of file
$fileSize = File::size($filePath);
// Size of limit
$uploadLimit = get_settings('upload_limit');
// File size handling
if ($uploadLimit && $fileSize > format_bytes($uploadLimit)) {
abort(413);
}
// If last then process file
if ($request->boolean('is_last')) {
$disk_local = Storage::disk('local');
// File name
$fileName = Str::uuid() . '.' . $request->input('extension');
// Get user
$user = $request->filled('parent_id')
? Folder::find($request->input('parent_id'))->getLatestParent()->user
: auth()->user();
// File Info
$fileSize = $disk_local->size("chunks/$chunkName");
$fileMimetype = $disk_local->mimeType("chunks/$chunkName");
// Check if user has enough space to upload file
if (! $user->canUpload($fileSize)) {
Storage::disk('local')->delete("chunks/$chunkName");
throw new InvalidUserActionException();
}
// Move finished file from chunk to file-manager directory
$disk_local->move("chunks/$chunkName", "files/$user->id/$fileName");
// Create multiple image thumbnails
($this->createImageThumbnail)($fileName, $file, $user->id);
// Move files to external storage
match (config('filesystems.default')) {
's3' => ($this->moveFileToExternalStorage)($fileName, $user->id),
'ftp', 'azure' => ($this->moveFileToFTPStorage)($fileName, $user->id),
default => null
};
// Create new file
$item = UserFile::create([
'mimetype' => $request->input('extension'),
'type' => get_file_type($fileMimetype),
'parent_id' => ($this->getFileParentId)($request, $user->id),
'name' => $request->input('filename'),
'basename' => $fileName,
'filesize' => $fileSize,
'user_id' => $user->id,
'creator_id' => auth()->id(),
]);
// Store exif metadata for files
($this->storeExifMetadata)($item, $file);
// Return new file
return $item;
}
}
}

View File

@@ -1,45 +1,43 @@
<?php
namespace Domain\Files\Controllers;
use Domain\Files\Models\File;
use App\Http\Controllers\Controller;
use Domain\Files\Requests\UploadRequest;
use Domain\Files\Resources\FileResource;
use Domain\Files\Actions\UploadFileAction;
use Domain\Files\Actions\ProcessFileAction;
use Support\Demo\Actions\FakeUploadFileAction;
use App\Users\Exceptions\InvalidUserActionException;
use Domain\Files\Actions\StoreFileChunksAction;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
class UploadFileController extends Controller
{
public function __construct(
public UploadFileAction $uploadFiles,
public FakeUploadFileAction $fakeUploadFile,
public ProcessFileAction $processFie,
public FakeUploadFileAction $fakeUploadFile,
public StoreFileChunksAction $storeFileChunks,
) {
}
/**
* Upload file for authenticated master|editor user
*
* @throws FileNotFoundException
*/
public function __invoke(UploadRequest $request)
{
public function __invoke(UploadRequest $request) {
if (is_demo_account()) {
return ($this->fakeUploadFile)($request);
}
try {
// Upload and store file record
if (! $request->boolean('is_last')) {
return ($this->uploadFiles)($request);
}
// Store file chunks
$chunkPath = ($this->storeFileChunks)($request);
$file = ($this->uploadFiles)($request);
// Proceed after last chunk
if ($request->boolean('is_last')) {
// Process file
$file = ($this->processFie)($request, null, $chunkPath);
return response(new FileResource($file), 201);
} catch (InvalidUserActionException $e) {
return response([
'type' => 'error',
'message' => $e->getMessage(),
], 401);
}
}
}

View File

@@ -6,7 +6,7 @@ use Domain\Sharing\Models\Share;
use App\Http\Controllers\Controller;
use Domain\Files\Requests\UploadRequest;
use Domain\Files\Resources\FileResource;
use Domain\Files\Actions\UploadFileAction;
use Domain\Files\Actions\ProcessFileAction;
use Support\Demo\Actions\FakeUploadFileAction;
use App\Users\Exceptions\InvalidUserActionException;
use Domain\Sharing\Actions\ProtectShareRecordAction;
@@ -19,8 +19,8 @@ use Illuminate\Contracts\Filesystem\FileNotFoundException;
class VisitorUploadFileController extends Controller
{
public function __construct(
private UploadFileAction $uploadFile,
private FakeUploadFileAction $fakeUploadFile,
private ProcessFileAction $uploadFile,
private FakeUploadFileAction $fakeUploadFile,
private ProtectShareRecordAction $protectShareRecord,
private VerifyAccessToItemAction $verifyAccessToItem,
) {

View File

@@ -1,4 +1,5 @@
<?php
namespace Domain\Files\Requests;
use Domain\Admin\Rules\DisabledMimetypes;
@@ -24,9 +25,9 @@ class UploadRequest extends FormRequest
public function rules()
{
return [
'filename' => 'required|string',
'name' => 'required|string',
'parent_id' => 'nullable|uuid',
'path' => 'required|string',
'path' => 'sometimes|string',
'is_last' => 'sometimes|string',
'extension' => 'sometimes|string|nullable',
'file' => ['required', 'file', new DisabledMimetypes],

View File

@@ -1,11 +1,10 @@
<?php
namespace Domain\Settings\Actions;
use ErrorException;
use League\Flysystem\Ftp\UnableToAuthenticate;
use League\Flysystem\UnableToWriteFile;
use Storage;
use ErrorException;
use League\Flysystem\UnableToWriteFile;
use League\Flysystem\Ftp\UnableToAuthenticate;
class TestFTPConnectionAction
{
@@ -40,4 +39,4 @@ class TestFTPConnectionAction
);
}
}
}
}

View File

@@ -2,10 +2,10 @@
namespace Domain\Settings\Controllers;
use Artisan;
use Domain\Settings\Actions\TestFTPConnectionAction;
use Illuminate\Http\Response;
use Domain\Settings\DTO\S3CredentialsData;
use Domain\Settings\Actions\TestS3ConnectionAction;
use Domain\Settings\Actions\TestFTPConnectionAction;
use Domain\Settings\Requests\StoreStorageCredentialsRequest;
class StoreStorageCredentialsController
@@ -13,7 +13,8 @@ class StoreStorageCredentialsController
public function __construct(
private TestFTPConnectionAction $testFTPConnection,
private TestS3ConnectionAction $testS3Connection,
) {}
) {
}
/**
* Set new email credentials to .env file
@@ -27,11 +28,10 @@ class StoreStorageCredentialsController
$driver = match ($request->input('storage.driver')) {
's3', 'storj', 'spaces', 'wasabi', 'backblaze', 'oss', 'other' => 's3',
'local' => 'local',
'ftp' => 'ftp',
'ftp' => 'ftp',
};
if (! app()->runningUnitTests()) {
// Test driver connection
match ($driver) {
's3' => ($this->testS3Connection)(

View File

@@ -2,11 +2,11 @@
namespace Domain\SetupWizard\Controllers;
use Artisan;
use Domain\Settings\Actions\TestFTPConnectionAction;
use Illuminate\Http\JsonResponse;
use App\Http\Controllers\Controller;
use Domain\Settings\DTO\S3CredentialsData;
use Domain\Settings\Actions\TestS3ConnectionAction;
use Domain\Settings\Actions\TestFTPConnectionAction;
use Domain\Settings\Actions\TestSESConnectionAction;
use Domain\Settings\Actions\TestSMTPConnectionAction;
use Domain\Settings\Actions\TestMailgunConnectionAction;
@@ -22,7 +22,8 @@ class StoreEnvironmentSettingsController extends Controller
private TestSMTPConnectionAction $testSMTPConnection,
private TestMailgunConnectionAction $testMailgunConnection,
private TestPostmarkConnectionAction $testPostmarkConnection,
) {}
) {
}
/**
* Store environment setup
@@ -35,7 +36,7 @@ class StoreEnvironmentSettingsController extends Controller
$StorageDriver = match ($request->input('storage.driver')) {
's3', 'storj', 'spaces', 'wasabi', 'backblaze', 'oss', 'other' => 's3',
'local' => 'local',
'ftp' => 'ftp',
'ftp' => 'ftp',
};
// Test driver connection

View File

@@ -4,7 +4,7 @@ namespace Domain\UploadRequest\Controllers;
use DB;
use Domain\Folders\Models\Folder;
use Domain\Files\Resources\FileResource;
use Domain\Files\Actions\UploadFileAction;
use Domain\Files\Actions\ProcessFileAction;
use Domain\UploadRequest\Models\UploadRequest;
use App\Users\Exceptions\InvalidUserActionException;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
@@ -12,7 +12,7 @@ use Illuminate\Contracts\Filesystem\FileNotFoundException;
class UploadFilesForUploadRequestController
{
public function __construct(
private UploadFileAction $uploadFile,
private ProcessFileAction $uploadFile,
) {
}

View File

@@ -19,7 +19,7 @@ class FakeUploadFileAction
return [
'data' => [
'id' => Str::uuid(),
'type' => get_file_type($file->getMimeType()),
'type' => getFileType($file->getMimeType()),
'attributes' => [
'filesize' => Metric::bytes($file->getSize())->format(),
'name' => $request->input('filename'),

View File

@@ -210,7 +210,6 @@ if (! function_exists('setEnvironmentValue')) {
if (count($values) > 0) {
foreach ($values as $envKey => $envValue) {
$str .= "\n"; // In case the searched variable is in the last line without \n
$keyPosition = strpos($str, "{$envKey}=");
@@ -640,7 +639,7 @@ if (! function_exists('get_file_type')) {
/**
* Get file type from mimetype
*/
function get_file_type(string $fileMimetype): string
function getFileType(string $fileMimetype): string
{
// Get mimetype from file
$mimetype = explode('/', $fileMimetype);
@@ -747,22 +746,32 @@ if (! function_exists('getPrettyName')) {
}
}
if (! function_exists('get_image_meta_data')) {
if (! function_exists('readExifData')) {
/**
* Get exif data from jpeg image
*
* @param $file
* @return array|null
*/
function get_image_meta_data($file)
function readExifData(string $file): object|null
{
if (get_file_type_from_mimetype($file->getMimeType()) === 'jpeg') {
try {
// Try to get the exif data
return mb_convert_encoding(Image::make($file->getRealPath())->exif(), 'UTF8', 'UTF8');
} catch (\Exception $e) {
return null;
}
$disk = Storage::disk('local');
$type = get_file_type_from_mimetype(
$disk->mimeType($file)
);
if ($type !== 'jpeg') {
return null;
}
try {
// Try to get the exif data
$data = Image::make($disk->path($file))->exif();
// Encode data
$encodedData = mb_convert_encoding($data, 'UTF8', 'UTF8');
return json_decode(json_encode($encodedData));
} catch (Exception $e) {
return null;
}
}
}