From a0e7781b066fb3541c81337ed1584022f785f752 Mon Sep 17 00:00:00 2001 From: Dillon Ngo Date: Tue, 13 Jan 2026 14:08:11 +0800 Subject: [PATCH] 1688 PO Automation Project, feedback from Brian --- ...TLPurchaseOrderTransactionV2CommandJob.php | 46 ++++++++- .../Services/CreatesClaudeResponse.php | 2 +- .../CreatesClaudeResponseWithFiles.php | 2 +- .../ImportPurchaseOrderTransactionLogic.php | 95 ++++++++++++++++++- ...reatePurchaseOrderTransactionProcessor.php | 9 +- .../ETLPDFPurchaseOrderClaudeProcessor.php | 6 +- docker-setup/nginx/default.conf | 2 + .../bookings/forms/PdfUpload1688Component.vue | 14 ++- 8 files changed, 159 insertions(+), 17 deletions(-) diff --git a/app/Classes/Jobs/Commands/V2/ETLPurchaseOrderTransactionV2CommandJob.php b/app/Classes/Jobs/Commands/V2/ETLPurchaseOrderTransactionV2CommandJob.php index cc26a6c3..072fdd85 100644 --- a/app/Classes/Jobs/Commands/V2/ETLPurchaseOrderTransactionV2CommandJob.php +++ b/app/Classes/Jobs/Commands/V2/ETLPurchaseOrderTransactionV2CommandJob.php @@ -21,15 +21,43 @@ class ETLPurchaseOrderTransactionV2CommandJob implements ShouldQueue /** @var int|null */ private $bookingId; + /** @var string */ + private $groupId; + + /** @var int */ + private $batchIndex; + + /** @var int */ + private $totalBatches; + + /** @var bool */ + private $isLast; + /** - * ETLPurchaseOrderTransactionV2CommandJob constructor. + * constructor. + * * @param array $files - * @param int $bookingId + * @param int|null $bookingId + * @param string $groupId + * @param int $batchIndex + * @param int $totalBatches + * @param bool $isLast */ - public function __construct(array $files, ?int $bookingId) - { + public function __construct( + array $files, + ?int $bookingId, + string $groupId, + int $batchIndex, + int $totalBatches, + bool $isLast = false + ) { $this->files = $files; $this->bookingId = $bookingId; + + $this->groupId = $groupId; + $this->batchIndex = $batchIndex; + $this->totalBatches = $totalBatches; + $this->isLast = $isLast; } public function handle() @@ -37,7 +65,15 @@ class ETLPurchaseOrderTransactionV2CommandJob implements ShouldQueue Log::info(Carbon::now() . ': Start job - ETL Purchase Order Transaction.'); $start = new Carbon(); - (App()->make(ETLPDFPurchaseOrderClaudeProcessor::class))->execute($this->files, $this->bookingId); + Log::info(Carbon::now() . ': Processing - ETL Purchase Order Transaction.', [ + 'group_id' => $this->groupId, + 'batch_index' => $this->batchIndex, + 'total_batches' => $this->totalBatches, + 'is_last' => $this->isLast, + 'file_count' => count($this->files), + ]); + + (App()->make(ETLPDFPurchaseOrderClaudeProcessor::class))->execute($this->files, $this->bookingId, $this->batchIndex !== 1); $end = new Carbon(); $elapsedTime = $start->diff($end)->format('%H:%I:%S'); diff --git a/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponse.php b/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponse.php index 2d89daa3..3e3fd830 100644 --- a/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponse.php +++ b/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponse.php @@ -19,7 +19,7 @@ class CreatesClaudeResponse public function execute(string $userPrompt, string $model = "") { try { // Default to the latest recommended Claude model - $modelId = $model ?? config('anthropic.default_model', 'claude-sonnet-4-5-20250929'); + $modelId = $model ?? config('anthropic.default_model', 'claude-haiku-4-5-20251001'); $data = [ 'model' => $modelId, diff --git a/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponseWithFiles.php b/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponseWithFiles.php index 0acd806c..ae888a6e 100644 --- a/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponseWithFiles.php +++ b/app/Classes/Modules/Anthropic/Services/CreatesClaudeResponseWithFiles.php @@ -37,7 +37,7 @@ class CreatesClaudeResponseWithFiles 'anthropic-beta' => 'files-api-2025-04-14', 'Content-Type' => 'application/json', ])->post(config('anthropic.base_url') . '/v1/messages', [ - 'model' => config('anthropic.default_model', 'claude-sonnet-4-5-20250929'), + 'model' => config('anthropic.default_model', 'claude-haiku-4-5-20251001'), 'max_tokens' => 16384, 'messages' => [ [ diff --git a/app/Classes/Modules/Transactions/ControllersLogic/ImportPurchaseOrderTransactionLogic.php b/app/Classes/Modules/Transactions/ControllersLogic/ImportPurchaseOrderTransactionLogic.php index aa5eb7e4..56972f5d 100644 --- a/app/Classes/Modules/Transactions/ControllersLogic/ImportPurchaseOrderTransactionLogic.php +++ b/app/Classes/Modules/Transactions/ControllersLogic/ImportPurchaseOrderTransactionLogic.php @@ -25,6 +25,7 @@ use App\Models\Document; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Str; use Maatwebsite\Excel\Facades\Excel; class ImportPurchaseOrderTransactionLogic extends AbstractControllerLogic @@ -98,7 +99,7 @@ class ImportPurchaseOrderTransactionLogic extends AbstractControllerLogic */ public function logic(Request $request, $id = ''): JsonResponse { - $products = []; // Initialize + $products = []; /** @var Booking $booking */ $booking = $this->fetchesBooking->execute(['id' => $request->route('id') ?? $id]); @@ -163,7 +164,8 @@ class ImportPurchaseOrderTransactionLogic extends AbstractControllerLogic //ETL only applicable for pdf file with 'EN' as part of its filename if ($totalSize > 0) { if ($totalSize > $maxSize) { - ETLPurchaseOrderTransactionV2CommandJob::dispatch($base64FilesEN, $request->route('id')); + // ETLPurchaseOrderTransactionV2CommandJob::dispatch($base64FilesEN, $request->route('id')); + $this->runAsJob($base64FilesEN, $request->route('id')); $this->notificationMessage = "Please refresh page in a few minutes"; return $this->response([]); } @@ -218,4 +220,93 @@ class ImportPurchaseOrderTransactionLogic extends AbstractControllerLogic return $this->resourceResponse(new TransactionResource($transaction)); } + + private function runAsJob($files, $bookingId){ + $maxFilesPerBatch = 4; + $maxBytesPerBatch = 350 * 1024; // 350 KB + + $currentBatch = []; + $currentBatchSize = 0; + $fileBatches = []; + + foreach ($files as $index => $fileBase64) { + if (preg_match('/^data:application\/pdf;base64,/', $fileBase64)) { + $cleanBase64 = preg_replace('/^data:application\/pdf;base64,/', '', $fileBase64); + } + else{ + $cleanBase64 = $fileBase64; + } + $fileContent = base64_decode($cleanBase64, true); + if ($fileContent === false) { + Log::info("Invalid Base64 for file index {$index}"); + continue; // skip invalid files + } + + $fileSize = strlen($fileContent); // size in bytes + + // If adding this file would exceed limits, start a new batch + $wouldExceedFileCount = count($currentBatch) >= $maxFilesPerBatch; + $wouldExceedTotalSize = ($currentBatchSize + $fileSize) > $maxBytesPerBatch; + + if ($wouldExceedFileCount || $wouldExceedTotalSize) { + // save current batch + $fileBatches[] = $currentBatch; + // reset + $currentBatch = []; + $currentBatchSize = 0; + } + + // Add file to batch + $currentBatch[] = $fileBase64; + $currentBatchSize += $fileSize; + + } + + // push final remaining batch + if (!empty($currentBatch)) { + $fileBatches[] = $currentBatch; + } + + $totalBatches = count($fileBatches); + $groupId = (string) Str::uuid(); + + foreach ($fileBatches as $index => $batch) { + + $isLast = ($index === $totalBatches - 1); + + // Calculate total size of this batch + $batchSize = 0; + foreach ($batch as $fileBase64) { + if (preg_match('/^data:application\/pdf;base64,/', $fileBase64)) { + $cleanBase64 = preg_replace('/^data:application\/pdf;base64,/', '', $fileBase64); + } + else{ + $cleanBase64 = $fileBase64; + } + $fileContent = base64_decode($cleanBase64, true); + if ($fileContent !== false) { + $batchSize += strlen($fileContent); + } + } + + // Log batch info before dispatching job + Log::info("Dispatching Job for Batch #" . ($index + 1), [ + 'group_id' => $groupId, + 'batch_index' => $index + 1, + 'total_batches' => $totalBatches, + 'is_last' => $isLast, + 'files_in_batch' => count($batch), + 'batch_size_bytes' => $batchSize, + ]); + + ETLPurchaseOrderTransactionV2CommandJob::dispatch( + $batch, + $bookingId, + $groupId, + $index + 1, + $totalBatches, + $isLast + ); + } + } } diff --git a/app/Classes/Modules/Transactions/Processors/CreatePurchaseOrderTransactionProcessor.php b/app/Classes/Modules/Transactions/Processors/CreatePurchaseOrderTransactionProcessor.php index 76773c75..8d6c7c71 100644 --- a/app/Classes/Modules/Transactions/Processors/CreatePurchaseOrderTransactionProcessor.php +++ b/app/Classes/Modules/Transactions/Processors/CreatePurchaseOrderTransactionProcessor.php @@ -53,10 +53,11 @@ class CreatePurchaseOrderTransactionProcessor /** * @param Booking $booking * @param TransactionObject $object + * @param bool $append * @return Transaction|\Illuminate\Database\Eloquent\Model * @throws \App\Classes\Exceptions\MalformedRequestException */ - public function execute(Booking $booking, TransactionObject $object){ + public function execute(Booking $booking, TransactionObject $object, bool $append = false){ /** @var Transaction $transaction */ $transaction = $booking->transactions()->where('type', TransactionType::PURCHASE_ORDER)->first(); @@ -64,7 +65,9 @@ class CreatePurchaseOrderTransactionProcessor $this->updatesTransactionStatus->execute($transaction, (float) number_format($object->getAmount(), 2, '.', '') === (float) number_format((float)$booking->fix_amount, 2, '.', '') ? ApprovalStatus::PENDING_VERIFICATION : ApprovalStatus::PENDING_SUBMISSION); - $this->deletesTransactionDetails->execute($transaction); + if(!$append){ + $this->deletesTransactionDetails->execute($transaction); + } foreach ($object->getDetails() as $product){ $this->createsTransactionDetail->execute($transaction, $product); @@ -73,4 +76,4 @@ class CreatePurchaseOrderTransactionProcessor return $transaction; } -} \ No newline at end of file +} diff --git a/app/Classes/Modules/Transactions/Processors/ETLPDFPurchaseOrderClaudeProcessor.php b/app/Classes/Modules/Transactions/Processors/ETLPDFPurchaseOrderClaudeProcessor.php index 9c227307..b633b32d 100644 --- a/app/Classes/Modules/Transactions/Processors/ETLPDFPurchaseOrderClaudeProcessor.php +++ b/app/Classes/Modules/Transactions/Processors/ETLPDFPurchaseOrderClaudeProcessor.php @@ -44,9 +44,10 @@ class ETLPDFPurchaseOrderClaudeProcessor /** * @param UploadedFile[] $files * @param int|null $bookingId + * @param bool $append * @return array|null */ - public function execute(array $files, ?int $bookingId = null): ?array + public function execute(array $files, ?int $bookingId = null, bool $append = false): ?array { $tempPaths = []; @@ -142,8 +143,7 @@ class ETLPDFPurchaseOrderClaudeProcessor $data ); - $this->createPurchaseOrderTransactionProcessor - ->execute($booking, $object); + $this->createPurchaseOrderTransactionProcessor->execute($booking, $object, $append); } return $data; diff --git a/docker-setup/nginx/default.conf b/docker-setup/nginx/default.conf index c1811261..4c178ce4 100644 --- a/docker-setup/nginx/default.conf +++ b/docker-setup/nginx/default.conf @@ -6,6 +6,8 @@ server { access_log /var/log/nginx/access.log; root /var/www/html/public; + client_max_body_size 10M; + server_name localhost; location / { diff --git a/resources/assets/vue/components/bookings/forms/PdfUpload1688Component.vue b/resources/assets/vue/components/bookings/forms/PdfUpload1688Component.vue index 9d34c1d2..63a76fb2 100644 --- a/resources/assets/vue/components/bookings/forms/PdfUpload1688Component.vue +++ b/resources/assets/vue/components/bookings/forms/PdfUpload1688Component.vue @@ -13,7 +13,17 @@ -
+
+
+ +
+
+