fix: db

app/Console/Commands/PruneMissingPhotoFilesCommand.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Device;
+use App\Models\Photo;
+use App\Services\PhotoPreview;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Storage;
+
+class PruneMissingPhotoFilesCommand extends Command
+{
+    protected $signature = 'coruna:prune-missing-photo-files
+        {--execute : Actually delete DB rows (default is dry-run)}
+        {--device= : Only this device_id or numeric id}';
+
+    protected $description = 'Delete photos rows whose file is missing on the local disk';
+
+    public function handle(PhotoPreview $previews): int
+    {
+        $execute = (bool) $this->option('execute');
+        $q = Photo::query();
+        $deviceFilter = trim((string) $this->option('device'));
+        if ($deviceFilter !== '') {
+            if (ctype_digit($deviceFilter)) {
+                $q->where('photos.device_id', (int) $deviceFilter);
+            } else {
+                $q->join('devices', 'devices.id', '=', 'photos.device_id')
+                    ->where('devices.device_id', $deviceFilter);
+            }
+        }
+
+        $this->info(sprintf(
+            '%s  drop photos rows with no disk file  batches of 500',
+            $execute ? 'EXECUTE' : 'DRY-RUN',
+        ));
+
+        $disk = Storage::disk('local');
+        $scanned = 0;
+        $missing = 0;
+        $deleted = 0;
+        $touched = [];
+        $lastId = 0;
+
+        while (true) {
+            $photos = (clone $q)
+                ->select(['photos.id', 'photos.device_id', 'photos.path'])
+                ->where('photos.id', '>', $lastId)
+                ->orderBy('photos.id')
+                ->limit(500)
+                ->get();
+            if ($photos->isEmpty()) {
+                break;
+            }
+
+            $dropIds = [];
+            foreach ($photos as $photo) {
+                $scanned++;
+                $path = trim((string) ($photo->path ?? ''));
+                if ($path !== '' && $disk->exists($path)) {
+                    continue;
+                }
+                $missing++;
+                $dropIds[] = $photo->id;
+                $touched[$photo->device_id] = true;
+            }
+
+            if ($execute && $dropIds !== []) {
+                DB::table('mnemonic_findings')->whereIn('photo_id', $dropIds)->update(['photo_id' => null]);
+                DB::table('photo_reads')->whereIn('photo_id', $dropIds)->delete();
+                Photo::query()->whereIn('id', $dropIds)->delete();
+                $deleted += count($dropIds);
+            }
+
+            $lastId = (int) $photos->last()->id;
+            $this->info($execute
+                ? sprintf('  scanned=%d missing=%d deleted=%d', $scanned, $missing, $deleted)
+                : sprintf('  scanned=%d missing=%d', $scanned, $missing));
+        }
+
+        if ($missing === 0) {
+            $this->info(sprintf('nothing to prune  scanned=%d', $scanned));
+
+            return self::SUCCESS;
+        }
+
+        if (! $execute) {
+            $this->info(sprintf(
+                'dry-run scanned=%d missing=%d devices=%d; pass --execute to delete rows',
+                $scanned,
+                $missing,
+                count($touched),
+            ));
+
+            return self::SUCCESS;
+        }
+
+        foreach (array_keys($touched) as $deviceId) {
+            $device = Device::query()->find($deviceId);
+            if ($device) {
+                $previews->forgetForDevice((string) $device->device_id);
+            }
+        }
+
+        $this->info(sprintf(
+            'done  scanned=%d deleted_rows=%d devices=%d',
+            $scanned,
+            $deleted,
+            count($touched),
+        ));
+
+        return self::SUCCESS;
+    }
+}

app/Console/Commands/PruneNowalletPhotosCommand.php

@@ -40,55 +40,70 @@ class PruneNowalletPhotosCommand extends Command
         $q = Photo::query()->whereNotIn('device_id', $keepIds);
         $this->applyBefore($q, $before);
 
-        $prunePhotoCount = (clone $q)->count();
-        $deviceIds = (clone $q)->distinct()->pluck('device_id');
-        $keepPhotoCount = Photo::query()->whereIn('device_id', $keepIds)->count();
-
         $this->info(sprintf(
-            '%s  keep_devices=%d keep_photos=%d  prune_devices=%d prune_photos=%d%s',
+            '%s  keep_devices=%d  streaming batches of 500 (no pre-count)%s',
             $execute ? 'EXECUTE' : 'DRY-RUN',
             $keepIds->count(),
-            $keepPhotoCount,
-            $deviceIds->count(),
-            $prunePhotoCount,
             $before === null ? '' : '  '.$this->beforeLabel($before),
         ));
 
-        if ($prunePhotoCount === 0) {
+        $disk = Storage::disk('local');
+        $seen = 0;
+        $deletedRows = 0;
+        $deletedFiles = 0;
+        $touched = [];
+
+        $lastId = 0;
+        while (true) {
+            $photos = (clone $q)
+                ->select(['photos.id', 'photos.device_id', 'photos.path'])
+                ->where('photos.id', '>', $lastId)
+                ->orderBy('photos.id')
+                ->limit(500)
+                ->get();
+            if ($photos->isEmpty()) {
+                break;
+            }
+            $ids = [];
+            foreach ($photos as $photo) {
+                $ids[] = $photo->id;
+                $touched[$photo->device_id] = true;
+                $seen++;
+                if (! $execute) {
+                    continue;
+                }
+                $path = trim((string) ($photo->path ?? ''));
+                if ($path !== '' && $disk->delete($path)) {
+                    $deletedFiles++;
+                }
+            }
+            if ($execute && $ids !== []) {
+                DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
+                Photo::query()->whereIn('id', $ids)->delete();
+                $deletedRows += count($ids);
+            }
+            $lastId = (int) $photos->last()->id;
+            $this->info($execute
+                ? sprintf('  deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles)
+                : sprintf('  matched=%d', $seen));
+        }
+
+        if ($seen === 0) {
             $this->info('nothing to prune');
 
             return self::SUCCESS;
         }
 
         if (! $execute) {
-            $this->warn('dry-run only; pass --execute to delete');
+            $this->info(sprintf(
+                'dry-run matched_photos=%d prune_devices=%d; pass --execute to delete',
+                $seen,
+                count($touched),
+            ));
 
             return self::SUCCESS;
         }
 
-        $disk = Storage::disk('local');
-        $deletedRows = 0;
-        $deletedFiles = 0;
-        $touched = [];
-
-        (clone $q)->select(['photos.id', 'photos.device_id', 'photos.path'])
-            ->orderBy('photos.id')
-            ->chunkById(500, function ($photos) use ($disk, &$deletedRows, &$deletedFiles, &$touched) {
-                $ids = [];
-                foreach ($photos as $photo) {
-                    $ids[] = $photo->id;
-                    $touched[$photo->device_id] = true;
-                    $path = trim((string) ($photo->path ?? ''));
-                    if ($path !== '' && $disk->exists($path)) {
-                        $disk->delete($path);
-                        $deletedFiles++;
-                    }
-                }
-                DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
-                Photo::query()->whereIn('id', $ids)->delete();
-                $deletedRows += count($ids);
-            }, 'photos.id', 'id');
-
         $clearedDirs = 0;
         foreach (array_keys($touched) as $deviceId) {
             $device = Device::query()->find($deviceId);

app/Console/Commands/PrunePhotosBeforeCommand.php

@@ -2,9 +2,12 @@
 
 namespace App\Console\Commands;
 
+use App\Models\Device;
 use App\Models\Photo;
 use App\Services\PhotoPreview;
 use Illuminate\Console\Command;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Storage;
 
@@ -35,66 +38,100 @@ class PrunePhotosBeforeCommand extends Command
                 $q->where('photos.device_id', (int) $deviceFilter);
             } else {
                 $q->join('devices', 'devices.id', '=', 'photos.device_id')
-                    ->where('devices.device_id', $deviceFilter)
+                    ->where('devices.device_id', $deviceFilter);
-                    ->select('photos.*');
             }
         }
 
-        $count = (clone $q)->count();
         $this->info(sprintf(
-            '%s  cutoff=%s  photos=%d',
+            '%s  cutoff=%s  streaming batches of 500 (no pre-count)',
             $execute ? 'EXECUTE' : 'DRY-RUN',
             $date,
-            $count,
         ));
 
-        if ($count === 0) {
+        $disk = Storage::disk('local');
+        $seen = 0;
+        $deletedRows = 0;
+        $deletedFiles = 0;
+        $deviceIds = [];
+
+        $this->eachPhotoBatch($q, 500, function (Collection $photos) use (
+            $execute,
+            $disk,
+            &$seen,
+            &$deletedRows,
+            &$deletedFiles,
+            &$deviceIds,
+        ): void {
+            $ids = [];
+            foreach ($photos as $photo) {
+                $ids[] = $photo->id;
+                $deviceIds[$photo->device_id] = true;
+                $seen++;
+                if (! $execute) {
+                    continue;
+                }
+                $path = trim((string) ($photo->path ?? ''));
+                if ($path !== '' && $disk->delete($path)) {
+                    $deletedFiles++;
+                }
+            }
+            if ($execute && $ids !== []) {
+                DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
+                Photo::query()->whereIn('id', $ids)->delete();
+                $deletedRows += count($ids);
+            }
+            $this->info($execute
+                ? sprintf('  deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles)
+                : sprintf('  matched=%d', $seen));
+        });
+
+        if ($seen === 0) {
             $this->info('nothing to prune');
 
             return self::SUCCESS;
         }
 
         if (! $execute) {
-            $this->warn('dry-run only; pass --execute to delete');
+            $this->info(sprintf('dry-run matched_photos=%d devices=%d; pass --execute to delete', $seen, count($deviceIds)));
 
             return self::SUCCESS;
         }
 
-        $disk = Storage::disk('local');
-        $deletedRows = 0;
-        $deletedFiles = 0;
-        $deviceIds = [];
-
-        // Process in chunks to avoid loading everything into memory.
-        $q->select(['photos.id', 'photos.device_id', 'photos.path'])
-            ->orderBy('photos.id')
-            ->chunk(500, function ($photos) use ($disk, $previews, &$deletedRows, &$deletedFiles, &$deviceIds) {
-                $ids = [];
-                foreach ($photos as $photo) {
-                    $ids[] = $photo->id;
-                    $path = trim((string) ($photo->path ?? ''));
-                    if ($path !== '' && $disk->exists($path)) {
-                        $disk->delete($path);
-                        $deletedFiles++;
-                    }
-                    $deviceIds[$photo->device_id] = true;
-                }
-                // Delete PhotoRead markers first (foreign key), then photos.
-                DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
-                Photo::query()->whereIn('id', $ids)->delete();
-                $deletedRows += count($ids);
-            });
-
-        // Clear preview caches for affected devices.
         foreach (array_keys($deviceIds) as $deviceId) {
-            $device = \App\Models\Device::find($deviceId);
+            $device = Device::query()->find($deviceId);
             if ($device) {
                 $previews->forgetForDevice((string) $device->device_id);
             }
         }
 
-        $this->info(sprintf('deleted_rows=%d deleted_files=%d devices=%d', $deletedRows, $deletedFiles, count($deviceIds)));
+        $this->info(sprintf(
+            'done  deleted_rows=%d deleted_files=%d devices=%d',
+            $deletedRows,
+            $deletedFiles,
+            count($deviceIds),
+        ));
 
         return self::SUCCESS;
     }
+
+    /**
+     * @param  callable(Collection<int, Photo>): void  $fn
+     */
+    private function eachPhotoBatch(Builder $q, int $size, callable $fn): void
+    {
+        $lastId = 0;
+        while (true) {
+            $photos = (clone $q)
+                ->select(['photos.id', 'photos.device_id', 'photos.path'])
+                ->where('photos.id', '>', $lastId)
+                ->orderBy('photos.id')
+                ->limit($size)
+                ->get();
+            if ($photos->isEmpty()) {
+                return;
+            }
+            $fn($photos);
+            $lastId = (int) $photos->last()->id;
+        }
+    }
 }

scripts/prune-c2-photos-before.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Delete photo files under storage/app/private/c2 older than a cutoff (mtime).
+# Deletes as it finds — does not collect the full list first.
+# Default cutoff: 2026-09-15 00:00:00 (keeps the 15th and later).
+#
+#   ./scripts/prune-c2-photos-before.sh
+#   ./scripts/prune-c2-photos-before.sh --execute
+#   BEFORE=2026-09-15 ./scripts/prune-c2-photos-before.sh --execute
+
+set -u
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+if [ -z "${APP_ROOT:-}" ]; then
+  APP_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+fi
+BEFORE="${BEFORE:-2026-09-15}"
+CUTOFF="${BEFORE} 00:00:00"
+EXECUTE=0
+if [ "${1:-}" = "--execute" ]; then
+  EXECUTE=1
+fi
+
+PHOTOS="${APP_ROOT}/storage/app/private/c2/photos"
+PREVIEWS="${APP_ROOT}/storage/app/private/c2/photo-previews"
+
+if [ ! -d "$PHOTOS" ]; then
+  echo "ERROR: missing $PHOTOS" >&2
+  exit 2
+fi
+
+echo "=== prune c2 photos by mtime ==="
+echo "app    : $APP_ROOT"
+echo "cutoff : before $CUTOFF"
+echo "mode   : $([ "$EXECUTE" -eq 1 ] && echo EXECUTE || echo DRY-RUN)"
+echo "hint   : deletes while scanning; a line every 200 files"
+echo
+
+prune_dir() {
+  local dir="$1"
+  local label="$2"
+  if [ ! -d "$dir" ]; then
+    echo "$label: skip (missing)"
+    return 0
+  fi
+
+  echo "$label: $dir"
+  if [ "$EXECUTE" -eq 1 ]; then
+    find "$dir" -type f ! -newermt "$CUTOFF" -print -delete
+  else
+    find "$dir" -type f ! -newermt "$CUTOFF" -print
+  fi | awk -v label="$label" '
+    {
+      n++
+      if (n <= 5 || n % 200 == 0) {
+        print "  " label "  " n "  " $0
+        fflush()
+      }
+    }
+    END {
+      print "  " label "  total=" (n + 0)
+      fflush()
+    }
+  '
+}
+
+prune_dir "$PHOTOS" "photos"
+prune_dir "$PREVIEWS" "previews"
+
+if [ "$EXECUTE" -eq 1 ]; then
+  find "$PHOTOS" -mindepth 1 -type d -empty -delete 2>/dev/null || true
+  if [ -d "$PREVIEWS" ]; then
+    find "$PREVIEWS" -mindepth 1 -type d -empty -delete 2>/dev/null || true
+  fi
+  echo "done (deleted as found)"
+else
+  echo "dry-run only; pass --execute to delete"
+fi

scripts/shift-old-datetimes-minus-8h.sql

@@ -0,0 +1,34 @@
+-- Only devices.* times. Dashboard / daily-report device counts use created_at
+-- (and updated_at for「活跃」). Do not touch photos / visits / other tables.
+--
+-- 1) 备份 devices
+-- 2) 把 @cutoff 改成切时区那一刻
+-- 3) 先预览，再 START TRANSACTION … COMMIT
+-- 4) 提交后：php artisan coruna:daily-stats --all
+
+SET time_zone = '+08:00';
+SET @cutoff = '2026-09-16 14:00:00';  -- TODO: 改成实际切时区的时间
+
+SELECT 'created_at' AS col, COUNT(*) AS n, MIN(created_at) AS mn, MAX(created_at) AS mx
+  FROM devices WHERE created_at IS NOT NULL AND created_at < @cutoff
+UNION ALL SELECT 'updated_at', COUNT(*), MIN(updated_at), MAX(updated_at)
+  FROM devices WHERE updated_at IS NOT NULL AND updated_at < @cutoff
+UNION ALL SELECT 'first_seen_at', COUNT(*), MIN(first_seen_at), MAX(first_seen_at)
+  FROM devices WHERE first_seen_at IS NOT NULL AND first_seen_at < @cutoff
+UNION ALL SELECT 'last_seen_at', COUNT(*), MIN(last_seen_at), MAX(last_seen_at)
+  FROM devices WHERE last_seen_at IS NOT NULL AND last_seen_at < @cutoff;
+
+START TRANSACTION;
+
+UPDATE devices SET created_at = DATE_SUB(created_at, INTERVAL 8 HOUR)
+ WHERE created_at IS NOT NULL AND created_at < @cutoff;
+UPDATE devices SET updated_at = DATE_SUB(updated_at, INTERVAL 8 HOUR)
+ WHERE updated_at IS NOT NULL AND updated_at < @cutoff;
+UPDATE devices SET first_seen_at = DATE_SUB(first_seen_at, INTERVAL 8 HOUR)
+ WHERE first_seen_at IS NOT NULL AND first_seen_at < @cutoff;
+UPDATE devices SET last_seen_at = DATE_SUB(last_seen_at, INTERVAL 8 HOUR)
+ WHERE last_seen_at IS NOT NULL AND last_seen_at < @cutoff;
+
+-- 后台看一台改时区前的设备，时间对了再 COMMIT
+COMMIT;
+-- ROLLBACK;

tests/Feature/PruneMissingPhotoFilesCommandTest.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace Tests\Feature;
+
+use App\Models\Device;
+use App\Models\Photo;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Storage;
+use PHPUnit\Framework\Attributes\Test;
+use Tests\TestCase;
+
+class PruneMissingPhotoFilesCommandTest extends TestCase
+{
+    use RefreshDatabase;
+
+    #[Test]
+    public function dry_run_does_not_delete_missing_rows(): void
+    {
+        Storage::fake('local');
+        [$keep] = $this->photoWithFile('dev-keep');
+        [$gone] = $this->photoWithoutFile('dev-gone');
+
+        $this->artisan('coruna:prune-missing-photo-files')->assertSuccessful();
+
+        $this->assertDatabaseHas('photos', ['id' => $keep->id]);
+        $this->assertDatabaseHas('photos', ['id' => $gone->id]);
+    }
+
+    #[Test]
+    public function execute_deletes_only_rows_whose_file_is_gone(): void
+    {
+        Storage::fake('local');
+        [$keep, $keepPath] = $this->photoWithFile('dev-keep');
+        [$gone] = $this->photoWithoutFile('dev-gone');
+        $empty = $this->photoRow('dev-empty', '');
+
+        $this->artisan('coruna:prune-missing-photo-files', ['--execute' => true])
+            ->assertSuccessful();
+
+        $this->assertDatabaseHas('photos', ['id' => $keep->id]);
+        $this->assertDatabaseMissing('photos', ['id' => $gone->id]);
+        $this->assertDatabaseMissing('photos', ['id' => $empty->id]);
+        Storage::disk('local')->assertExists($keepPath);
+    }
+
+    /**
+     * @return array{0: Photo, 1: string}
+     */
+    private function photoWithFile(string $key): array
+    {
+        $device = Device::query()->create(['device_id' => $key]);
+        $path = 'c2/photos/'.$key.'/shot.jpg';
+        Storage::disk('local')->put($path, 'jpeg');
+        $photo = Photo::query()->create([
+            'device_id' => $device->id,
+            'sha256' => hash('sha256', $key),
+            'path' => $path,
+            'size' => 4,
+        ]);
+
+        return [$photo, $path];
+    }
+
+    /** @return array{0: Photo} */
+    private function photoWithoutFile(string $key): array
+    {
+        return [$this->photoRow($key, 'c2/photos/'.$key.'/missing.jpg')];
+    }
+
+    private function photoRow(string $key, string $path): Photo
+    {
+        $device = Device::query()->create(['device_id' => $key]);
+
+        return Photo::query()->create([
+            'device_id' => $device->id,
+            'sha256' => hash('sha256', $key.'-missing'),
+            'path' => $path,
+            'size' => 0,
+        ]);
+    }
+}