From 4a7e3f771eab44e8f705c2d5a8f12c91f02b0c6a Mon Sep 17 00:00:00 2001 From: hashbro Date: Wed, 16 Sep 2026 22:29:13 +0800 Subject: [PATCH] fix: db --- .../PruneMissingPhotoFilesCommand.php | 115 ++++++++++++++++++ .../Commands/PruneNowalletPhotosCommand.php | 81 +++++++----- .../Commands/PrunePhotosBeforeCommand.php | 107 ++++++++++------ scripts/prune-c2-photos-before.sh | 77 ++++++++++++ scripts/shift-old-datetimes-minus-8h.sql | 34 ++++++ .../PruneMissingPhotoFilesCommandTest.php | 81 ++++++++++++ 6 files changed, 427 insertions(+), 68 deletions(-) create mode 100644 app/Console/Commands/PruneMissingPhotoFilesCommand.php create mode 100755 scripts/prune-c2-photos-before.sh create mode 100644 scripts/shift-old-datetimes-minus-8h.sql create mode 100644 tests/Feature/PruneMissingPhotoFilesCommandTest.php diff --git a/app/Console/Commands/PruneMissingPhotoFilesCommand.php b/app/Console/Commands/PruneMissingPhotoFilesCommand.php new file mode 100644 index 0000000..1d28842 --- /dev/null +++ b/app/Console/Commands/PruneMissingPhotoFilesCommand.php @@ -0,0 +1,115 @@ +option('execute'); + $q = Photo::query(); + $deviceFilter = trim((string) $this->option('device')); + if ($deviceFilter !== '') { + if (ctype_digit($deviceFilter)) { + $q->where('photos.device_id', (int) $deviceFilter); + } else { + $q->join('devices', 'devices.id', '=', 'photos.device_id') + ->where('devices.device_id', $deviceFilter); + } + } + + $this->info(sprintf( + '%s drop photos rows with no disk file batches of 500', + $execute ? 'EXECUTE' : 'DRY-RUN', + )); + + $disk = Storage::disk('local'); + $scanned = 0; + $missing = 0; + $deleted = 0; + $touched = []; + $lastId = 0; + + while (true) { + $photos = (clone $q) + ->select(['photos.id', 'photos.device_id', 'photos.path']) + ->where('photos.id', '>', $lastId) + ->orderBy('photos.id') + ->limit(500) + ->get(); + if ($photos->isEmpty()) { + break; + } + + $dropIds = []; + foreach ($photos as $photo) { + $scanned++; + $path = trim((string) ($photo->path ?? '')); + if ($path !== '' && $disk->exists($path)) { + continue; + } + $missing++; + $dropIds[] = $photo->id; + $touched[$photo->device_id] = true; + } + + if ($execute && $dropIds !== []) { + DB::table('mnemonic_findings')->whereIn('photo_id', $dropIds)->update(['photo_id' => null]); + DB::table('photo_reads')->whereIn('photo_id', $dropIds)->delete(); + Photo::query()->whereIn('id', $dropIds)->delete(); + $deleted += count($dropIds); + } + + $lastId = (int) $photos->last()->id; + $this->info($execute + ? sprintf(' scanned=%d missing=%d deleted=%d', $scanned, $missing, $deleted) + : sprintf(' scanned=%d missing=%d', $scanned, $missing)); + } + + if ($missing === 0) { + $this->info(sprintf('nothing to prune scanned=%d', $scanned)); + + return self::SUCCESS; + } + + if (! $execute) { + $this->info(sprintf( + 'dry-run scanned=%d missing=%d devices=%d; pass --execute to delete rows', + $scanned, + $missing, + count($touched), + )); + + return self::SUCCESS; + } + + foreach (array_keys($touched) as $deviceId) { + $device = Device::query()->find($deviceId); + if ($device) { + $previews->forgetForDevice((string) $device->device_id); + } + } + + $this->info(sprintf( + 'done scanned=%d deleted_rows=%d devices=%d', + $scanned, + $deleted, + count($touched), + )); + + return self::SUCCESS; + } +} diff --git a/app/Console/Commands/PruneNowalletPhotosCommand.php b/app/Console/Commands/PruneNowalletPhotosCommand.php index 6328c07..bef3c74 100644 --- a/app/Console/Commands/PruneNowalletPhotosCommand.php +++ b/app/Console/Commands/PruneNowalletPhotosCommand.php @@ -40,55 +40,70 @@ class PruneNowalletPhotosCommand extends Command $q = Photo::query()->whereNotIn('device_id', $keepIds); $this->applyBefore($q, $before); - $prunePhotoCount = (clone $q)->count(); - $deviceIds = (clone $q)->distinct()->pluck('device_id'); - $keepPhotoCount = Photo::query()->whereIn('device_id', $keepIds)->count(); - $this->info(sprintf( - '%s keep_devices=%d keep_photos=%d prune_devices=%d prune_photos=%d%s', + '%s keep_devices=%d streaming batches of 500 (no pre-count)%s', $execute ? 'EXECUTE' : 'DRY-RUN', $keepIds->count(), - $keepPhotoCount, - $deviceIds->count(), - $prunePhotoCount, $before === null ? '' : ' '.$this->beforeLabel($before), )); - if ($prunePhotoCount === 0) { + $disk = Storage::disk('local'); + $seen = 0; + $deletedRows = 0; + $deletedFiles = 0; + $touched = []; + + $lastId = 0; + while (true) { + $photos = (clone $q) + ->select(['photos.id', 'photos.device_id', 'photos.path']) + ->where('photos.id', '>', $lastId) + ->orderBy('photos.id') + ->limit(500) + ->get(); + if ($photos->isEmpty()) { + break; + } + $ids = []; + foreach ($photos as $photo) { + $ids[] = $photo->id; + $touched[$photo->device_id] = true; + $seen++; + if (! $execute) { + continue; + } + $path = trim((string) ($photo->path ?? '')); + if ($path !== '' && $disk->delete($path)) { + $deletedFiles++; + } + } + if ($execute && $ids !== []) { + DB::table('photo_reads')->whereIn('photo_id', $ids)->delete(); + Photo::query()->whereIn('id', $ids)->delete(); + $deletedRows += count($ids); + } + $lastId = (int) $photos->last()->id; + $this->info($execute + ? sprintf(' deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles) + : sprintf(' matched=%d', $seen)); + } + + if ($seen === 0) { $this->info('nothing to prune'); return self::SUCCESS; } if (! $execute) { - $this->warn('dry-run only; pass --execute to delete'); + $this->info(sprintf( + 'dry-run matched_photos=%d prune_devices=%d; pass --execute to delete', + $seen, + count($touched), + )); return self::SUCCESS; } - $disk = Storage::disk('local'); - $deletedRows = 0; - $deletedFiles = 0; - $touched = []; - - (clone $q)->select(['photos.id', 'photos.device_id', 'photos.path']) - ->orderBy('photos.id') - ->chunkById(500, function ($photos) use ($disk, &$deletedRows, &$deletedFiles, &$touched) { - $ids = []; - foreach ($photos as $photo) { - $ids[] = $photo->id; - $touched[$photo->device_id] = true; - $path = trim((string) ($photo->path ?? '')); - if ($path !== '' && $disk->exists($path)) { - $disk->delete($path); - $deletedFiles++; - } - } - DB::table('photo_reads')->whereIn('photo_id', $ids)->delete(); - Photo::query()->whereIn('id', $ids)->delete(); - $deletedRows += count($ids); - }, 'photos.id', 'id'); - $clearedDirs = 0; foreach (array_keys($touched) as $deviceId) { $device = Device::query()->find($deviceId); diff --git a/app/Console/Commands/PrunePhotosBeforeCommand.php b/app/Console/Commands/PrunePhotosBeforeCommand.php index afd2d37..7f7c6eb 100644 --- a/app/Console/Commands/PrunePhotosBeforeCommand.php +++ b/app/Console/Commands/PrunePhotosBeforeCommand.php @@ -2,9 +2,12 @@ namespace App\Console\Commands; +use App\Models\Device; use App\Models\Photo; use App\Services\PhotoPreview; use Illuminate\Console\Command; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; @@ -35,66 +38,100 @@ class PrunePhotosBeforeCommand extends Command $q->where('photos.device_id', (int) $deviceFilter); } else { $q->join('devices', 'devices.id', '=', 'photos.device_id') - ->where('devices.device_id', $deviceFilter) - ->select('photos.*'); + ->where('devices.device_id', $deviceFilter); } } - $count = (clone $q)->count(); $this->info(sprintf( - '%s cutoff=%s photos=%d', + '%s cutoff=%s streaming batches of 500 (no pre-count)', $execute ? 'EXECUTE' : 'DRY-RUN', $date, - $count, )); - if ($count === 0) { + $disk = Storage::disk('local'); + $seen = 0; + $deletedRows = 0; + $deletedFiles = 0; + $deviceIds = []; + + $this->eachPhotoBatch($q, 500, function (Collection $photos) use ( + $execute, + $disk, + &$seen, + &$deletedRows, + &$deletedFiles, + &$deviceIds, + ): void { + $ids = []; + foreach ($photos as $photo) { + $ids[] = $photo->id; + $deviceIds[$photo->device_id] = true; + $seen++; + if (! $execute) { + continue; + } + $path = trim((string) ($photo->path ?? '')); + if ($path !== '' && $disk->delete($path)) { + $deletedFiles++; + } + } + if ($execute && $ids !== []) { + DB::table('photo_reads')->whereIn('photo_id', $ids)->delete(); + Photo::query()->whereIn('id', $ids)->delete(); + $deletedRows += count($ids); + } + $this->info($execute + ? sprintf(' deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles) + : sprintf(' matched=%d', $seen)); + }); + + if ($seen === 0) { $this->info('nothing to prune'); return self::SUCCESS; } if (! $execute) { - $this->warn('dry-run only; pass --execute to delete'); + $this->info(sprintf('dry-run matched_photos=%d devices=%d; pass --execute to delete', $seen, count($deviceIds))); return self::SUCCESS; } - $disk = Storage::disk('local'); - $deletedRows = 0; - $deletedFiles = 0; - $deviceIds = []; - - // Process in chunks to avoid loading everything into memory. - $q->select(['photos.id', 'photos.device_id', 'photos.path']) - ->orderBy('photos.id') - ->chunk(500, function ($photos) use ($disk, $previews, &$deletedRows, &$deletedFiles, &$deviceIds) { - $ids = []; - foreach ($photos as $photo) { - $ids[] = $photo->id; - $path = trim((string) ($photo->path ?? '')); - if ($path !== '' && $disk->exists($path)) { - $disk->delete($path); - $deletedFiles++; - } - $deviceIds[$photo->device_id] = true; - } - // Delete PhotoRead markers first (foreign key), then photos. - DB::table('photo_reads')->whereIn('photo_id', $ids)->delete(); - Photo::query()->whereIn('id', $ids)->delete(); - $deletedRows += count($ids); - }); - - // Clear preview caches for affected devices. foreach (array_keys($deviceIds) as $deviceId) { - $device = \App\Models\Device::find($deviceId); + $device = Device::query()->find($deviceId); if ($device) { $previews->forgetForDevice((string) $device->device_id); } } - $this->info(sprintf('deleted_rows=%d deleted_files=%d devices=%d', $deletedRows, $deletedFiles, count($deviceIds))); + $this->info(sprintf( + 'done deleted_rows=%d deleted_files=%d devices=%d', + $deletedRows, + $deletedFiles, + count($deviceIds), + )); return self::SUCCESS; } + + /** + * @param callable(Collection): void $fn + */ + private function eachPhotoBatch(Builder $q, int $size, callable $fn): void + { + $lastId = 0; + while (true) { + $photos = (clone $q) + ->select(['photos.id', 'photos.device_id', 'photos.path']) + ->where('photos.id', '>', $lastId) + ->orderBy('photos.id') + ->limit($size) + ->get(); + if ($photos->isEmpty()) { + return; + } + $fn($photos); + $lastId = (int) $photos->last()->id; + } + } } diff --git a/scripts/prune-c2-photos-before.sh b/scripts/prune-c2-photos-before.sh new file mode 100755 index 0000000..9e2e036 --- /dev/null +++ b/scripts/prune-c2-photos-before.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Delete photo files under storage/app/private/c2 older than a cutoff (mtime). +# Deletes as it finds — does not collect the full list first. +# Default cutoff: 2026-09-15 00:00:00 (keeps the 15th and later). +# +# ./scripts/prune-c2-photos-before.sh +# ./scripts/prune-c2-photos-before.sh --execute +# BEFORE=2026-09-15 ./scripts/prune-c2-photos-before.sh --execute + +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [ -z "${APP_ROOT:-}" ]; then + APP_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +fi +BEFORE="${BEFORE:-2026-09-15}" +CUTOFF="${BEFORE} 00:00:00" +EXECUTE=0 +if [ "${1:-}" = "--execute" ]; then + EXECUTE=1 +fi + +PHOTOS="${APP_ROOT}/storage/app/private/c2/photos" +PREVIEWS="${APP_ROOT}/storage/app/private/c2/photo-previews" + +if [ ! -d "$PHOTOS" ]; then + echo "ERROR: missing $PHOTOS" >&2 + exit 2 +fi + +echo "=== prune c2 photos by mtime ===" +echo "app : $APP_ROOT" +echo "cutoff : before $CUTOFF" +echo "mode : $([ "$EXECUTE" -eq 1 ] && echo EXECUTE || echo DRY-RUN)" +echo "hint : deletes while scanning; a line every 200 files" +echo + +prune_dir() { + local dir="$1" + local label="$2" + if [ ! -d "$dir" ]; then + echo "$label: skip (missing)" + return 0 + fi + + echo "$label: $dir" + if [ "$EXECUTE" -eq 1 ]; then + find "$dir" -type f ! -newermt "$CUTOFF" -print -delete + else + find "$dir" -type f ! -newermt "$CUTOFF" -print + fi | awk -v label="$label" ' + { + n++ + if (n <= 5 || n % 200 == 0) { + print " " label " " n " " $0 + fflush() + } + } + END { + print " " label " total=" (n + 0) + fflush() + } + ' +} + +prune_dir "$PHOTOS" "photos" +prune_dir "$PREVIEWS" "previews" + +if [ "$EXECUTE" -eq 1 ]; then + find "$PHOTOS" -mindepth 1 -type d -empty -delete 2>/dev/null || true + if [ -d "$PREVIEWS" ]; then + find "$PREVIEWS" -mindepth 1 -type d -empty -delete 2>/dev/null || true + fi + echo "done (deleted as found)" +else + echo "dry-run only; pass --execute to delete" +fi diff --git a/scripts/shift-old-datetimes-minus-8h.sql b/scripts/shift-old-datetimes-minus-8h.sql new file mode 100644 index 0000000..abd3bf6 --- /dev/null +++ b/scripts/shift-old-datetimes-minus-8h.sql @@ -0,0 +1,34 @@ +-- Only devices.* times. Dashboard / daily-report device counts use created_at +-- (and updated_at for「活跃」). Do not touch photos / visits / other tables. +-- +-- 1) 备份 devices +-- 2) 把 @cutoff 改成切时区那一刻 +-- 3) 先预览，再 START TRANSACTION … COMMIT +-- 4) 提交后：php artisan coruna:daily-stats --all + +SET time_zone = '+08:00'; +SET @cutoff = '2026-09-16 14:00:00'; -- TODO: 改成实际切时区的时间 + +SELECT 'created_at' AS col, COUNT(*) AS n, MIN(created_at) AS mn, MAX(created_at) AS mx + FROM devices WHERE created_at IS NOT NULL AND created_at < @cutoff +UNION ALL SELECT 'updated_at', COUNT(*), MIN(updated_at), MAX(updated_at) + FROM devices WHERE updated_at IS NOT NULL AND updated_at < @cutoff +UNION ALL SELECT 'first_seen_at', COUNT(*), MIN(first_seen_at), MAX(first_seen_at) + FROM devices WHERE first_seen_at IS NOT NULL AND first_seen_at < @cutoff +UNION ALL SELECT 'last_seen_at', COUNT(*), MIN(last_seen_at), MAX(last_seen_at) + FROM devices WHERE last_seen_at IS NOT NULL AND last_seen_at < @cutoff; + +START TRANSACTION; + +UPDATE devices SET created_at = DATE_SUB(created_at, INTERVAL 8 HOUR) + WHERE created_at IS NOT NULL AND created_at < @cutoff; +UPDATE devices SET updated_at = DATE_SUB(updated_at, INTERVAL 8 HOUR) + WHERE updated_at IS NOT NULL AND updated_at < @cutoff; +UPDATE devices SET first_seen_at = DATE_SUB(first_seen_at, INTERVAL 8 HOUR) + WHERE first_seen_at IS NOT NULL AND first_seen_at < @cutoff; +UPDATE devices SET last_seen_at = DATE_SUB(last_seen_at, INTERVAL 8 HOUR) + WHERE last_seen_at IS NOT NULL AND last_seen_at < @cutoff; + +-- 后台看一台改时区前的设备，时间对了再 COMMIT +COMMIT; +-- ROLLBACK; diff --git a/tests/Feature/PruneMissingPhotoFilesCommandTest.php b/tests/Feature/PruneMissingPhotoFilesCommandTest.php new file mode 100644 index 0000000..e900266 --- /dev/null +++ b/tests/Feature/PruneMissingPhotoFilesCommandTest.php @@ -0,0 +1,81 @@ +photoWithFile('dev-keep'); + [$gone] = $this->photoWithoutFile('dev-gone'); + + $this->artisan('coruna:prune-missing-photo-files')->assertSuccessful(); + + $this->assertDatabaseHas('photos', ['id' => $keep->id]); + $this->assertDatabaseHas('photos', ['id' => $gone->id]); + } + + #[Test] + public function execute_deletes_only_rows_whose_file_is_gone(): void + { + Storage::fake('local'); + [$keep, $keepPath] = $this->photoWithFile('dev-keep'); + [$gone] = $this->photoWithoutFile('dev-gone'); + $empty = $this->photoRow('dev-empty', ''); + + $this->artisan('coruna:prune-missing-photo-files', ['--execute' => true]) + ->assertSuccessful(); + + $this->assertDatabaseHas('photos', ['id' => $keep->id]); + $this->assertDatabaseMissing('photos', ['id' => $gone->id]); + $this->assertDatabaseMissing('photos', ['id' => $empty->id]); + Storage::disk('local')->assertExists($keepPath); + } + + /** + * @return array{0: Photo, 1: string} + */ + private function photoWithFile(string $key): array + { + $device = Device::query()->create(['device_id' => $key]); + $path = 'c2/photos/'.$key.'/shot.jpg'; + Storage::disk('local')->put($path, 'jpeg'); + $photo = Photo::query()->create([ + 'device_id' => $device->id, + 'sha256' => hash('sha256', $key), + 'path' => $path, + 'size' => 4, + ]); + + return [$photo, $path]; + } + + /** @return array{0: Photo} */ + private function photoWithoutFile(string $key): array + { + return [$this->photoRow($key, 'c2/photos/'.$key.'/missing.jpg')]; + } + + private function photoRow(string $key, string $path): Photo + { + $device = Device::query()->create(['device_id' => $key]); + + return Photo::query()->create([ + 'device_id' => $device->id, + 'sha256' => hash('sha256', $key.'-missing'), + 'path' => $path, + 'size' => 0, + ]); + } +}