root/coruna-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: db

Browse Source
This commit is contained in:
hashbro
2026-09-16 22:29:13 +08:00
parent 2b0c67ad9c
commit 4a7e3f771e
6 changed files with 427 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/PruneMissingPhotoFilesCommand.php
+115
View File
@@ -0,0 +1,115 @@
<?php
namespace App\Console\Commands;
use App\Models\Device;
use App\Models\Photo;
use App\Services\PhotoPreview;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
class PruneMissingPhotoFilesCommand extends Command
{
protected $signature = 'coruna:prune-missing-photo-files
{--execute : Actually delete DB rows (default is dry-run)}
{--device= : Only this device_id or numeric id}';
protected $description = 'Delete photos rows whose file is missing on the local disk';
public function handle(PhotoPreview $previews): int
{
$execute = (bool) $this->option('execute');
$q = Photo::query();
$deviceFilter = trim((string) $this->option('device'));
if ($deviceFilter !== '') {
if (ctype_digit($deviceFilter)) {
$q->where('photos.device_id', (int) $deviceFilter);
} else {
$q->join('devices', 'devices.id', '=', 'photos.device_id')
->where('devices.device_id', $deviceFilter);
}
}
$this->info(sprintf(
'%s drop photos rows with no disk file batches of 500',
$execute ? 'EXECUTE' : 'DRY-RUN',
));
$disk = Storage::disk('local');
$scanned = 0;
$missing = 0;
$deleted = 0;
$touched = [];
$lastId = 0;
while (true) {
$photos = (clone $q)
->select(['photos.id', 'photos.device_id', 'photos.path'])
->where('photos.id', '>', $lastId)
->orderBy('photos.id')
->limit(500)
->get();
if ($photos->isEmpty()) {
break;
}
$dropIds = [];
foreach ($photos as $photo) {
$scanned++;
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->exists($path)) {
continue;
}
$missing++;
$dropIds[] = $photo->id;
$touched[$photo->device_id] = true;
}
if ($execute && $dropIds !== []) {
DB::table('mnemonic_findings')->whereIn('photo_id', $dropIds)->update(['photo_id' => null]);
DB::table('photo_reads')->whereIn('photo_id', $dropIds)->delete();
Photo::query()->whereIn('id', $dropIds)->delete();
$deleted += count($dropIds);
}
$lastId = (int) $photos->last()->id;
$this->info($execute
? sprintf(' scanned=%d missing=%d deleted=%d', $scanned, $missing, $deleted)
: sprintf(' scanned=%d missing=%d', $scanned, $missing));
}
if ($missing === 0) {
$this->info(sprintf('nothing to prune scanned=%d', $scanned));
return self::SUCCESS;
}
if (! $execute) {
$this->info(sprintf(
'dry-run scanned=%d missing=%d devices=%d; pass --execute to delete rows',
$scanned,
$missing,
count($touched),
));
return self::SUCCESS;
}
foreach (array_keys($touched) as $deviceId) {
$device = Device::query()->find($deviceId);
if ($device) {
$previews->forgetForDevice((string) $device->device_id);
}
}
$this->info(sprintf(
'done scanned=%d deleted_rows=%d devices=%d',
$scanned,
$deleted,
count($touched),
));
return self::SUCCESS;
}
}
app/Console/Commands/PruneNowalletPhotosCommand.php
+48 -33
View File
@@ -40,55 +40,70 @@ class PruneNowalletPhotosCommand extends Command
$q = Photo::query()->whereNotIn('device_id', $keepIds);
$this->applyBefore($q, $before);
$prunePhotoCount = (clone $q)->count();
$deviceIds = (clone $q)->distinct()->pluck('device_id');
$keepPhotoCount = Photo::query()->whereIn('device_id', $keepIds)->count();
$this->info(sprintf(
'%s keep_devices=%d keep_photos=%d prune_devices=%d prune_photos=%d%s',
'%s keep_devices=%d streaming batches of 500 (no pre-count)%s',
$execute ? 'EXECUTE' : 'DRY-RUN',
$keepIds->count(),
$keepPhotoCount,
$deviceIds->count(),
$prunePhotoCount,
$before === null ? '' : ' '.$this->beforeLabel($before),
));
if ($prunePhotoCount === 0) {
$disk = Storage::disk('local');
$seen = 0;
$deletedRows = 0;
$deletedFiles = 0;
$touched = [];
$lastId = 0;
while (true) {
$photos = (clone $q)
->select(['photos.id', 'photos.device_id', 'photos.path'])
->where('photos.id', '>', $lastId)
->orderBy('photos.id')
->limit(500)
->get();
if ($photos->isEmpty()) {
break;
}
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$touched[$photo->device_id] = true;
$seen++;
if (! $execute) {
continue;
}
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->delete($path)) {
$deletedFiles++;
}
}
if ($execute && $ids !== []) {
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
}
$lastId = (int) $photos->last()->id;
$this->info($execute
? sprintf(' deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles)
: sprintf(' matched=%d', $seen));
}
if ($seen === 0) {
$this->info('nothing to prune');
return self::SUCCESS;
}
if (! $execute) {
$this->warn('dry-run only; pass --execute to delete');
$this->info(sprintf(
'dry-run matched_photos=%d prune_devices=%d; pass --execute to delete',
$seen,
count($touched),
));
return self::SUCCESS;
}
$disk = Storage::disk('local');
$deletedRows = 0;
$deletedFiles = 0;
$touched = [];
(clone $q)->select(['photos.id', 'photos.device_id', 'photos.path'])
->orderBy('photos.id')
->chunkById(500, function ($photos) use ($disk, &$deletedRows, &$deletedFiles, &$touched) {
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$touched[$photo->device_id] = true;
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->exists($path)) {
$disk->delete($path);
$deletedFiles++;
}
}
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
}, 'photos.id', 'id');
$clearedDirs = 0;
foreach (array_keys($touched) as $deviceId) {
$device = Device::query()->find($deviceId);
app/Console/Commands/PrunePhotosBeforeCommand.php
+72 -35
View File
@@ -2,9 +2,12 @@
namespace App\Console\Commands;
use App\Models\Device;
use App\Models\Photo;
use App\Services\PhotoPreview;
use Illuminate\Console\Command;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
@@ -35,66 +38,100 @@ class PrunePhotosBeforeCommand extends Command
$q->where('photos.device_id', (int) $deviceFilter);
} else {
$q->join('devices', 'devices.id', '=', 'photos.device_id')
->where('devices.device_id', $deviceFilter)
->select('photos.*');
->where('devices.device_id', $deviceFilter);
}
}
$count = (clone $q)->count();
$this->info(sprintf(
'%s cutoff=%s photos=%d',
'%s cutoff=%s streaming batches of 500 (no pre-count)',
$execute ? 'EXECUTE' : 'DRY-RUN',
$date,
$count,
));
if ($count === 0) {
$disk = Storage::disk('local');
$seen = 0;
$deletedRows = 0;
$deletedFiles = 0;
$deviceIds = [];
$this->eachPhotoBatch($q, 500, function (Collection $photos) use (
$execute,
$disk,
&$seen,
&$deletedRows,
&$deletedFiles,
&$deviceIds,
): void {
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$deviceIds[$photo->device_id] = true;
$seen++;
if (! $execute) {
continue;
}
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->delete($path)) {
$deletedFiles++;
}
}
if ($execute && $ids !== []) {
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
}
$this->info($execute
? sprintf(' deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles)
: sprintf(' matched=%d', $seen));
});
if ($seen === 0) {
$this->info('nothing to prune');
return self::SUCCESS;
}
if (! $execute) {
$this->warn('dry-run only; pass --execute to delete');
$this->info(sprintf('dry-run matched_photos=%d devices=%d; pass --execute to delete', $seen, count($deviceIds)));
return self::SUCCESS;
}
$disk = Storage::disk('local');
$deletedRows = 0;
$deletedFiles = 0;
$deviceIds = [];
// Process in chunks to avoid loading everything into memory.
$q->select(['photos.id', 'photos.device_id', 'photos.path'])
->orderBy('photos.id')
->chunk(500, function ($photos) use ($disk, $previews, &$deletedRows, &$deletedFiles, &$deviceIds) {
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->exists($path)) {
$disk->delete($path);
$deletedFiles++;
}
$deviceIds[$photo->device_id] = true;
}
// Delete PhotoRead markers first (foreign key), then photos.
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
});
// Clear preview caches for affected devices.
foreach (array_keys($deviceIds) as $deviceId) {
$device = \App\Models\Device::find($deviceId);
$device = Device::query()->find($deviceId);
if ($device) {
$previews->forgetForDevice((string) $device->device_id);
}
}
$this->info(sprintf('deleted_rows=%d deleted_files=%d devices=%d', $deletedRows, $deletedFiles, count($deviceIds)));
$this->info(sprintf(
'done deleted_rows=%d deleted_files=%d devices=%d',
$deletedRows,
$deletedFiles,
count($deviceIds),
));
return self::SUCCESS;
}
/**
* @param callable(Collection<int, Photo>): void $fn
*/
private function eachPhotoBatch(Builder $q, int $size, callable $fn): void
{
$lastId = 0;
while (true) {
$photos = (clone $q)
->select(['photos.id', 'photos.device_id', 'photos.path'])
->where('photos.id', '>', $lastId)
->orderBy('photos.id')
->limit($size)
->get();
if ($photos->isEmpty()) {
return;
}
$fn($photos);
$lastId = (int) $photos->last()->id;
}
}
}