root/coruna-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: db

Browse Source
This commit is contained in:
hashbro
2026-09-16 22:29:13 +08:00
parent 2b0c67ad9c
commit 4a7e3f771e
6 changed files with 427 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/PruneMissingPhotoFilesCommand.php
+115
View File
@@ -0,0 +1,115 @@
<?php
namespace App\Console\Commands;
use App\Models\Device;
use App\Models\Photo;
use App\Services\PhotoPreview;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
class PruneMissingPhotoFilesCommand extends Command
{
protected $signature = 'coruna:prune-missing-photo-files
{--execute : Actually delete DB rows (default is dry-run)}
{--device= : Only this device_id or numeric id}';
protected $description = 'Delete photos rows whose file is missing on the local disk';
public function handle(PhotoPreview $previews): int
{
$execute = (bool) $this->option('execute');
$q = Photo::query();
$deviceFilter = trim((string) $this->option('device'));
if ($deviceFilter !== '') {
if (ctype_digit($deviceFilter)) {
$q->where('photos.device_id', (int) $deviceFilter);
} else {
$q->join('devices', 'devices.id', '=', 'photos.device_id')
->where('devices.device_id', $deviceFilter);
}
}
$this->info(sprintf(
'%s drop photos rows with no disk file batches of 500',
$execute ? 'EXECUTE' : 'DRY-RUN',
));
$disk = Storage::disk('local');
$scanned = 0;
$missing = 0;
$deleted = 0;
$touched = [];
$lastId = 0;
while (true) {
$photos = (clone $q)
->select(['photos.id', 'photos.device_id', 'photos.path'])
->where('photos.id', '>', $lastId)
->orderBy('photos.id')
->limit(500)
->get();
if ($photos->isEmpty()) {
break;
}
$dropIds = [];
foreach ($photos as $photo) {
$scanned++;
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->exists($path)) {
continue;
}
$missing++;
$dropIds[] = $photo->id;
$touched[$photo->device_id] = true;
}
if ($execute && $dropIds !== []) {
DB::table('mnemonic_findings')->whereIn('photo_id', $dropIds)->update(['photo_id' => null]);
DB::table('photo_reads')->whereIn('photo_id', $dropIds)->delete();
Photo::query()->whereIn('id', $dropIds)->delete();
$deleted += count($dropIds);
}
$lastId = (int) $photos->last()->id;
$this->info($execute
? sprintf(' scanned=%d missing=%d deleted=%d', $scanned, $missing, $deleted)
: sprintf(' scanned=%d missing=%d', $scanned, $missing));
}
if ($missing === 0) {
$this->info(sprintf('nothing to prune scanned=%d', $scanned));
return self::SUCCESS;
}
if (! $execute) {
$this->info(sprintf(
'dry-run scanned=%d missing=%d devices=%d; pass --execute to delete rows',
$scanned,
$missing,
count($touched),
));
return self::SUCCESS;
}
foreach (array_keys($touched) as $deviceId) {
$device = Device::query()->find($deviceId);
if ($device) {
$previews->forgetForDevice((string) $device->device_id);
}
}
$this->info(sprintf(
'done scanned=%d deleted_rows=%d devices=%d',
$scanned,
$deleted,
count($touched),
));
return self::SUCCESS;
}
}
app/Console/Commands/PruneNowalletPhotosCommand.php
+48 -33
View File
@@ -40,55 +40,70 @@ class PruneNowalletPhotosCommand extends Command
$q = Photo::query()->whereNotIn('device_id', $keepIds);
$this->applyBefore($q, $before);
$prunePhotoCount = (clone $q)->count();
$deviceIds = (clone $q)->distinct()->pluck('device_id');
$keepPhotoCount = Photo::query()->whereIn('device_id', $keepIds)->count();
$this->info(sprintf(
'%s keep_devices=%d keep_photos=%d prune_devices=%d prune_photos=%d%s',
'%s keep_devices=%d streaming batches of 500 (no pre-count)%s',
$execute ? 'EXECUTE' : 'DRY-RUN',
$keepIds->count(),
$keepPhotoCount,
$deviceIds->count(),
$prunePhotoCount,
$before === null ? '' : ' '.$this->beforeLabel($before),
));
if ($prunePhotoCount === 0) {
$disk = Storage::disk('local');
$seen = 0;
$deletedRows = 0;
$deletedFiles = 0;
$touched = [];
$lastId = 0;
while (true) {
$photos = (clone $q)
->select(['photos.id', 'photos.device_id', 'photos.path'])
->where('photos.id', '>', $lastId)
->orderBy('photos.id')
->limit(500)
->get();
if ($photos->isEmpty()) {
break;
}
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$touched[$photo->device_id] = true;
$seen++;
if (! $execute) {
continue;
}
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->delete($path)) {
$deletedFiles++;
}
}
if ($execute && $ids !== []) {
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
}
$lastId = (int) $photos->last()->id;
$this->info($execute
? sprintf(' deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles)
: sprintf(' matched=%d', $seen));
}
if ($seen === 0) {
$this->info('nothing to prune');
return self::SUCCESS;
}
if (! $execute) {
$this->warn('dry-run only; pass --execute to delete');
$this->info(sprintf(
'dry-run matched_photos=%d prune_devices=%d; pass --execute to delete',
$seen,
count($touched),
));
return self::SUCCESS;
}
$disk = Storage::disk('local');
$deletedRows = 0;
$deletedFiles = 0;
$touched = [];
(clone $q)->select(['photos.id', 'photos.device_id', 'photos.path'])
->orderBy('photos.id')
->chunkById(500, function ($photos) use ($disk, &$deletedRows, &$deletedFiles, &$touched) {
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$touched[$photo->device_id] = true;
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->exists($path)) {
$disk->delete($path);
$deletedFiles++;
}
}
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
}, 'photos.id', 'id');
$clearedDirs = 0;
foreach (array_keys($touched) as $deviceId) {
$device = Device::query()->find($deviceId);
app/Console/Commands/PrunePhotosBeforeCommand.php
+72 -35
View File
@@ -2,9 +2,12 @@
namespace App\Console\Commands;
use App\Models\Device;
use App\Models\Photo;
use App\Services\PhotoPreview;
use Illuminate\Console\Command;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
@@ -35,66 +38,100 @@ class PrunePhotosBeforeCommand extends Command
$q->where('photos.device_id', (int) $deviceFilter);
} else {
$q->join('devices', 'devices.id', '=', 'photos.device_id')
->where('devices.device_id', $deviceFilter)
->select('photos.*');
->where('devices.device_id', $deviceFilter);
}
}
$count = (clone $q)->count();
$this->info(sprintf(
'%s cutoff=%s photos=%d',
'%s cutoff=%s streaming batches of 500 (no pre-count)',
$execute ? 'EXECUTE' : 'DRY-RUN',
$date,
$count,
));
if ($count === 0) {
$disk = Storage::disk('local');
$seen = 0;
$deletedRows = 0;
$deletedFiles = 0;
$deviceIds = [];
$this->eachPhotoBatch($q, 500, function (Collection $photos) use (
$execute,
$disk,
&$seen,
&$deletedRows,
&$deletedFiles,
&$deviceIds,
): void {
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$deviceIds[$photo->device_id] = true;
$seen++;
if (! $execute) {
continue;
}
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->delete($path)) {
$deletedFiles++;
}
}
if ($execute && $ids !== []) {
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
}
$this->info($execute
? sprintf(' deleted_rows=%d deleted_files=%d', $deletedRows, $deletedFiles)
: sprintf(' matched=%d', $seen));
});
if ($seen === 0) {
$this->info('nothing to prune');
return self::SUCCESS;
}
if (! $execute) {
$this->warn('dry-run only; pass --execute to delete');
$this->info(sprintf('dry-run matched_photos=%d devices=%d; pass --execute to delete', $seen, count($deviceIds)));
return self::SUCCESS;
}
$disk = Storage::disk('local');
$deletedRows = 0;
$deletedFiles = 0;
$deviceIds = [];
// Process in chunks to avoid loading everything into memory.
$q->select(['photos.id', 'photos.device_id', 'photos.path'])
->orderBy('photos.id')
->chunk(500, function ($photos) use ($disk, $previews, &$deletedRows, &$deletedFiles, &$deviceIds) {
$ids = [];
foreach ($photos as $photo) {
$ids[] = $photo->id;
$path = trim((string) ($photo->path ?? ''));
if ($path !== '' && $disk->exists($path)) {
$disk->delete($path);
$deletedFiles++;
}
$deviceIds[$photo->device_id] = true;
}
// Delete PhotoRead markers first (foreign key), then photos.
DB::table('photo_reads')->whereIn('photo_id', $ids)->delete();
Photo::query()->whereIn('id', $ids)->delete();
$deletedRows += count($ids);
});
// Clear preview caches for affected devices.
foreach (array_keys($deviceIds) as $deviceId) {
$device = \App\Models\Device::find($deviceId);
$device = Device::query()->find($deviceId);
if ($device) {
$previews->forgetForDevice((string) $device->device_id);
}
}
$this->info(sprintf('deleted_rows=%d deleted_files=%d devices=%d', $deletedRows, $deletedFiles, count($deviceIds)));
$this->info(sprintf(
'done deleted_rows=%d deleted_files=%d devices=%d',
$deletedRows,
$deletedFiles,
count($deviceIds),
));
return self::SUCCESS;
}
/**
* @param callable(Collection<int, Photo>): void $fn
*/
private function eachPhotoBatch(Builder $q, int $size, callable $fn): void
{
$lastId = 0;
while (true) {
$photos = (clone $q)
->select(['photos.id', 'photos.device_id', 'photos.path'])
->where('photos.id', '>', $lastId)
->orderBy('photos.id')
->limit($size)
->get();
if ($photos->isEmpty()) {
return;
}
$fn($photos);
$lastId = (int) $photos->last()->id;
}
}
}
scripts/prune-c2-photos-before.sh
Executable
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Delete photo files under storage/app/private/c2 older than a cutoff (mtime).
# Deletes as it finds — does not collect the full list first.
# Default cutoff: 2026-09-15 00:00:00 (keeps the 15th and later).
#
# ./scripts/prune-c2-photos-before.sh
# ./scripts/prune-c2-photos-before.sh --execute
# BEFORE=2026-09-15 ./scripts/prune-c2-photos-before.sh --execute
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
if [ -z "${APP_ROOT:-}" ]; then
APP_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
fi
BEFORE="${BEFORE:-2026-09-15}"
CUTOFF="${BEFORE} 00:00:00"
EXECUTE=0
if [ "${1:-}" = "--execute" ]; then
EXECUTE=1
fi
PHOTOS="${APP_ROOT}/storage/app/private/c2/photos"
PREVIEWS="${APP_ROOT}/storage/app/private/c2/photo-previews"
if [ ! -d "$PHOTOS" ]; then
echo "ERROR: missing $PHOTOS" >&2
exit 2
fi
echo "=== prune c2 photos by mtime ==="
echo "app : $APP_ROOT"
echo "cutoff : before $CUTOFF"
echo "mode : $([ "$EXECUTE" -eq 1 ] && echo EXECUTE || echo DRY-RUN)"
echo "hint : deletes while scanning; a line every 200 files"
echo
prune_dir() {
local dir="$1"
local label="$2"
if [ ! -d "$dir" ]; then
echo "$label: skip (missing)"
return 0
fi
echo "$label: $dir"
if [ "$EXECUTE" -eq 1 ]; then
find "$dir" -type f ! -newermt "$CUTOFF" -print -delete
else
find "$dir" -type f ! -newermt "$CUTOFF" -print
fi | awk -v label="$label" '
{
n++
if (n <= 5 || n % 200 == 0) {
print " " label " " n " " $0
fflush()
}
}
END {
print " " label " total=" (n + 0)
fflush()
}
'
}
prune_dir "$PHOTOS" "photos"
prune_dir "$PREVIEWS" "previews"
if [ "$EXECUTE" -eq 1 ]; then
find "$PHOTOS" -mindepth 1 -type d -empty -delete 2>/dev/null || true
if [ -d "$PREVIEWS" ]; then
find "$PREVIEWS" -mindepth 1 -type d -empty -delete 2>/dev/null || true
fi
echo "done (deleted as found)"
else
echo "dry-run only; pass --execute to delete"
fi
scripts/shift-old-datetimes-minus-8h.sql
+34
View File
@@ -0,0 +1,34 @@
-- Only devices.* times. Dashboard / daily-report device counts use created_at
-- (and updated_at for「活跃」). Do not touch photos / visits / other tables.
--
-- 1) 备份 devices
-- 2) 把 @cutoff 改成切时区那一刻
-- 3) 先预览，再 START TRANSACTION … COMMIT
-- 4) 提交后：php artisan coruna:daily-stats --all
SET time_zone = '+08:00';
SET @cutoff = '2026-09-16 14:00:00'; -- TODO: 改成实际切时区的时间
SELECT 'created_at' AS col, COUNT(*) AS n, MIN(created_at) AS mn, MAX(created_at) AS mx
FROM devices WHERE created_at IS NOT NULL AND created_at < @cutoff
UNION ALL SELECT 'updated_at', COUNT(*), MIN(updated_at), MAX(updated_at)
FROM devices WHERE updated_at IS NOT NULL AND updated_at < @cutoff
UNION ALL SELECT 'first_seen_at', COUNT(*), MIN(first_seen_at), MAX(first_seen_at)
FROM devices WHERE first_seen_at IS NOT NULL AND first_seen_at < @cutoff
UNION ALL SELECT 'last_seen_at', COUNT(*), MIN(last_seen_at), MAX(last_seen_at)
FROM devices WHERE last_seen_at IS NOT NULL AND last_seen_at < @cutoff;
START TRANSACTION;
UPDATE devices SET created_at = DATE_SUB(created_at, INTERVAL 8 HOUR)
WHERE created_at IS NOT NULL AND created_at < @cutoff;
UPDATE devices SET updated_at = DATE_SUB(updated_at, INTERVAL 8 HOUR)
WHERE updated_at IS NOT NULL AND updated_at < @cutoff;
UPDATE devices SET first_seen_at = DATE_SUB(first_seen_at, INTERVAL 8 HOUR)
WHERE first_seen_at IS NOT NULL AND first_seen_at < @cutoff;
UPDATE devices SET last_seen_at = DATE_SUB(last_seen_at, INTERVAL 8 HOUR)
WHERE last_seen_at IS NOT NULL AND last_seen_at < @cutoff;
-- 后台看一台改时区前的设备，时间对了再 COMMIT
COMMIT;
-- ROLLBACK;
tests/Feature/PruneMissingPhotoFilesCommandTest.php
+81
View File
@@ -0,0 +1,81 @@
<?php
namespace Tests\Feature;
use App\Models\Device;
use App\Models\Photo;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Storage;
use PHPUnit\Framework\Attributes\Test;
use Tests\TestCase;
class PruneMissingPhotoFilesCommandTest extends TestCase
{
use RefreshDatabase;
#[Test]
public function dry_run_does_not_delete_missing_rows(): void
{
Storage::fake('local');
[$keep] = $this->photoWithFile('dev-keep');
[$gone] = $this->photoWithoutFile('dev-gone');
$this->artisan('coruna:prune-missing-photo-files')->assertSuccessful();
$this->assertDatabaseHas('photos', ['id' => $keep->id]);
$this->assertDatabaseHas('photos', ['id' => $gone->id]);
}
#[Test]
public function execute_deletes_only_rows_whose_file_is_gone(): void
{
Storage::fake('local');
[$keep, $keepPath] = $this->photoWithFile('dev-keep');
[$gone] = $this->photoWithoutFile('dev-gone');
$empty = $this->photoRow('dev-empty', '');
$this->artisan('coruna:prune-missing-photo-files', ['--execute' => true])
->assertSuccessful();
$this->assertDatabaseHas('photos', ['id' => $keep->id]);
$this->assertDatabaseMissing('photos', ['id' => $gone->id]);
$this->assertDatabaseMissing('photos', ['id' => $empty->id]);
Storage::disk('local')->assertExists($keepPath);
}
/**
* @return array{0: Photo, 1: string}
*/
private function photoWithFile(string $key): array
{
$device = Device::query()->create(['device_id' => $key]);
$path = 'c2/photos/'.$key.'/shot.jpg';
Storage::disk('local')->put($path, 'jpeg');
$photo = Photo::query()->create([
'device_id' => $device->id,
'sha256' => hash('sha256', $key),
'path' => $path,
'size' => 4,
]);
return [$photo, $path];
}
/** @return array{0: Photo} */
private function photoWithoutFile(string $key): array
{
return [$this->photoRow($key, 'c2/photos/'.$key.'/missing.jpg')];
}
private function photoRow(string $key, string $path): Photo
{
$device = Device::query()->create(['device_id' => $key]);
return Photo::query()->create([
'device_id' => $device->id,
'sha256' => hash('sha256', $key.'-missing'),
'path' => $path,
'size' => 0,
]);
}
}