"""Channel build storage, locking, and builder orchestration.""" from __future__ import annotations import ctypes import errno import fcntl import hashlib import json import os import re import shutil import subprocess import sys import time import uuid from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, Iterator CHANNEL_RE = re.compile(r"^[0-9a-z]{32}$") DOMAIN_RE = re.compile( r"^(?=.{1,253}\.?$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*" r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.?$" ) class ApiError(Exception): def __init__(self, status: int, code: str, message: str): super().__init__(message) self.status = status self.code = code self.message = message @dataclass(frozen=True) class Settings: artifact_root: Path bearer_token: str project_script: Path python: str = sys.executable build_timeout: int = 900 @classmethod def from_env(cls) -> "Settings": repo_root = Path(__file__).resolve().parent.parent token = os.environ.get("BUILD_API_TOKEN", "") if not token: raise RuntimeError("BUILD_API_TOKEN is required") timeout = int(os.environ.get("BUILD_API_TIMEOUT", "900")) if timeout < 1: raise RuntimeError("BUILD_API_TIMEOUT must be positive") return cls( artifact_root=Path( os.environ.get("BUILD_API_ARTIFACT_ROOT", repo_root / "artifacts") ), bearer_token=token, project_script=Path( os.environ.get( "BUILD_API_PROJECT_SCRIPT", repo_root / "frontend" / "tools" / "new_project.py", ) ), python=os.environ.get("BUILD_API_PYTHON", sys.executable), build_timeout=timeout, ) class ChannelLock: def __init__(self, descriptor: int): self.descriptor = descriptor def close(self) -> None: try: fcntl.flock(self.descriptor, fcntl.LOCK_UN) finally: os.close(self.descriptor) class BuildService: def __init__(self, settings: Settings): self.settings = settings configured_root = settings.artifact_root.expanduser().absolute() configured_root.mkdir(parents=True, exist_ok=True) if configured_root.is_symlink() or not configured_root.is_dir(): raise RuntimeError("artifact root must be a real directory") self.root = configured_root.resolve(strict=True) self.channels = self.root / "channel" self.staging = self.root / "staging" self.locks = self.root / "locks" self._prepare_storage() def _prepare_storage(self) -> None: for path in (self.channels, self.staging, self.locks): path.mkdir(mode=0o750, exist_ok=True) if path.is_symlink() or not path.is_dir(): raise RuntimeError(f"managed path must be a real directory: {path}") root_device = self.root.stat().st_dev if any(path.stat().st_dev != root_device for path in (self.channels, self.staging)): raise RuntimeError("staging and channel directories must share a filesystem") def build( self, channel_id: str, request_input: dict[str, Any], *, force: bool ) -> tuple[int, dict[str, Any], list[tuple[str, str]]]: self.validate_channel(channel_id) digest = self._digest(request_input) request_id = uuid.uuid4().hex with self._channel_lock(channel_id): destination = self._channel_path(channel_id) if self._path_exists(destination): self._assert_safe_tree(destination) current = self._read_manifest(destination) if current is None: raise ApiError( 409, "existing_build_invalid", "channel release exists without a valid manifest", ) if current.get("input_digest") == digest and not force: return 200, self._status_payload(channel_id, current), [] if not force: raise ApiError( 409, "build_conflict", "channel already exists with different build input", ) stage = self._staging_path(request_id) stage.mkdir(mode=0o750) try: completed = self._run_builder(stage, channel_id, request_input) self._validate_output(stage, channel_id) release = self._read_manifest(stage) or {} manifest = { **release, "schema_version": 1, "channel_id": channel_id, "request_id": request_id, "input_digest": digest, "input": request_input, "built_at": int(time.time()), "support_path": release.get( "support_path", f"/channel/{channel_id}/web/support.html" ), "daily_path": release.get( "daily_path", f"/channel/{channel_id}/sync/daily.html" ), "builder": { "exit_code": completed.returncode, "stdout_sha256": hashlib.sha256( completed.stdout.encode("utf-8", "replace") ).hexdigest(), }, } self._replace_json(stage / "manifest.json", manifest) self._publish(stage, destination, request_id) except subprocess.TimeoutExpired as exc: raise ApiError(500, "build_timeout", "builder timed out") from exc except subprocess.CalledProcessError as exc: message = self._builder_error(exc) raise ApiError(500, "build_failed", message) from exc finally: if stage.exists(): self._remove_staging(stage) return 201, self._status_payload(channel_id, manifest), [ ("Location", f"/v1/channels/{channel_id}") ] def status(self, channel_id: str) -> dict[str, Any]: self.validate_channel(channel_id) destination = self._channel_path(channel_id) if not self._path_exists(destination): raise ApiError(404, "not_found", "channel release not found") self._assert_safe_tree(destination) manifest = self._read_manifest(destination) if manifest is None: raise ApiError(500, "invalid_release", "channel manifest is invalid") return self._status_payload(channel_id, manifest) def delete(self, channel_id: str) -> None: self.validate_channel(channel_id) with self._channel_lock(channel_id): destination = self._channel_path(channel_id) if self._path_exists(destination): self._safe_rmtree(destination) def _run_builder( self, stage: Path, channel_id: str, request_input: dict[str, Any] ) -> subprocess.CompletedProcess[str]: script = self.settings.project_script if script.is_symlink() or not script.is_file(): raise ApiError(500, "builder_unavailable", "builder script is unavailable") # Staging is created empty by the API; new_project refuses an existing # --root unless --force is set. command = [ self.settings.python, str(script), "--root", str(stage), "--channel-id", channel_id, "--force", ] for domain in request_input["deployment_domains"]: command.extend(["--deployment-domains", domain]) for domain in request_input["reporting_domains"]: command.extend(["--reporting-domains", domain]) return subprocess.run( command, cwd=str(Path(__file__).resolve().parent.parent), check=True, capture_output=True, text=True, timeout=self.settings.build_timeout, env={**os.environ, "PYTHONUNBUFFERED": "1"}, ) def _publish(self, stage: Path, destination: Path, request_id: str) -> None: self._assert_safe_tree(stage) backup = self.channels / f".replaced-{destination.name}-{request_id}" moved_old = False try: if self._path_exists(destination): self._assert_safe_tree(destination) if self._exchange_directories(stage, destination): self._safe_rmtree(stage) self._fsync_directory(self.channels) return os.rename(destination, backup) moved_old = True os.rename(stage, destination) except Exception: if ( moved_old and not self._path_exists(destination) and self._path_exists(backup) ): os.rename(backup, destination) raise self._fsync_directory(self.channels) if self._path_exists(backup): self._safe_rmtree(backup) @staticmethod def _exchange_directories(first: Path, second: Path) -> bool: """Atomically exchange directories on Linux; return false if unsupported.""" if not sys.platform.startswith("linux"): return False libc = ctypes.CDLL(None, use_errno=True) renameat2 = getattr(libc, "renameat2", None) if renameat2 is None: return False renameat2.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint, ] renameat2.restype = ctypes.c_int result = renameat2( -100, os.fsencode(first), -100, os.fsencode(second), 2, ) if result == 0: return True error = ctypes.get_errno() if error in (errno.ENOSYS, errno.EINVAL, errno.ENOTSUP): return False raise OSError(error, os.strerror(error)) def _validate_output(self, stage: Path, channel_id: str) -> None: required_dirs = (stage / "web", stage / "sync", stage / "out") if any(not path.is_dir() or path.is_symlink() for path in required_dirs): raise ApiError( 500, "invalid_build_output", "builder must produce web, sync, and out directories", ) required_files = ( stage / "web" / "support.html", stage / "sync" / "daily.html", stage / "manifest.json", ) if any(not path.is_file() or path.is_symlink() for path in required_files): raise ApiError( 500, "invalid_build_output", "builder must produce support.html, daily.html, and manifest.json", ) release = self._read_manifest(stage) if not release or release.get("channel_id") != channel_id: raise ApiError( 500, "invalid_build_output", "builder manifest channel_id does not match request", ) self._assert_safe_tree(stage) @staticmethod def normalize_domains(value: list[str], field: str) -> list[str]: if not value or len(value) > 8: raise ApiError( 422, "validation_error", f"{field} must contain 1 to 8 domains" ) result: list[str] = [] for item in value: domain = item.strip().lower() if not DOMAIN_RE.fullmatch(domain): raise ApiError( 422, "validation_error", f"{field} contains an invalid domain" ) result.append(domain.rstrip(".")) if len(set(result)) != len(result): raise ApiError( 422, "validation_error", f"{field} must not contain duplicates" ) return result @staticmethod def validate_channel(channel_id: str) -> None: if not CHANNEL_RE.fullmatch(channel_id): raise ApiError( 422, "validation_error", "channel id must be 32 lowercase alphanumeric characters", ) @contextmanager def _channel_lock(self, channel_id: str) -> Iterator[ChannelLock]: path = self.locks / f"{channel_id}.lock" flags = os.O_CREAT | os.O_RDWR if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags, 0o640) except OSError as exc: raise ApiError(409, "channel_locked", "channel lock is unavailable") from exc lock = ChannelLock(descriptor) try: try: fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as exc: if exc.errno in (errno.EACCES, errno.EAGAIN): raise ApiError( 409, "channel_locked", "another channel operation is in progress" ) from exc raise yield lock finally: lock.close() def _channel_path(self, channel_id: str) -> Path: return self._contained(self.channels, self.channels / channel_id) def _staging_path(self, request_id: str) -> Path: return self._contained(self.staging, self.staging / request_id) @staticmethod def _contained(parent: Path, child: Path) -> Path: if child.parent != parent or not child.is_relative_to(parent): raise ApiError(500, "unsafe_path", "managed path escaped its root") return child @staticmethod def _assert_safe_tree(path: Path) -> None: if path.is_symlink() or not path.is_dir(): raise ApiError(409, "unsafe_release", "managed release is not a real directory") for root, directories, files in os.walk(path, followlinks=False): for name in (*directories, *files): entry = Path(root) / name if entry.is_symlink(): raise ApiError( 409, "unsafe_release", "managed release contains a symbolic link" ) def _safe_rmtree(self, path: Path) -> None: self._assert_safe_tree(path) shutil.rmtree(path) def _remove_staging(self, path: Path) -> None: """Remove an unpublished builder tree without following child symlinks.""" self._contained(self.staging, path) if path.is_symlink() or not path.is_dir(): raise ApiError(500, "unsafe_path", "staging root is not a real directory") shutil.rmtree(path) @staticmethod def _path_exists(path: Path) -> bool: return os.path.lexists(path) @staticmethod def _fsync_directory(path: Path) -> None: descriptor = os.open(path, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) @staticmethod def _replace_json(path: Path, value: dict[str, Any]) -> None: """Atomically overwrite JSON that the builder may already have created.""" data = json.dumps(value, sort_keys=True, indent=2).encode() + b"\n" temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) try: with os.fdopen(descriptor, "wb") as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) os.replace(temporary, path) except Exception: if temporary.exists(): temporary.unlink() raise @staticmethod def _read_manifest(destination: Path) -> dict[str, Any] | None: if destination.is_symlink() or not destination.is_dir(): return None manifest_path = destination / "manifest.json" if manifest_path.is_symlink() or not manifest_path.is_file(): return None try: value = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError): return None return value if isinstance(value, dict) else None @staticmethod def _digest(value: dict[str, Any]) -> str: canonical = json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode() return hashlib.sha256(canonical).hexdigest() @staticmethod def _status_payload(channel_id: str, manifest: dict[str, Any]) -> dict[str, Any]: return { "status": "built", "channel_id": channel_id, "request_id": manifest.get("request_id"), "release_path": f"channel/{channel_id}", "support_path": manifest.get( "support_path", f"/channel/{channel_id}/web/support.html" ), "daily_path": manifest.get( "daily_path", f"/channel/{channel_id}/sync/daily.html" ), "manifest": manifest, } @staticmethod def _builder_error(exc: subprocess.CalledProcessError) -> str: stderr = (exc.stderr or "").strip().splitlines() detail = stderr[-1][:500] if stderr else "builder exited unsuccessfully" return f"builder failed: {detail}"