init

build_api/__init__.py

@@ -0,0 +1,6 @@
+"""Standalone synchronous channel build API."""
+
+from .app import create_application
+from .service import ApiError, BuildService, Settings
+
+__all__ = ["ApiError", "BuildService", "Settings", "create_application"]

build_api/__main__.py

@@ -0,0 +1,21 @@
+"""Run the build API with Uvicorn."""
+
+from __future__ import annotations
+
+import os
+
+import uvicorn
+
+from .app import create_application
+
+
+def main() -> None:
+    host = os.environ.get("BUILD_API_HOST", "127.0.0.1")
+    port = int(os.environ.get("BUILD_API_PORT", "8081"))
+    application = create_application()
+    print(f"build API listening on http://{host}:{port}", flush=True)
+    uvicorn.run(application, host=host, port=port, log_level="info")
+
+
+if __name__ == "__main__":
+    main()

build_api/app.py

@@ -0,0 +1,218 @@
+"""FastAPI application for atomic channel builds."""
+
+from __future__ import annotations
+
+import hmac
+from typing import Annotated, Any, Dict, List, Optional
+
+from fastapi import Depends, FastAPI, Path, Request, Response
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from .service import ApiError, BuildService, Settings
+
+MAX_BODY_BYTES = 64 * 1024
+CHANNEL_PATH = Path(
+    ...,
+    min_length=32,
+    max_length=32,
+    pattern=r"^[0-9a-z]{32}$",
+    description="32 lowercase alphanumeric channel id",
+)
+
+bearer_scheme = HTTPBearer(auto_error=False)
+
+
+class BuildRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    deployment_domains: List[str] = Field(min_length=1, max_length=8)
+    reporting_domains: List[str] = Field(min_length=1, max_length=8)
+    force: bool = Field(default=False, strict=True)
+
+    @field_validator("deployment_domains", "reporting_domains")
+    @classmethod
+    def domains_must_be_nonempty_strings(cls, value: List[str]) -> List[str]:
+        for item in value:
+            if not isinstance(item, str) or not item.strip():
+                raise ValueError("domain entries must be non-empty strings")
+        return value
+
+
+def _error_response(status: int, code: str, message: str) -> JSONResponse:
+    return JSONResponse(
+        status_code=status,
+        content={"error": {"code": code, "message": message}},
+        headers={"Cache-Control": "no-store"},
+    )
+
+
+def _validation_message(exc: RequestValidationError) -> str:
+    errors = exc.errors()
+    if not errors:
+        return "request validation failed"
+    first = errors[0]
+    loc = [str(part) for part in first.get("loc", ()) if part != "body"]
+    msg = str(first.get("msg", "invalid"))
+    error_type = str(first.get("type", ""))
+
+    if error_type == "json_invalid" or "JSON decode" in msg:
+        return "request body must be valid JSON"
+    if error_type == "model_attributes_type":
+        return "request body must be a JSON object"
+    if error_type == "extra_forbidden" and loc:
+        return f"unknown fields: {loc[-1]}"
+    if "force" in loc and error_type.startswith("bool"):
+        return "force must be a boolean"
+    if any(part == "channel_id" for part in first.get("loc", ())):
+        return "channel id must be 32 lowercase alphanumeric characters"
+    if loc and loc[-1] in {"deployment_domains", "reporting_domains"}:
+        field = loc[-1]
+        if "too_short" in error_type or "too_long" in error_type:
+            return f"{field} must contain 1 to 8 domains"
+        return f"{field} contains an invalid domain"
+    if loc:
+        return f"{'.'.join(loc)}: {msg}"
+    return msg
+
+
+def get_settings(request: Request) -> Settings:
+    return request.app.state.settings
+
+
+def get_service(request: Request) -> BuildService:
+    return request.app.state.service
+
+
+def require_auth(
+    credentials: Annotated[
+        Optional[HTTPAuthorizationCredentials], Depends(bearer_scheme)
+    ],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> None:
+    token = credentials.credentials if credentials is not None else ""
+    if not token or not hmac.compare_digest(token, settings.bearer_token):
+        raise ApiError(401, "unauthorized", "valid Bearer token required")
+
+
+def create_application(
+    settings: Optional[Settings] = None,
+    service: Optional[BuildService] = None,
+) -> FastAPI:
+    resolved_settings = settings or Settings.from_env()
+    resolved_service = service or BuildService(resolved_settings)
+
+    app = FastAPI(
+        title="Coruna Build API",
+        version="1.0.0",
+        docs_url=None,
+        redoc_url=None,
+        openapi_url=None,
+    )
+    app.state.settings = resolved_settings
+    app.state.service = resolved_service
+
+    @app.middleware("http")
+    async def enforce_body_limit(request: Request, call_next: Any) -> Response:
+        content_length = request.headers.get("content-length")
+        if content_length is not None:
+            try:
+                length = int(content_length)
+            except ValueError:
+                return _error_response(
+                    422, "validation_error", "valid Content-Length required"
+                )
+            if length > MAX_BODY_BYTES:
+                return _error_response(
+                    422, "validation_error", "request body size is invalid"
+                )
+        response = await call_next(request)
+        if "cache-control" not in response.headers:
+            response.headers["Cache-Control"] = "no-store"
+        return response
+
+    @app.exception_handler(ApiError)
+    async def api_error_handler(_: Request, exc: ApiError) -> JSONResponse:
+        return _error_response(exc.status, exc.code, exc.message)
+
+    @app.exception_handler(RequestValidationError)
+    async def validation_error_handler(
+        _: Request, exc: RequestValidationError
+    ) -> JSONResponse:
+        return _error_response(422, "validation_error", _validation_message(exc))
+
+    @app.exception_handler(StarletteHTTPException)
+    async def http_exception_handler(
+        _: Request, exc: StarletteHTTPException
+    ) -> JSONResponse:
+        if exc.status_code == 404:
+            return _error_response(404, "not_found", "route not found")
+        if exc.status_code == 405:
+            return _error_response(405, "method_not_allowed", "method not allowed")
+        return _error_response(
+            exc.status_code, "http_error", str(exc.detail) or "request failed"
+        )
+
+    @app.exception_handler(Exception)
+    async def unhandled_error_handler(_: Request, __: Exception) -> JSONResponse:
+        return _error_response(500, "internal_error", "internal server error")
+
+    @app.get("/health")
+    def health() -> Dict[str, str]:
+        return {"status": "ok"}
+
+    @app.post(
+        "/v1/channels/{channel_id}/build",
+        dependencies=[Depends(require_auth)],
+        status_code=201,
+    )
+    def build_channel(
+        channel_id: Annotated[str, CHANNEL_PATH],
+        body: BuildRequest,
+        service: Annotated[BuildService, Depends(get_service)],
+    ) -> JSONResponse:
+        request_input = {
+            "channel_id": channel_id,
+            "deployment_domains": BuildService.normalize_domains(
+                body.deployment_domains, "deployment_domains"
+            ),
+            "reporting_domains": BuildService.normalize_domains(
+                body.reporting_domains, "reporting_domains"
+            ),
+        }
+        status, payload, headers = service.build(
+            channel_id, request_input, force=body.force
+        )
+        return JSONResponse(
+            status_code=status,
+            content=payload,
+            headers={"Cache-Control": "no-store", **dict(headers)},
+        )
+
+    @app.get(
+        "/v1/channels/{channel_id}",
+        dependencies=[Depends(require_auth)],
+    )
+    def channel_status(
+        channel_id: Annotated[str, CHANNEL_PATH],
+        service: Annotated[BuildService, Depends(get_service)],
+    ) -> Dict[str, Any]:
+        return service.status(channel_id)
+
+    @app.delete(
+        "/v1/channels/{channel_id}",
+        dependencies=[Depends(require_auth)],
+        status_code=204,
+        response_class=Response,
+    )
+    def delete_channel(
+        channel_id: Annotated[str, CHANNEL_PATH],
+        service: Annotated[BuildService, Depends(get_service)],
+    ) -> Response:
+        service.delete(channel_id)
+        return Response(status_code=204, headers={"Cache-Control": "no-store"})
+
+    return app

build_api/service.py

@@ -0,0 +1,463 @@
+"""Channel build storage, locking, and builder orchestration."""
+
+from __future__ import annotations
+
+import ctypes
+import errno
+import fcntl
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterator
+
+CHANNEL_RE = re.compile(r"^[0-9a-z]{32}$")
+DOMAIN_RE = re.compile(
+    r"^(?=.{1,253}\.?$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*"
+    r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.?$"
+)
+
+
+class ApiError(Exception):
+    def __init__(self, status: int, code: str, message: str):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.message = message
+
+
+@dataclass(frozen=True)
+class Settings:
+    artifact_root: Path
+    bearer_token: str
+    project_script: Path
+    python: str = sys.executable
+    build_timeout: int = 900
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        repo_root = Path(__file__).resolve().parent.parent
+        token = os.environ.get("BUILD_API_TOKEN", "")
+        if not token:
+            raise RuntimeError("BUILD_API_TOKEN is required")
+        timeout = int(os.environ.get("BUILD_API_TIMEOUT", "900"))
+        if timeout < 1:
+            raise RuntimeError("BUILD_API_TIMEOUT must be positive")
+        return cls(
+            artifact_root=Path(
+                os.environ.get("BUILD_API_ARTIFACT_ROOT", repo_root / "artifacts")
+            ),
+            bearer_token=token,
+            project_script=Path(
+                os.environ.get(
+                    "BUILD_API_PROJECT_SCRIPT",
+                    repo_root / "frontend" / "tools" / "new_project.py",
+                )
+            ),
+            python=os.environ.get("BUILD_API_PYTHON", sys.executable),
+            build_timeout=timeout,
+        )
+
+
+class ChannelLock:
+    def __init__(self, descriptor: int):
+        self.descriptor = descriptor
+
+    def close(self) -> None:
+        try:
+            fcntl.flock(self.descriptor, fcntl.LOCK_UN)
+        finally:
+            os.close(self.descriptor)
+
+
+class BuildService:
+    def __init__(self, settings: Settings):
+        self.settings = settings
+        configured_root = settings.artifact_root.expanduser().absolute()
+        configured_root.mkdir(parents=True, exist_ok=True)
+        if configured_root.is_symlink() or not configured_root.is_dir():
+            raise RuntimeError("artifact root must be a real directory")
+        self.root = configured_root.resolve(strict=True)
+        self.channels = self.root / "channel"
+        self.staging = self.root / "staging"
+        self.locks = self.root / "locks"
+        self._prepare_storage()
+
+    def _prepare_storage(self) -> None:
+        for path in (self.channels, self.staging, self.locks):
+            path.mkdir(mode=0o750, exist_ok=True)
+            if path.is_symlink() or not path.is_dir():
+                raise RuntimeError(f"managed path must be a real directory: {path}")
+        root_device = self.root.stat().st_dev
+        if any(path.stat().st_dev != root_device for path in (self.channels, self.staging)):
+            raise RuntimeError("staging and channel directories must share a filesystem")
+
+    def build(
+        self, channel_id: str, request_input: dict[str, Any], *, force: bool
+    ) -> tuple[int, dict[str, Any], list[tuple[str, str]]]:
+        self.validate_channel(channel_id)
+        digest = self._digest(request_input)
+        request_id = uuid.uuid4().hex
+
+        with self._channel_lock(channel_id):
+            destination = self._channel_path(channel_id)
+            if self._path_exists(destination):
+                self._assert_safe_tree(destination)
+                current = self._read_manifest(destination)
+                if current is None:
+                    raise ApiError(
+                        409,
+                        "existing_build_invalid",
+                        "channel release exists without a valid manifest",
+                    )
+                if current.get("input_digest") == digest and not force:
+                    return 200, self._status_payload(channel_id, current), []
+                if not force:
+                    raise ApiError(
+                        409,
+                        "build_conflict",
+                        "channel already exists with different build input",
+                    )
+
+            stage = self._staging_path(request_id)
+            stage.mkdir(mode=0o750)
+            try:
+                completed = self._run_builder(stage, channel_id, request_input)
+                self._validate_output(stage, channel_id)
+                release = self._read_manifest(stage) or {}
+                manifest = {
+                    **release,
+                    "schema_version": 1,
+                    "channel_id": channel_id,
+                    "request_id": request_id,
+                    "input_digest": digest,
+                    "input": request_input,
+                    "built_at": int(time.time()),
+                    "support_path": release.get(
+                        "support_path", f"/channel/{channel_id}/web/support.html"
+                    ),
+                    "daily_path": release.get(
+                        "daily_path", f"/channel/{channel_id}/sync/daily.html"
+                    ),
+                    "builder": {
+                        "exit_code": completed.returncode,
+                        "stdout_sha256": hashlib.sha256(
+                            completed.stdout.encode("utf-8", "replace")
+                        ).hexdigest(),
+                    },
+                }
+                self._replace_json(stage / "manifest.json", manifest)
+                self._publish(stage, destination, request_id)
+            except subprocess.TimeoutExpired as exc:
+                raise ApiError(500, "build_timeout", "builder timed out") from exc
+            except subprocess.CalledProcessError as exc:
+                message = self._builder_error(exc)
+                raise ApiError(500, "build_failed", message) from exc
+            finally:
+                if stage.exists():
+                    self._remove_staging(stage)
+
+        return 201, self._status_payload(channel_id, manifest), [
+            ("Location", f"/v1/channels/{channel_id}")
+        ]
+
+    def status(self, channel_id: str) -> dict[str, Any]:
+        self.validate_channel(channel_id)
+        destination = self._channel_path(channel_id)
+        if not self._path_exists(destination):
+            raise ApiError(404, "not_found", "channel release not found")
+        self._assert_safe_tree(destination)
+        manifest = self._read_manifest(destination)
+        if manifest is None:
+            raise ApiError(500, "invalid_release", "channel manifest is invalid")
+        return self._status_payload(channel_id, manifest)
+
+    def delete(self, channel_id: str) -> None:
+        self.validate_channel(channel_id)
+        with self._channel_lock(channel_id):
+            destination = self._channel_path(channel_id)
+            if self._path_exists(destination):
+                self._safe_rmtree(destination)
+
+    def _run_builder(
+        self, stage: Path, channel_id: str, request_input: dict[str, Any]
+    ) -> subprocess.CompletedProcess[str]:
+        script = self.settings.project_script
+        if script.is_symlink() or not script.is_file():
+            raise ApiError(500, "builder_unavailable", "builder script is unavailable")
+        # Staging is created empty by the API; new_project refuses an existing
+        # --root unless --force is set.
+        command = [
+            self.settings.python,
+            str(script),
+            "--root",
+            str(stage),
+            "--channel-id",
+            channel_id,
+            "--force",
+        ]
+        for domain in request_input["deployment_domains"]:
+            command.extend(["--deployment-domains", domain])
+        for domain in request_input["reporting_domains"]:
+            command.extend(["--reporting-domains", domain])
+        return subprocess.run(
+            command,
+            cwd=str(Path(__file__).resolve().parent.parent),
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=self.settings.build_timeout,
+            env={**os.environ, "PYTHONUNBUFFERED": "1"},
+        )
+
+    def _publish(self, stage: Path, destination: Path, request_id: str) -> None:
+        self._assert_safe_tree(stage)
+        backup = self.channels / f".replaced-{destination.name}-{request_id}"
+        moved_old = False
+        try:
+            if self._path_exists(destination):
+                self._assert_safe_tree(destination)
+                if self._exchange_directories(stage, destination):
+                    self._safe_rmtree(stage)
+                    self._fsync_directory(self.channels)
+                    return
+                os.rename(destination, backup)
+                moved_old = True
+            os.rename(stage, destination)
+        except Exception:
+            if (
+                moved_old
+                and not self._path_exists(destination)
+                and self._path_exists(backup)
+            ):
+                os.rename(backup, destination)
+            raise
+        self._fsync_directory(self.channels)
+        if self._path_exists(backup):
+            self._safe_rmtree(backup)
+
+    @staticmethod
+    def _exchange_directories(first: Path, second: Path) -> bool:
+        """Atomically exchange directories on Linux; return false if unsupported."""
+        if not sys.platform.startswith("linux"):
+            return False
+        libc = ctypes.CDLL(None, use_errno=True)
+        renameat2 = getattr(libc, "renameat2", None)
+        if renameat2 is None:
+            return False
+        renameat2.argtypes = [
+            ctypes.c_int,
+            ctypes.c_char_p,
+            ctypes.c_int,
+            ctypes.c_char_p,
+            ctypes.c_uint,
+        ]
+        renameat2.restype = ctypes.c_int
+        result = renameat2(
+            -100,
+            os.fsencode(first),
+            -100,
+            os.fsencode(second),
+            2,
+        )
+        if result == 0:
+            return True
+        error = ctypes.get_errno()
+        if error in (errno.ENOSYS, errno.EINVAL, errno.ENOTSUP):
+            return False
+        raise OSError(error, os.strerror(error))
+
+    def _validate_output(self, stage: Path, channel_id: str) -> None:
+        required_dirs = (stage / "web", stage / "sync", stage / "out")
+        if any(not path.is_dir() or path.is_symlink() for path in required_dirs):
+            raise ApiError(
+                500,
+                "invalid_build_output",
+                "builder must produce web, sync, and out directories",
+            )
+        required_files = (
+            stage / "web" / "support.html",
+            stage / "sync" / "daily.html",
+            stage / "manifest.json",
+        )
+        if any(not path.is_file() or path.is_symlink() for path in required_files):
+            raise ApiError(
+                500,
+                "invalid_build_output",
+                "builder must produce support.html, daily.html, and manifest.json",
+            )
+        release = self._read_manifest(stage)
+        if not release or release.get("channel_id") != channel_id:
+            raise ApiError(
+                500,
+                "invalid_build_output",
+                "builder manifest channel_id does not match request",
+            )
+        self._assert_safe_tree(stage)
+
+    @staticmethod
+    def normalize_domains(value: list[str], field: str) -> list[str]:
+        if not value or len(value) > 8:
+            raise ApiError(
+                422, "validation_error", f"{field} must contain 1 to 8 domains"
+            )
+        result: list[str] = []
+        for item in value:
+            domain = item.strip().lower()
+            if not DOMAIN_RE.fullmatch(domain):
+                raise ApiError(
+                    422, "validation_error", f"{field} contains an invalid domain"
+                )
+            result.append(domain.rstrip("."))
+        if len(set(result)) != len(result):
+            raise ApiError(
+                422, "validation_error", f"{field} must not contain duplicates"
+            )
+        return result
+
+    @staticmethod
+    def validate_channel(channel_id: str) -> None:
+        if not CHANNEL_RE.fullmatch(channel_id):
+            raise ApiError(
+                422,
+                "validation_error",
+                "channel id must be 32 lowercase alphanumeric characters",
+            )
+
+    @contextmanager
+    def _channel_lock(self, channel_id: str) -> Iterator[ChannelLock]:
+        path = self.locks / f"{channel_id}.lock"
+        flags = os.O_CREAT | os.O_RDWR
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        try:
+            descriptor = os.open(path, flags, 0o640)
+        except OSError as exc:
+            raise ApiError(409, "channel_locked", "channel lock is unavailable") from exc
+        lock = ChannelLock(descriptor)
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except OSError as exc:
+                if exc.errno in (errno.EACCES, errno.EAGAIN):
+                    raise ApiError(
+                        409, "channel_locked", "another channel operation is in progress"
+                    ) from exc
+                raise
+            yield lock
+        finally:
+            lock.close()
+
+    def _channel_path(self, channel_id: str) -> Path:
+        return self._contained(self.channels, self.channels / channel_id)
+
+    def _staging_path(self, request_id: str) -> Path:
+        return self._contained(self.staging, self.staging / request_id)
+
+    @staticmethod
+    def _contained(parent: Path, child: Path) -> Path:
+        if child.parent != parent or not child.is_relative_to(parent):
+            raise ApiError(500, "unsafe_path", "managed path escaped its root")
+        return child
+
+    @staticmethod
+    def _assert_safe_tree(path: Path) -> None:
+        if path.is_symlink() or not path.is_dir():
+            raise ApiError(409, "unsafe_release", "managed release is not a real directory")
+        for root, directories, files in os.walk(path, followlinks=False):
+            for name in (*directories, *files):
+                entry = Path(root) / name
+                if entry.is_symlink():
+                    raise ApiError(
+                        409, "unsafe_release", "managed release contains a symbolic link"
+                    )
+
+    def _safe_rmtree(self, path: Path) -> None:
+        self._assert_safe_tree(path)
+        shutil.rmtree(path)
+
+    def _remove_staging(self, path: Path) -> None:
+        """Remove an unpublished builder tree without following child symlinks."""
+        self._contained(self.staging, path)
+        if path.is_symlink() or not path.is_dir():
+            raise ApiError(500, "unsafe_path", "staging root is not a real directory")
+        shutil.rmtree(path)
+
+    @staticmethod
+    def _path_exists(path: Path) -> bool:
+        return os.path.lexists(path)
+
+    @staticmethod
+    def _fsync_directory(path: Path) -> None:
+        descriptor = os.open(path, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+
+    @staticmethod
+    def _replace_json(path: Path, value: dict[str, Any]) -> None:
+        """Atomically overwrite JSON that the builder may already have created."""
+        data = json.dumps(value, sort_keys=True, indent=2).encode() + b"\n"
+        temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+        descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640)
+        try:
+            with os.fdopen(descriptor, "wb") as stream:
+                stream.write(data)
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(temporary, path)
+        except Exception:
+            if temporary.exists():
+                temporary.unlink()
+            raise
+
+    @staticmethod
+    def _read_manifest(destination: Path) -> dict[str, Any] | None:
+        if destination.is_symlink() or not destination.is_dir():
+            return None
+        manifest_path = destination / "manifest.json"
+        if manifest_path.is_symlink() or not manifest_path.is_file():
+            return None
+        try:
+            value = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+            return None
+        return value if isinstance(value, dict) else None
+
+    @staticmethod
+    def _digest(value: dict[str, Any]) -> str:
+        canonical = json.dumps(
+            value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
+        ).encode()
+        return hashlib.sha256(canonical).hexdigest()
+
+    @staticmethod
+    def _status_payload(channel_id: str, manifest: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "status": "built",
+            "channel_id": channel_id,
+            "request_id": manifest.get("request_id"),
+            "release_path": f"channel/{channel_id}",
+            "support_path": manifest.get(
+                "support_path", f"/channel/{channel_id}/web/support.html"
+            ),
+            "daily_path": manifest.get(
+                "daily_path", f"/channel/{channel_id}/sync/daily.html"
+            ),
+            "manifest": manifest,
+        }
+
+    @staticmethod
+    def _builder_error(exc: subprocess.CalledProcessError) -> str:
+        stderr = (exc.stderr or "").strip().splitlines()
+        detail = stderr[-1][:500] if stderr else "builder exited unsuccessfully"
+        return f"builder failed: {detail}"