diff --git a/Dockerfile b/Dockerfile index 91624ac64..20fe286f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,6 +111,22 @@ RUN make HAVE_CHD=1 -f ./Makefile.RAHasher \ && cp ./bin64/RAHasher /usr/bin/RAHasher RUN rm -rf /tmp/RALibretro +# Install rom-converto (optional); its static musl build runs on this glibc image. +# Keep the version and both sums in sync with docker/Dockerfile. +ARG TARGETARCH +RUN ROM_CONVERTO_VERSION=v0.23.0 \ + && arch="${TARGETARCH:-$(dpkg --print-architecture)}" \ + && case "${arch}" in \ + amd64) rc="linux-x64-musl"; sum="37d03a67ad97fc02768890e763b4c80072fdfd15f2c5b97e44f9a2b50fc40876" ;; \ + arm64) rc="linux-arm64-musl"; sum="8abe37ca2d1a0e3d719fb8e795e011c672e3f1c088521cf0bb1e467dc658dc3c" ;; \ + *) echo "unsupported architecture: ${arch}" && exit 1 ;; \ + esac \ + && curl -fsSL -o "/tmp/rom-converto-cli-${rc}" \ + "https://github.com/DevYukine/rom-converto/releases/download/${ROM_CONVERTO_VERSION}/rom-converto-cli-${rc}" \ + && echo "${sum} /tmp/rom-converto-cli-${rc}" | sha256sum -c --status \ + && install -m 0755 "/tmp/rom-converto-cli-${rc}" /usr/bin/rom-converto \ + && rm -f "/tmp/rom-converto-cli-${rc}" + # Install frontend dependencies COPY frontend/package.json /app/frontend/ WORKDIR /app/frontend diff --git a/backend/adapters/services/rom_converto.py b/backend/adapters/services/rom_converto.py new file mode 100644 index 000000000..dc9af9b15 --- /dev/null +++ b/backend/adapters/services/rom_converto.py @@ -0,0 +1,341 @@ +import asyncio +import contextlib +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from config import ROM_CONVERTO_ENABLED, ROM_CONVERTO_TIMEOUT +from logger.formatter import LIGHTMAGENTA +from logger.formatter import highlight as hl +from logger.logger import log +from utils.filesystem import SERVED_FILE_MODE +from utils.platform_slugs import UniversalPlatformSlug as UPS + +# The capabilities probe must never hang download paths; a real +# manifest print is instant. +_PROBE_TIMEOUT_SECONDS = 30 + +_STDERR_TAIL_BYTES = 400 + +_BINARY: Final = "rom-converto" + + +class RomConvertoError(Exception): ... + + +class RomConvertoBinaryNotFoundError(RomConvertoError): ... + + +class RomConvertoTimeoutError(RomConvertoError): ... + + +class RomConvertoOperationError(RomConvertoError): + """A conversion command exited nonzero.""" + + +# `xbox convert` keeps only a dump's game partition, so a library never stores its output. +_LOSSY_ARGV: Final[frozenset[tuple[str, ...]]] = frozenset({("xbox", "convert")}) + + +@dataclass(frozen=True) +class Operation: + """A subcommand bringing `input_exts` files to `target`; source and output are appended to `argv`.""" + + target: str + platforms: frozenset[str] + argv: tuple[str, ...] + input_exts: frozenset[str] + output_ext: str + + @property + def lossless(self) -> bool: + return self.argv not in _LOSSY_ARGV + + def output_name(self, src: Path, input_ext: str) -> str: + """`src` renamed from the matched lowercase `input_ext` to the output extension.""" + return f"{src.name[: len(src.name) - len(input_ext)]}{self.output_ext}" + + +def _op( + target: str, + platforms: set[UPS], + argv: str, + input_exts: str, + output_ext: str, +) -> Operation: + return Operation( + target=target, + platforms=frozenset(platforms), + argv=tuple(argv.split()), + input_exts=frozenset(input_exts.split()), + output_ext=output_ext, + ) + + +_DVD_PLATFORMS = {UPS.PSP, UPS.PS2} +_CD_PLATFORMS = {UPS.PSX, UPS.SATURN, UPS.SEGACD, UPS.DC} + +# The CLI's single-file format conversions. Decrypt/encrypt and directory-shaped +# operations (Wii U packs, Switch merge/split, `extract`, Xbox 360 GoD) are left out. +OPERATIONS: Final[tuple[Operation, ...]] = ( + # 3DS + _op("z3ds", {UPS.N3DS}, "ctr compress", ".cia", ".zcia"), + _op("z3ds", {UPS.N3DS}, "ctr compress", ".cci .3ds", ".zcci"), + _op("z3ds", {UPS.N3DS}, "ctr compress", ".cxi", ".zcxi"), + _op("cia", {UPS.N3DS}, "ctr decompress", ".zcia", ".cia"), + _op("cia", {UPS.N3DS}, "ctr convert", ".3ds .cci", ".cia"), + _op("cci", {UPS.N3DS}, "ctr decompress", ".zcci", ".cci"), + _op("cci", {UPS.N3DS}, "ctr convert", ".cia", ".cci"), + # Sony discs + _op("iso", _DVD_PLATFORMS, "cso decompress", ".cso .zso .dax", ".iso"), + # CD-mode CHDs extract to .bin/.cue, so only DVD platforms get this. + _op("iso", _DVD_PLATFORMS, "chd extract", ".chd", ".iso"), + _op("iso", _DVD_PLATFORMS, "cue to-iso", ".cue", ".iso"), + _op("iso", {UPS.PSP}, "psp to-iso", ".pbp", ".iso"), + _op("cso", _DVD_PLATFORMS, "cso compress --format cso", ".iso", ".cso"), + _op("cso", _DVD_PLATFORMS, "chd to-cso --format cso", ".chd", ".cso"), + _op("zso", _DVD_PLATFORMS, "cso compress --format zso", ".iso", ".zso"), + _op("zso", _DVD_PLATFORMS, "chd to-cso --format zso", ".chd", ".zso"), + _op("chd", _DVD_PLATFORMS | _CD_PLATFORMS, "chd compress", ".cue .iso", ".chd"), + _op("chd", _DVD_PLATFORMS, "cso to-chd", ".cso .zso .dax", ".chd"), + # GameCube / Wii + _op("rvz", {UPS.NGC}, "dol compress", ".iso .gcm", ".rvz"), + _op("rvz", {UPS.NGC}, "dol migrate", ".gcz .nkit.iso .nkit.gcz", ".rvz"), + _op("iso", {UPS.NGC}, "dol decompress", ".rvz", ".iso"), + _op("rvz", {UPS.WII}, "rvl compress", ".iso .wbfs", ".rvz"), + _op("rvz", {UPS.WII}, "rvl migrate", ".wia .gcz .nkit.iso .nkit.gcz", ".rvz"), + _op("iso", {UPS.WII}, "rvl decompress", ".rvz", ".iso"), + _op("wbfs", {UPS.WII}, "rvl decompress", ".rvz", ".wbfs"), + # Switch (prod.keys resolved by the CLI itself) + _op("nsz", {UPS.SWITCH, UPS.SWITCH_2}, "nx compress", ".nsp", ".nsz"), + _op("xcz", {UPS.SWITCH, UPS.SWITCH_2}, "nx compress", ".xci", ".xcz"), + _op("nsp", {UPS.SWITCH, UPS.SWITCH_2}, "nx decompress", ".nsz", ".nsp"), + _op("xci", {UPS.SWITCH, UPS.SWITCH_2}, "nx decompress", ".xcz", ".xci"), + # Xbox + _op("xiso", {UPS.XBOX}, "xbox convert", ".iso", ".xiso"), + _op("zar", {UPS.XBOX360}, "xenon compress", ".iso", ".zar"), +) + +# Targets that store the image as-is, so converting to one can grow the file. +UNCOMPRESSED_TARGETS: Final[frozenset[str]] = frozenset( + {"cci", "cia", "iso", "nsp", "wbfs", "xci", "xiso"} +) + + +# Platform slug -> the targets a library can be stored in. +LIBRARY_TARGETS_BY_PLATFORM: Final[dict[str, frozenset[str]]] = { + slug: frozenset( + op.target for op in OPERATIONS if slug in op.platforms and op.lossless + ) + for slug in sorted( + {slug for op in OPERATIONS if op.lossless for slug in op.platforms} + ) +} + + +def _download_formats() -> dict[str, dict[str, list[str]]]: + table: dict[str, dict[str, set[str]]] = {} + for op in OPERATIONS: + for slug in op.platforms: + for ext in op.input_exts: + table.setdefault(slug, {}).setdefault(ext, set()).add(op.target) + return { + slug: {ext: sorted(targets) for ext, targets in sorted(exts.items())} + for slug, exts in sorted(table.items()) + } + + +# Platform slug -> input extension -> the targets a download of it can be converted to. +DOWNLOAD_FORMATS: Final[dict[str, dict[str, list[str]]]] = _download_formats() + + +# Extensions whose format goes by another name than the extension itself. +_FORMAT_ALIASES: Final[dict[str, str]] = { + ".3ds": "cci", + ".gcm": "iso", + ".zcci": "z3ds", + ".zcia": "z3ds", + ".zcxi": "z3ds", +} +_KNOWN_EXTS: Final[frozenset[str]] = frozenset( + {ext for op in OPERATIONS for ext in (*op.input_exts, op.output_ext)} + | _FORMAT_ALIASES.keys() +) + + +def file_format(file_name: str) -> str: + """The format `file_name` is in, as a target name or its extension (`nkit-iso`).""" + name = file_name.lower() + ext = max((e for e in _KNOWN_EXTS if name.endswith(e)), key=len, default=None) + if ext is None: + return Path(name).suffix.lstrip(".") + return _FORMAT_ALIASES.get(ext, ext.lstrip(".").replace(".", "-")) + + +def canonical_format(name: str) -> str: + """The target name a client means by `name`, so `3ds` reads as `cci`.""" + return _FORMAT_ALIASES.get(f".{name}", name) + + +def normalize_platform_formats(raw: dict[str, str]) -> dict[str, str]: + """`raw` with slugs and library targets trimmed and lowercased. + + Raises: + ValueError: A platform has no library targets, or a target isn't one of them. + """ + cleaned = { + str(slug).strip().lower(): str(target).strip().lower() + for slug, target in raw.items() + } + for slug, target in cleaned.items(): + targets = LIBRARY_TARGETS_BY_PLATFORM.get(slug) + if targets is None: + raise ValueError( + f"rom-converto has no library formats for {slug!r}. " + f"Supported: {sorted(LIBRARY_TARGETS_BY_PLATFORM)}." + ) + if target not in targets: + raise ValueError( + f"{target!r} is not a conversion target for {slug}. " + f"Valid options: {sorted(targets)}." + ) + return cleaned + + +def resolve_operation( + platform_slug: str, target: str, file_name: str, *, lossless: bool = False +) -> tuple[Operation, str] | None: + """The operation and matched extension bringing `file_name` to `target`, or None if none applies.""" + name = file_name.lower() + best: tuple[Operation, str] | None = None + for op in OPERATIONS: + if op.target != target or platform_slug not in op.platforms: + continue + if lossless and not op.lossless: + continue + for ext in op.input_exts: + # Prefer the longer extension so `.nkit.iso` is not read as `.iso`. + if name.endswith(ext) and (best is None or len(ext) > len(best[1])): + best = (op, ext) + return best + + +def _tail(text: str) -> str: + return text.strip()[-_STDERR_TAIL_BYTES:] + + +async def _run(argv: list[str], timeout_seconds: float) -> tuple[int, str, str]: + """Run a rom-converto subcommand and return (returncode, stdout, stderr).""" + binary = await asyncio.to_thread(shutil.which, _BINARY) + if binary is None: + raise RomConvertoBinaryNotFoundError(f"{_BINARY} binary not found on PATH") + # The CLI otherwise asks api.github.com for a newer release on every run. + proc = await asyncio.create_subprocess_exec( + binary, + "--no-update-check", + *argv, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout_seconds) + except TimeoutError as err: + _kill(proc) + # Drain the pipes so the transport closes instead of leaking its fds. + await proc.communicate() + raise RomConvertoTimeoutError( + f"rom-converto {' '.join(argv[:2])} timed out after {timeout_seconds}s" + ) from err + except BaseException: + # A cancelled caller must not leave a conversion running unowned. + _kill(proc) + raise + + return ( + proc.returncode or 0, + stdout.decode("utf-8", errors="replace"), + stderr.decode("utf-8", errors="replace"), + ) + + +def _settle_output(operation: Operation, out: Path) -> None: + """Check `out` is all the command wrote beside it, and make it servable.""" + if [p.name for p in out.parent.iterdir()] != [out.name]: + raise RomConvertoOperationError( + f"rom-converto {' '.join(operation.argv)} did not write exactly {out.name}" + ) + # rom-converto may write an owner-only file, which nginx can't serve. + out.chmod(SERVED_FILE_MODE) + + +def _kill(proc: asyncio.subprocess.Process) -> None: + with contextlib.suppress(ProcessLookupError): + proc.kill() + + +class RomConvertoService: + """Service to convert ROMs using the rom-converto CLI.""" + + def __init__(self) -> None: + self._available: bool | None = None + self._probe_lock = asyncio.Lock() + + async def is_enabled(self) -> bool: + if not ROM_CONVERTO_ENABLED: + return False + if self._available is not None: + return self._available + async with self._probe_lock: + if self._available is not None: + return self._available + # A corrupt or wrong-arch binary passes which(); a missing one + # stays uncached so installing it needs no restart. + failure: str | None + try: + code, stdout, _ = await _run(["capabilities"], _PROBE_TIMEOUT_SECONDS) + except RomConvertoBinaryNotFoundError: + return False + except (RomConvertoTimeoutError, OSError) as exc: + failure = str(exc) + else: + failure = f"code {code}" if code != 0 else None + if failure is not None: + log.warning( + f"{hl(_BINARY)} failed its capability " + f"probe ({failure}); disabling integration until restart" + ) + self._available = False + return False + try: + manifest = json.loads(stdout) + except json.JSONDecodeError: + manifest = None + version = ( + manifest.get("version") if isinstance(manifest, dict) else None + ) or "unknown version" + log.info( + f"Detected {hl('rom-converto', color=LIGHTMAGENTA)} {hl(str(version))}" + ) + self._available = True + return True + + async def convert(self, operation: Operation, src: Path, out: Path) -> None: + """Run `operation` on `src`, writing `out`, which must be alone in its directory. + + Raises: + RomConvertoOperationError: The command failed or wrote more than `out`. + """ + argv = [*operation.argv, str(src), str(out)] + code, stdout, stderr = await _run(argv, ROM_CONVERTO_TIMEOUT) + if code != 0: + diagnostic = _tail(stderr) or _tail(stdout) + raise RomConvertoOperationError( + f"rom-converto {' '.join(operation.argv)} failed with code {code}: {diagnostic}" + ) + await asyncio.to_thread(_settle_output, operation, out) + + +rom_converto_service = RomConvertoService() diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 89d1821e9..88b62a27e 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -65,6 +65,19 @@ ROM_PATCHER_MAX_CONCURRENCY: Final[int] = max( 1, safe_int(_get_env("ROM_PATCHER_MAX_CONCURRENCY"), 2) ) +# ROM CONVERTO +ROM_CONVERTO_ENABLED: Final[bool] = safe_str_to_bool(_get_env("ROM_CONVERTO_ENABLED")) +# Seconds per rom-converto CLI operation. +ROM_CONVERTO_TIMEOUT: Final[int] = max( + 1, safe_int(_get_env("ROM_CONVERTO_TIMEOUT"), 600) +) +# Concurrent conversion subprocesses per process (each web and RQ worker). +ROM_CONVERTO_MAX_CONCURRENCY: Final[int] = max( + 1, safe_int(_get_env("ROM_CONVERTO_MAX_CONCURRENCY"), 2) +) +# Disk cache for converted downloads, under the tree nginx serves at /cache/. +ROM_CONVERTO_CACHE_PATH: Final[str] = f"{ROMM_BASE_PATH}/cache/converts" + # DATABASE DB_HOST: Final[str | None] = _get_env("DB_HOST") DB_PORT: Final[int] = safe_int(_get_env("DB_PORT"), 3306) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index a00d5d207..5e4e59763 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -1,3 +1,4 @@ +import dataclasses import enum import functools import glob @@ -14,6 +15,7 @@ import yaml from sqlalchemy import URL from yaml.loader import SafeLoader +from adapters.services.rom_converto import normalize_platform_formats from config import ( DB_HOST, DB_NAME, @@ -401,6 +403,22 @@ VALID_SCAN_PRIORITY_SOURCES = frozenset( VALID_SCAN_REGION_MODES = frozenset({"prefer_rom_tags", "prefer_config"}) +@dataclasses.dataclass +class ConvertoConfig: + download_conversion_enabled: bool = False + cache_ttl_hours: int = 24 + # 0 leaves the cache unbounded. + cache_max_size_gb: int = 20 + platform_formats: dict[str, str] = dataclasses.field(default_factory=dict) + + +# The smallest value each integer converto.* setting accepts. +CONVERTO_INT_MINIMUMS: Final[dict[str, int]] = { + "cache_ttl_hours": 1, + "cache_max_size_gb": 0, +} + + class EjsControls(TypedDict): _0: dict[int, EjsControlsButton] # button_number -> EjsControlsButton _1: dict[int, EjsControlsButton] @@ -506,6 +524,7 @@ class Config: GAMELIST_MEDIA_IMAGE: MetadataMediaType STREAMING_ENABLED: bool STREAMING_CONTAINERS: list[StreamingContainer] + CONVERTO: ConvertoConfig def __init__(self, **entries: Any) -> None: self.__dict__.update(entries) @@ -879,6 +898,12 @@ class ConfigManager: STREAMING_CONTAINERS=pydash.get( self._raw_config, "streaming.containers", [] ), + CONVERTO=ConvertoConfig( + **{ + key: pydash.get(self._raw_config, f"converto.{key}", default) + for key, default in dataclasses.asdict(ConvertoConfig()).items() + } + ), STRUCTURE_TEMPLATES=pydash.get( self._raw_config, "filesystem.structure", {} ), @@ -1281,6 +1306,31 @@ class ConfigManager: len(legacy_containers), ) + if not isinstance(self.config.CONVERTO.download_conversion_enabled, bool): + log.critical( + "Invalid config.yml: converto.download_conversion_enabled must be a boolean" + ) + sys.exit(3) + + for key, minimum in CONVERTO_INT_MINIMUMS.items(): + value = getattr(self.config.CONVERTO, key) + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + log.critical( + f"Invalid config.yml: converto.{key} must be an integer >= {minimum}" + ) + sys.exit(3) + + try: + self.config.CONVERTO.platform_formats = normalize_platform_formats( + self._validated_platform_map( + self.config.CONVERTO.platform_formats or {}, + "converto.platform_formats", + ) + ) + except ValueError as exc: + log.critical(f"Invalid config.yml: converto.platform_formats: {exc}") + sys.exit(3) + def get_config(self) -> Config: try: with open(self.config_file, "r") as config_file: @@ -1366,6 +1416,7 @@ class ConfigManager: "export": self.config.PEGASUS_AUTO_EXPORT_ON_SCAN, }, }, + "converto": dataclasses.asdict(self.config.CONVERTO), } # The streaming section isn't editable at runtime, but it must survive @@ -1500,5 +1551,10 @@ class ConfigManager: self.config.PEGASUS_AUTO_EXPORT_ON_SCAN = pegasus_export self._update_config_file() + def update_converto_settings(self, converto: ConvertoConfig) -> None: + """Replace the whole converto.* section and persist it to config.yml.""" + self.config.CONVERTO = converto + self._update_config_file() + config_manager = ConfigManager() diff --git a/backend/endpoints/configs.py b/backend/endpoints/configs.py index 2c9adfc2b..a8d900451 100644 --- a/backend/endpoints/configs.py +++ b/backend/endpoints/configs.py @@ -1,7 +1,13 @@ from fastapi import HTTPException, Request, status -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, Field, field_validator +from adapters.services.rom_converto import ( + DOWNLOAD_FORMATS, + LIBRARY_TARGETS_BY_PLATFORM, + normalize_platform_formats, +) from config.config_manager import ( + CONVERTO_INT_MINIMUMS, DEFAULT_EXCLUDED_EXTENSIONS, DEFAULT_EXCLUDED_FILES, DEFAULT_EXCLUDED_MULTI_FILE_DIRS, @@ -9,6 +15,7 @@ from config.config_manager import ( VALID_GAMELIST_IMAGE_TYPES, VALID_GAMELIST_THUMBNAIL_TYPES, VALID_SCAN_PRIORITY_SOURCES, + ConvertoConfig, ExclusionType, MetadataMediaType, ) @@ -117,6 +124,19 @@ class ScanSettingsPayload(BaseModel): return value +class ConvertoSettingsPayload(BaseModel): + """The converto.* settings editable at runtime; `cache_ttl_hours` stays config.yml-only.""" + + download_conversion_enabled: bool + cache_max_size_gb: int = Field(ge=CONVERTO_INT_MINIMUMS["cache_max_size_gb"]) + platform_formats: dict[str, str] + + @field_validator("platform_formats") + @classmethod + def validate_platform_formats(cls, value: dict[str, str]) -> dict[str, str]: + return normalize_platform_formats(value) + + @router.get("") def get_config(request: Request) -> ConfigResponse: """Get config endpoint @@ -169,6 +189,12 @@ def get_config(request: Request) -> ConfigResponse: GAMELIST_MEDIA_THUMBNAIL=cfg.GAMELIST_MEDIA_THUMBNAIL, GAMELIST_MEDIA_IMAGE=cfg.GAMELIST_MEDIA_IMAGE, PEGASUS_AUTO_EXPORT_ON_SCAN=cfg.PEGASUS_AUTO_EXPORT_ON_SCAN, + CONVERTO=cfg.CONVERTO, + CONVERTO_LIBRARY_TARGETS={ + slug: sorted(targets) + for slug, targets in LIBRARY_TARGETS_BY_PLATFORM.items() + }, + CONVERTO_DOWNLOAD_FORMATS=DOWNLOAD_FORMATS, ) @@ -338,3 +364,24 @@ async def update_scan_settings(request: Request, payload: ScanSettingsPayload) - # cached gallery sidecar is stale the moment the order changes. if region_priority_changed: db_rom_handler.invalidate_filter_values_cache() + + +@protected_route(router.put, "/converto_settings", [Scope.PLATFORMS_WRITE]) +async def update_converto_settings( + request: Request, payload: ConvertoSettingsPayload +) -> None: + """Update the runtime-editable converto.* settings""" + + current = cm.get_config().CONVERTO + try: + cm.update_converto_settings( + ConvertoConfig( + **payload.model_dump(), cache_ttl_hours=current.cache_ttl_hours + ) + ) + except ConfigNotWritableException as exc: + log.critical(exc.message) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=exc.message + ) from exc + _record_config(request, "converto_settings", "update") diff --git a/backend/endpoints/responses/config.py b/backend/endpoints/responses/config.py index edba7b316..a0db55cda 100644 --- a/backend/endpoints/responses/config.py +++ b/backend/endpoints/responses/config.py @@ -1,6 +1,11 @@ from typing import TypedDict -from config.config_manager import EjsControls, MetadataMediaType, NetplayICEServer +from config.config_manager import ( + ConvertoConfig, + EjsControls, + MetadataMediaType, + NetplayICEServer, +) class ConfigResponse(TypedDict): @@ -40,3 +45,8 @@ class ConfigResponse(TypedDict): GAMELIST_MEDIA_THUMBNAIL: MetadataMediaType GAMELIST_MEDIA_IMAGE: MetadataMediaType PEGASUS_AUTO_EXPORT_ON_SCAN: bool + CONVERTO: ConvertoConfig + # Platform slug -> the formats its library can be stored in. + CONVERTO_LIBRARY_TARGETS: dict[str, list[str]] + # Platform slug -> input extension -> the formats a download can be converted to. + CONVERTO_DOWNLOAD_FORMATS: dict[str, dict[str, list[str]]] diff --git a/backend/endpoints/roms/__init__.py b/backend/endpoints/roms/__init__.py index 92f58b3fb..58f2785b0 100644 --- a/backend/endpoints/roms/__init__.py +++ b/backend/endpoints/roms/__init__.py @@ -122,6 +122,7 @@ from models.rom import ( PINNED_MEDIA_MAX_ITEMS, TITLE_ID_MAX_LENGTH, Rom, + RomFile, RomIdentity, RomUserStatus, SaveTargetLayout, @@ -130,6 +131,13 @@ from models.rom import ( ) from utils import switch from utils.background_tasks import fire_and_forget +from utils.conversion_cache import ( + RETRY_AFTER_SECONDS, + FormatOutcome, + get_redirect_path, + parse_formats, + resolve_format_download, +) from utils.database import safe_int, safe_str_to_bool from utils.filesystem import check_filename_length, sanitize_filename from utils.hashing import crc32_to_hex @@ -1312,11 +1320,86 @@ def get_rom( return DetailedRomSchema.from_orm_with_request(rom, request) +CONTENT_RESPONSES: Final[dict[int | str, dict[str, Any]]] = { + status.HTTP_202_ACCEPTED: { + "description": "The requested format is converting; retry after Retry-After." + }, + status.HTTP_404_NOT_FOUND: {}, + status.HTTP_406_NOT_ACCEPTABLE: { + "description": "No requested format can be served for this download." + }, +} + +FormatQuery = Annotated[ + str | None, + Query( + alias="format", + description="Comma-separated formats the client can read, such as " + "`zso,iso`. The stored file is served when its format is listed, else " + "a converted copy: 202 with Retry-After while it converts, 406 when no " + "listed format can be produced. Single-file downloads only.", + ), +] + + +async def _negotiate_format( + request: Request, + rom: Rom, + files: list[RomFile], + raw_formats: str | None, + *, + start: bool, +) -> Response | None: + """The response a `?format=` download gets, or None to serve the stored file.""" + formats = parse_formats(raw_formats or "") + if not formats: + return None + if len(files) != 1: + raise HTTPException( + status_code=status.HTTP_406_NOT_ACCEPTABLE, + detail="A format can only be requested for a single file", + ) + + file = files[0] + # DISABLE_DOWNLOAD_ENDPOINT_AUTH and KIOSK_MODE open this route to visitors, + # and a conversion is minutes of CPU. + allowed = request.user.is_authenticated and not request.user.is_kiosk_guest + resolution = await resolve_format_download( + rom, file, formats, allowed=allowed, start=start, touch=start + ) + if resolution.outcome == FormatOutcome.ORIGINAL: + return None + if resolution.outcome == FormatOutcome.PENDING: + return Response( + status_code=status.HTTP_202_ACCEPTED, + headers={"Retry-After": str(RETRY_AFTER_SECONDS)}, + ) + if resolution.outcome == FormatOutcome.UNAVAILABLE or resolution.path is None: + raise HTTPException( + status_code=status.HTTP_406_NOT_ACCEPTABLE, + detail=f"{file.file_name} can't be served as {', '.join(formats)}", + ) + + converted = resolution.path + if DEV_MODE: + return FileResponse( + path=converted, + filename=converted.name, + headers={ + "Content-Disposition": content_disposition(converted.name), + "Content-Type": "application/octet-stream", + }, + ) + return FileRedirectResponse( + download_path=get_redirect_path(converted), filename=converted.name + ) + + @protected_route( router.head, "/{id}/content/{file_name}", [] if DISABLE_DOWNLOAD_ENDPOINT_AUTH else [Scope.ROMS_READ], - responses={status.HTTP_404_NOT_FOUND: {}}, + responses=CONTENT_RESPONSES, ) async def head_rom_content( request: Request, @@ -1328,6 +1411,7 @@ async def head_rom_content( description="Comma-separated list of file ids to download for multi-part roms." ), ] = None, + formats: FormatQuery = None, ) -> Response: """Retrieve head information for a rom file download.""" @@ -1350,6 +1434,10 @@ async def head_rom_content( detail=f"No files found for ROM {id}", ) + # Report what a GET would serve, but never start a conversion. + if negotiated := await _negotiate_format(request, rom, files, formats, start=False): + return negotiated + # Serve the file directly in development mode for emulatorjs if DEV_MODE: if len(files) == 1: @@ -1410,7 +1498,7 @@ async def head_rom_content( router.get, "/{id}/content/{file_name}", [] if DISABLE_DOWNLOAD_ENDPOINT_AUTH else [Scope.ROMS_READ], - responses={status.HTTP_404_NOT_FOUND: {}}, + responses=CONTENT_RESPONSES, ) async def get_rom_content( request: Request, @@ -1429,6 +1517,7 @@ async def get_rom_content( "recorded as a player load rather than a download." ), ] = "download", + formats: FormatQuery = None, ) -> Response: """Download a rom. @@ -1494,6 +1583,13 @@ async def get_rom_content( ) return response + if negotiated := await _negotiate_format(request, rom, files, formats, start=True): + return ( + served(negotiated) + if negotiated.status_code == status.HTTP_200_OK + else negotiated + ) + m3u_files = playlist_files(files) # Serve the file directly in development mode for emulatorjs diff --git a/backend/endpoints/tasks.py b/backend/endpoints/tasks.py index 797021117..bad016289 100644 --- a/backend/endpoints/tasks.py +++ b/backend/endpoints/tasks.py @@ -56,6 +56,7 @@ VISIBLE_SCHEDULED_TASKS: Final[dict[str, TaskSpec]] = { "build_recommendations", "convert_images_to_webp", "cleanup_zip_cache", + "cleanup_conversion_cache", "cleanup_orphaned_resources", ) } diff --git a/backend/handler/rom_conversion.py b/backend/handler/rom_conversion.py index 6102ed229..1f2cf0e0a 100644 --- a/backend/handler/rom_conversion.py +++ b/backend/handler/rom_conversion.py @@ -7,7 +7,8 @@ from logger.formatter import highlight as hl from logger.logger import log from models.rom import Rom -_STAGE_PREFIX = ".romm_tmp_" +# Scans and the watcher skip dot-prefixed entries, so staged files stay unseen. +STAGE_PREFIX = ".romm_tmp_" # Parallel uploads each promote the ROM, maybe on different gunicorn workers. # Keyed by the target folder, since lone files sharing a stem promote into one. @@ -39,7 +40,7 @@ async def _promote(rom: Rom) -> Rom: origin = f"{fs_path}/{fs_name}" dest_dir = f"{fs_path}/{folder}" final = f"{dest_dir}/{fs_name}" - staged = f"{fs_path}/{_STAGE_PREFIX}{fs_name}" + staged = f"{fs_path}/{STAGE_PREFIX}{fs_name}" extensionless = folder == fs_name # Extensionless dest_dir is the file's own path; only a directory collides. diff --git a/backend/tasks/manual/convert_library.py b/backend/tasks/manual/convert_library.py new file mode 100644 index 000000000..7a698fdc0 --- /dev/null +++ b/backend/tasks/manual/convert_library.py @@ -0,0 +1,315 @@ +"""Convert each matched ROM to its platform's library format, in place.""" + +import asyncio +import contextlib +import os +import re +import shutil +import tempfile +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Final + +from sqlalchemy.exc import SQLAlchemyError + +from adapters.services.rom_converto import ( + Operation, + RomConvertoError, + file_format, + resolve_operation, + rom_converto_service, +) +from config.config_manager import config_manager as cm +from handler.database import db_platform_handler, db_rom_handler +from handler.filesystem import fs_rom_handler +from handler.rom_conversion import STAGE_PREFIX +from handler.rom_files import refresh_rom_files +from logger.formatter import highlight as hl +from logger.logger import log +from models.rom import Rom, RomFile, RomFileCategory +from tasks.registry import CONVERT_LIBRARY_SPEC +from tasks.scheduled.convert_images_to_webp import ConversionStats +from tasks.tasks import Task +from utils.context import initialize_context +from utils.filesystem import LINK_FALLBACK_ERRNOS + +_CUE_FILE_LINE: Final = re.compile(r'^\s*FILE\s+(?:"([^"]+)"|(\S+))', re.IGNORECASE) +_CUE_EXT: Final = ".cue" +_PLAYLIST_EXT: Final = ".m3u" +_CONVERT_STAGE_PREFIX: Final = f"{STAGE_PREFIX}convert_" + + +@dataclass +class ConvertLibraryStats: + converted: int = 0 + already_converted: int = 0 + # Converted files no longer hash-match a DAT, so only matched roms convert. + unmatched: int = 0 + unsupported: int = 0 + failed: int = 0 + bytes_saved: int = 0 + + +def _cue_tracks(cue: Path) -> list[Path]: + """The track files a cue sheet references, all beside it.""" + tracks: list[Path] = [] + for line in cue.read_text(encoding="utf-8-sig", errors="replace").splitlines(): + match = _CUE_FILE_LINE.match(line) + if match is None: + continue + name = match.group(1) or match.group(2) + # A track outside the cue's folder isn't this rom's to delete. + if Path(name).name != name: + raise ValueError(f"{cue.name} references a track outside its folder") + tracks.append(cue.with_name(name)) + return tracks + + +def _shared_tracks(cue: Path, tracks: list[Path]) -> set[Path]: + """The tracks of `cue` that another cue beside it also references.""" + others: set[Path] = set() + for other in cue.parent.iterdir(): + if other == cue or other.suffix.lower() != _CUE_EXT or not other.is_file(): + continue + with contextlib.suppress(OSError, ValueError): + others.update(_cue_tracks(other)) + return set(tracks) & others + + +def _renamed_entry(line: str, renames: dict[str, str]) -> str: + entry = line.strip() + return renames.get(os.path.normpath(entry), line) if entry else line + + +def _rewritten_playlists(folder: Path, renames: dict[str, str]) -> dict[Path, str]: + """The playlists in `folder` that name a renamed file, with their new text.""" + rewrites: dict[Path, str] = {} + for playlist in folder.iterdir(): + if playlist.suffix.lower() != _PLAYLIST_EXT or not playlist.is_file(): + continue + lines = playlist.read_text(encoding="utf-8-sig", errors="replace").splitlines() + rewritten = [_renamed_entry(line, renames) for line in lines] + if rewritten != lines: + rewrites[playlist] = "\n".join(rewritten) + "\n" + return rewrites + + +async def _rewrite_playlists(folder: Path, renames: dict[str, str]) -> None: + """Point each playlist entry for a converted file at its new name.""" + rewrites = await asyncio.to_thread(_rewritten_playlists, folder, renames) + for playlist, text in rewrites.items(): + async with fs_rom_handler._atomic_write(playlist) as staged: + await asyncio.to_thread(staged.write_text, text) + + +def _sources(src: Path, final: Path, input_ext: str) -> list[Path]: + """The files a conversion of `src` replaces, refusing one that would overwrite.""" + if final.exists(): + raise FileExistsError(f"{final.name} already exists") + if input_ext != _CUE_EXT: + return [src] + tracks = _cue_tracks(src) + if shared := _shared_tracks(src, tracks): + raise ValueError( + f"{src.name} shares {sorted(p.name for p in shared)} with another cue" + ) + return [src, *tracks] + + +def _drop_stale_stages(folder: Path) -> None: + """Remove stage dirs a killed conversion left, none recent enough to be live.""" + cutoff = time.time() - CONVERT_LIBRARY_SPEC.timeout + for stage in folder.glob(f"{_CONVERT_STAGE_PREFIX}*"): + with contextlib.suppress(OSError): + if stage.is_dir() and stage.stat().st_mtime < cutoff: + shutil.rmtree(stage, ignore_errors=True) + + +def _publish(stage_dir: Path, final: Path) -> None: + """Move the staged output to `final`, never replacing a file that appeared there.""" + staged = stage_dir / final.name + try: + os.link(staged, final) + except OSError as exc: + if exc.errno not in LINK_FALLBACK_ERRNOS: + raise + if final.exists(): + raise FileExistsError(f"{final.name} already exists") from None + os.replace(staged, final) + + +def _replace_sources(sources: list[Path], final: Path) -> int: + """Delete the converted files and return the bytes the conversion saved.""" + saved = sum(p.stat().st_size for p in sources if p.exists()) - final.stat().st_size + for path in sources: + try: + path.unlink(missing_ok=True) + except OSError as exc: + log.warning( + f"Converted to {hl(final.name)} but could not delete {path}: {exc}" + ) + return saved + + +async def _convert_beside( + src: Path, final: Path, operation: Operation, stage_parent: Path +) -> None: + """Convert `src` to `final` beside it, staging in `stage_parent` and leaving `src` in place.""" + stage_dir = Path( + await asyncio.to_thread( + tempfile.mkdtemp, prefix=_CONVERT_STAGE_PREFIX, dir=stage_parent + ) + ) + try: + await rom_converto_service.convert( + operation, src=src, out=stage_dir / final.name + ) + await asyncio.to_thread(_publish, stage_dir, final) + finally: + await asyncio.to_thread(shutil.rmtree, stage_dir, True) + + +def _names_taken(rom: Rom, names: list[str]) -> list[str]: + """The `names` another rom row beside `rom` already holds.""" + if not names: + return [] + owned = db_rom_handler.get_roms_by_fs_name(rom.platform_id, names) + return sorted(n for n in names if f"{rom.fs_path}/{n}" in owned) + + +def _game_files(rom: Rom) -> list[RomFile]: + return [ + f + for f in rom.files + if f.is_top_level + and not f.missing_from_fs + and f.category in (None, RomFileCategory.GAME) + ] + + +class ConvertLibraryTask(Task): + def __init__(self) -> None: + super().__init__(CONVERT_LIBRARY_SPEC) + + async def _convert_rom( + self, rom: Rom, target: str, stats: ConvertLibraryStats + ) -> None: + single_file = rom.has_simple_single_file + planned: list[tuple[RomFile, Operation, str]] = [] + for rom_file in _game_files(rom): + if file_format(rom_file.file_name) == target: + stats.already_converted += 1 + continue + resolved = resolve_operation( + rom.platform_slug, target, rom_file.file_name, lossless=True + ) + if resolved is None: + stats.unsupported += 1 + continue + name = resolved[0].output_name(Path(rom.fs_name), resolved[1]) + if single_file and _names_taken(rom, [name]): + log.warning( + f"Not converting {hl(rom.fs_name)}: a rom named {name} exists" + ) + stats.failed += 1 + continue + planned.append((rom_file, *resolved)) + if not planned: + return + if not rom.is_identified: + stats.unmatched += 1 + return + + # A rom folder lists dot-prefixed subfolders as its files, so stage beside it. + stage_parent = fs_rom_handler.validate_path(rom.fs_path) + await asyncio.to_thread(_drop_stale_stages, stage_parent) + # Originals go only once the rom and its playlists point at the new files. + published: list[tuple[Path, Path, list[Path]]] = [] + for rom_file, operation, input_ext in planned: + src = fs_rom_handler.validate_path(rom_file.full_path) + final = src.with_name(operation.output_name(src, input_ext)) + try: + sources = await asyncio.to_thread(_sources, src, final, input_ext) + if single_file and ( + taken := _names_taken(rom, [t.name for t in sources[1:]]) + ): + raise ValueError(f"{taken} belong to other roms") + await _convert_beside(src, final, operation, stage_parent) + except (RomConvertoError, OSError, ValueError) as exc: + log.warning(f"Could not convert {hl(rom_file.file_name)}: {exc}") + stats.failed += 1 + continue + published.append((src, final, sources)) + + if not published: + return + renames = {src.name: final.name for src, final, _ in published} + try: + if single_file: + db_rom_handler.update_rom(rom.id, {"fs_name": renames[rom.fs_name]}) + else: + await _rewrite_playlists( + fs_rom_handler.validate_path(rom.full_path), renames + ) + except (OSError, SQLAlchemyError) as exc: + log.warning(f"Kept the originals of {hl(rom.fs_name)}: {exc}") + stats.failed += len(published) + if single_file: + for _, final, _ in published: + final.unlink(missing_ok=True) + return + + for src, final, sources in published: + stats.bytes_saved += await asyncio.to_thread( + _replace_sources, sources, final + ) + stats.converted += 1 + log.info(f"Converted {hl(src.name)} to {hl(final.name)}") + refreshed = db_rom_handler.get_rom(rom.id) + if refreshed is not None: + await refresh_rom_files(refreshed) + + @initialize_context() + async def run(self, platform_id: int | None = None) -> dict[str, int]: + """Convert every matched ROM on platforms that have a library format.""" + log.info(f"Starting {self.spec.title} task...") + stats = ConvertLibraryStats() + if not await rom_converto_service.is_enabled(): + log.warning( + f"{self.spec.title} needs rom-converto; set ROM_CONVERTO_ENABLED" + ) + return asdict(stats) + + formats = cm.get_config().CONVERTO.platform_formats + rom_ids: list[tuple[int, str]] = [] + for platform in db_platform_handler.get_platforms(): + if platform_id is not None and platform.id != platform_id: + continue + target = formats.get(platform.slug) + if target: + rom_ids.extend( + (rom_id, target) + for rom_id in db_rom_handler.get_rom_ids(platform_ids=[platform.id]) + ) + + progress = ConversionStats() + progress.update(total=len(rom_ids)) + for index, (rom_id, target) in enumerate(rom_ids): + rom = db_rom_handler.get_rom(rom_id) + if rom is not None: + try: + await self._convert_rom(rom, target, stats) + except Exception as exc: + log.exception(f"Could not convert {hl(rom.fs_name)}: {exc}") + stats.failed += 1 + progress.update(processed=index + 1, errors=stats.failed) + + log.info( + f"{self.spec.title} completed: {stats.converted} converted, " + f"{stats.failed} failed, {stats.bytes_saved} bytes saved" + ) + return asdict(stats) + + +convert_library_task = ConvertLibraryTask() diff --git a/backend/tasks/registry.py b/backend/tasks/registry.py index aadfee445..0ce2f73d3 100644 --- a/backend/tasks/registry.py +++ b/backend/tasks/registry.py @@ -113,6 +113,15 @@ CLEANUP_ZIP_CACHE_SPEC: Final = TaskSpec( cron_string=SCHEDULED_CLEANUP_ZIP_CACHE_CRON, ) +CLEANUP_CONVERSION_CACHE_SPEC: Final = TaskSpec( + implementation="tasks.scheduled.cleanup_conversion_cache.cleanup_conversion_cache_task", + title="Scheduled conversion cache cleanup", + description="Removes stale converted download files based on TTL", + task_type=TaskType.CLEANUP, + enabled=True, + cron_string="0 4 * * *", +) + CLEANUP_ORPHANED_RESOURCES_SPEC: Final = TaskSpec( implementation="tasks.scheduled.cleanup_orphaned_resources.cleanup_orphaned_resources_task", title="Cleanup orphaned resources", @@ -232,6 +241,20 @@ RECOMPUTE_SAVE_CONTENT_HASHES_SPEC: Final = TaskSpec( manual_run=True, ) +CONVERT_LIBRARY_SPEC: Final = TaskSpec( + implementation="tasks.manual.convert_library.convert_library_task", + title="Convert library", + description=( + "Convert each matched ROM to its platform's library format, " + "replacing the original files" + ), + task_type=TaskType.CONVERSION, + enabled=True, + manual_run=True, + # One conversion after another, each up to ROM_CONVERTO_TIMEOUT. + timeout=SCAN_TIMEOUT, +) + # The keys are the names the API and the cron schedule address a task by, and # they end up in the job payload, so they outlive any given release. Every task # that runs on a schedule belongs here; which of them the API surfaces is the @@ -243,6 +266,7 @@ SCHEDULED_TASKS: Final[dict[str, TaskSpec]] = { "build_recommendations": BUILD_RECOMMENDATIONS_SPEC, "convert_images_to_webp": CONVERT_IMAGES_TO_WEBP_SPEC, "cleanup_zip_cache": CLEANUP_ZIP_CACHE_SPEC, + "cleanup_conversion_cache": CLEANUP_CONVERSION_CACHE_SPEC, "cleanup_orphaned_resources": CLEANUP_ORPHANED_RESOURCES_SPEC, "cleanup_netplay": CLEANUP_NETPLAY_SPEC, "cleanup_upload_tmp": CLEANUP_UPLOAD_TMP_SPEC, @@ -258,6 +282,7 @@ MANUAL_TASKS: Final[dict[str, TaskSpec]] = { "cleanup_missing_firmware": CLEANUP_MISSING_FIRMWARE_SPEC, "sync_folder_scan": SYNC_FOLDER_SCAN_SPEC, "recompute_save_content_hashes": RECOMPUTE_SAVE_CONTENT_HASHES_SPEC, + "convert_library": CONVERT_LIBRARY_SPEC, } diff --git a/backend/tasks/scheduled/cleanup_conversion_cache.py b/backend/tasks/scheduled/cleanup_conversion_cache.py new file mode 100644 index 000000000..79bd7cde0 --- /dev/null +++ b/backend/tasks/scheduled/cleanup_conversion_cache.py @@ -0,0 +1,20 @@ +from logger.logger import log +from tasks.registry import CLEANUP_CONVERSION_CACHE_SPEC +from tasks.tasks import PeriodicTask +from utils.conversion_cache import cleanup_stale_conversions + + +class CleanupConversionCacheTask(PeriodicTask): + def __init__(self) -> None: + super().__init__(CLEANUP_CONVERSION_CACHE_SPEC) + + async def run(self) -> None: + if not self.spec.enabled: + return + + deleted = cleanup_stale_conversions() + if deleted: + log.info(f"Cleaned up {deleted} stale converted download caches") + + +cleanup_conversion_cache_task = CleanupConversionCacheTask() diff --git a/backend/tests/adapters/services/test_rom_converto.py b/backend/tests/adapters/services/test_rom_converto.py new file mode 100644 index 000000000..858a9fb4c --- /dev/null +++ b/backend/tests/adapters/services/test_rom_converto.py @@ -0,0 +1,392 @@ +import asyncio +import stat +from pathlib import Path +from unittest.mock import patch + +import pytest + +from adapters.services import rom_converto +from adapters.services.rom_converto import ( + DOWNLOAD_FORMATS, + LIBRARY_TARGETS_BY_PLATFORM, + Operation, + RomConvertoBinaryNotFoundError, + RomConvertoOperationError, + RomConvertoService, + RomConvertoTimeoutError, + canonical_format, + file_format, + resolve_operation, +) +from utils.filesystem import SERVED_FILE_MODE + + +class FakeProc: + def __init__( + self, + returncode: int = 0, + stdout: bytes = b"", + stderr: bytes = b"", + delay: float = 0.0, + ): + self._stdout = stdout + self._stderr = stderr + self._delay = delay + self.killed = False + self.returncode = returncode + + async def communicate(self) -> tuple[bytes, bytes]: + if self._delay and not self.killed: + await asyncio.sleep(self._delay) + return self._stdout, self._stderr + + def kill(self) -> None: + self.killed = True + + +@pytest.fixture +def service() -> RomConvertoService: + return RomConvertoService() + + +class TestRun: + async def test_binary_missing_raises(self) -> None: + with ( + patch("shutil.which", return_value=None), + pytest.raises(RomConvertoBinaryNotFoundError), + ): + await rom_converto._run(["info", "--json", "x"], timeout_seconds=1) + + async def test_timeout_kills_process(self) -> None: + proc = FakeProc(delay=5.0) + with ( + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", return_value=proc), + pytest.raises(RomConvertoTimeoutError), + ): + await rom_converto._run(["info", "--json", "x"], timeout_seconds=0.01) + assert proc.killed is True + + async def test_cancellation_kills_process(self) -> None: + proc = FakeProc(delay=5.0) + with ( + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", return_value=proc), + ): + run = asyncio.create_task( + rom_converto._run(["info", "--json", "x"], timeout_seconds=10) + ) + await asyncio.sleep(0.01) + run.cancel() + with pytest.raises(asyncio.CancelledError): + await run + assert proc.killed is True + + +class TestIsEnabled: + async def test_disabled_when_config_disabled(self, service: RomConvertoService): + with ( + patch.object(rom_converto, "ROM_CONVERTO_ENABLED", False), + patch("shutil.which", return_value="/usr/bin/rom-converto"), + ): + assert await service.is_enabled() is False + + async def test_probe_fail_cached_false(self, service: RomConvertoService): + proc = FakeProc(returncode=1) + spawn_count = 0 + + async def spawn(*args, **kwargs): + nonlocal spawn_count + spawn_count += 1 + return proc + + with ( + patch.object(rom_converto, "ROM_CONVERTO_ENABLED", True), + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", spawn), + ): + assert await service.is_enabled() is False + assert await service.is_enabled() is False + assert spawn_count == 1 + + @pytest.mark.parametrize( + "error", + [OSError(8, "Exec format error"), RomConvertoTimeoutError("timed out")], + ) + async def test_probe_that_cannot_run_is_cached_false( + self, service: RomConvertoService, error: Exception + ): + spawn_count = 0 + + async def spawn(*args, **kwargs): + nonlocal spawn_count + spawn_count += 1 + raise error + + with ( + patch.object(rom_converto, "ROM_CONVERTO_ENABLED", True), + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", spawn), + ): + assert await service.is_enabled() is False + assert await service.is_enabled() is False + assert spawn_count == 1 + + async def test_probe_ok_logs_version(self, service: RomConvertoService, mocker): + log_info = mocker.patch("adapters.services.rom_converto.log.info") + proc = FakeProc(stdout=b'{"version": "0.21.0"}') + with ( + patch.object(rom_converto, "ROM_CONVERTO_ENABLED", True), + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", return_value=proc), + ): + assert await service.is_enabled() is True + assert log_info.call_count == 1 + + async def test_missing_binary_not_cached(self, service: RomConvertoService): + with ( + patch.object(rom_converto, "ROM_CONVERTO_ENABLED", True), + patch("shutil.which", return_value=None), + ): + assert await service.is_enabled() is False + + proc = FakeProc(stdout=b'{"version": "0.21.0"}') + with ( + patch.object(rom_converto, "ROM_CONVERTO_ENABLED", True), + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", return_value=proc), + ): + assert await service.is_enabled() is True + + +def _capture_spawn(proc: FakeProc, calls: list[tuple[str, ...]]): + async def spawn(*args, **kwargs): + calls.append(args) + return proc + + return spawn + + +class TestUpdateCheck: + async def test_every_run_disables_the_update_check(self): + calls: list[tuple[str, ...]] = [] + with ( + patch("shutil.which", return_value="/usr/bin/rom-converto"), + patch("asyncio.create_subprocess_exec", _capture_spawn(FakeProc(), calls)), + ): + await rom_converto._run(["capabilities"], timeout_seconds=1) + + assert calls[0][:3] == ( + "/usr/bin/rom-converto", + "--no-update-check", + "capabilities", + ) + + +class TestConvert: + def _op(self) -> Operation: + return Operation( + target="chd", + platforms=frozenset({"psp"}), + argv=("chd", "compress"), + input_exts=frozenset({".iso"}), + output_ext=".chd", + ) + + async def test_argv_is_operation_argv_plus_src_and_out( + self, service: RomConvertoService, tmp_path: Path + ): + recorded: list[list[str]] = [] + + src = tmp_path / "game.iso" + out = tmp_path / "stage" / "game.chd" + out.parent.mkdir() + + async def fake_run(argv: list[str], timeout_seconds: float): + recorded.append(argv) + out.write_bytes(b"converted") + return 0, "", "" + + with patch.object(rom_converto, "_run", fake_run): + await service.convert(self._op(), src, out) + + assert recorded == [["chd", "compress", str(src), str(out)]] + + async def test_an_owner_only_output_is_made_readable_for_nginx( + self, service: RomConvertoService, tmp_path: Path + ): + out = tmp_path / "game.chd" + + async def fake_run(argv: list[str], timeout_seconds: float): + out.write_bytes(b"converted") + out.chmod(0o600) + return 0, "", "" + + with patch.object(rom_converto, "_run", fake_run): + await service.convert(self._op(), tmp_path / "src" / "game.iso", out) + + assert stat.S_IMODE(out.stat().st_mode) == SERVED_FILE_MODE + + @pytest.mark.parametrize( + "written", + [ + pytest.param(["game.chd", "game.bin"], id="split"), + pytest.param([], id="nothing"), + ], + ) + async def test_output_other_than_out_raises_operation_error( + self, service: RomConvertoService, tmp_path: Path, written: list[str] + ): + async def fake_run(argv: list[str], timeout_seconds: float): + for name in written: + (tmp_path / name).write_bytes(b"x") + return 0, "", "" + + with ( + patch.object(rom_converto, "_run", fake_run), + pytest.raises(RomConvertoOperationError, match="did not write exactly"), + ): + await service.convert(self._op(), tmp_path / "x.iso", tmp_path / "game.chd") + + async def test_nonzero_raises_operation_error_with_diagnostic( + self, service: RomConvertoService, tmp_path: Path + ): + async def fake_run(argv: list[str], timeout_seconds: float): + return 1, "", "error: bad disc key" + + src = tmp_path / "game.iso" + out = tmp_path / "game.chd" + with ( + patch.object(rom_converto, "_run", fake_run), + pytest.raises(RomConvertoOperationError) as exc_info, + ): + await service.convert(self._op(), src, out) + + assert "bad disc key" in str(exc_info.value) + + +class TestFileFormat: + @pytest.mark.parametrize( + ("file_name", "expected"), + [ + pytest.param("Game.ISO", "iso", id="target-extension"), + pytest.param("game.zcia", "z3ds", id="compressed-3ds"), + pytest.param("game.3ds", "cci", id="alias"), + pytest.param("game.nkit.iso", "nkit-iso", id="compound-extension"), + pytest.param("game.pbp", "pbp", id="input-only-extension"), + pytest.param("game.zip", "zip", id="unknown-extension"), + pytest.param("README", "", id="no-extension"), + ], + ) + def test_names_the_format(self, file_name: str, expected: str): + assert file_format(file_name) == expected + + @pytest.mark.parametrize( + ("name", "expected"), + [ + pytest.param("iso", "iso", id="plain"), + pytest.param("3ds", "cci", id="alias"), + pytest.param("zcia", "z3ds", id="compressed-alias"), + pytest.param("nkit-iso", "nkit-iso", id="compound-not-iso"), + ], + ) + def test_canonical_format_reads_aliases(self, name: str, expected: str): + assert canonical_format(name) == expected + + +class TestResolveOperation: + @pytest.mark.parametrize( + ("platform", "target", "file_name", "argv", "input_ext", "output_name"), + [ + ( + "ngc", + "rvz", + "Game.nkit.iso", + ("dol", "migrate"), + ".nkit.iso", + "Game.rvz", + ), + ("ngc", "rvz", "Game.iso", ("dol", "compress"), ".iso", "Game.rvz"), + ("wii", "wbfs", "Game.rvz", ("rvl", "decompress"), ".rvz", "Game.wbfs"), + ("3ds", "z3ds", "Game.3ds", ("ctr", "compress"), ".3ds", "Game.zcci"), + ("3ds", "cci", "My Game.CIA", ("ctr", "convert"), ".cia", "My Game.cci"), + ("psp", "chd", "Game.ISO", ("chd", "compress"), ".iso", "Game.chd"), + ], + ) + def test_resolves_operation_and_output_name( + self, + platform: str, + target: str, + file_name: str, + argv: tuple[str, ...], + input_ext: str, + output_name: str, + ): + resolved = resolve_operation(platform, target, file_name) + + assert resolved is not None + op, ext = resolved + assert (op.argv, ext) == (argv, input_ext) + assert op.output_name(Path(file_name), ext) == output_name + + @pytest.mark.parametrize( + ("platform", "target", "file_name"), + [ + pytest.param("psp", "iso", "Game.iso", id="already-in-target-format"), + pytest.param("psx", "iso", "Game.chd", id="cd-chd-has-no-extract"), + pytest.param("psx", "iso", "Game.cue", id="psx-cue-can-have-audio"), + pytest.param("saturn", "iso", "Game.cue", id="saturn-cue-can-have-audio"), + pytest.param("segacd", "iso", "Game.cue", id="segacd-cue-can-have-audio"), + pytest.param("dc", "iso", "Game.cue", id="dc-cue-can-have-audio"), + ], + ) + def test_returns_none_when_nothing_applies( + self, platform: str, target: str, file_name: str + ): + assert resolve_operation(platform, target, file_name) is None + + @pytest.mark.parametrize( + "platform", + [ + pytest.param("psp", id="psp"), + pytest.param("ps2", id="ps2"), + ], + ) + def test_dvd_cue_still_converts_to_iso(self, platform: str): + resolved = resolve_operation(platform, "iso", "Game.cue") + + assert resolved is not None + op, ext = resolved + assert op.argv == ("cue", "to-iso") + assert op.output_name(Path("Game.cue"), ext) == "Game.iso" + + +class TestLibraryTargetsByPlatform: + def test_psp_targets(self): + assert LIBRARY_TARGETS_BY_PLATFORM["psp"] == {"chd", "cso", "iso", "zso"} + + def test_platforms_without_format_conversions(self): + for slug in ("wiiu", "psvita", "nds", "ps3"): + assert slug not in LIBRARY_TARGETS_BY_PLATFORM + + def test_a_lossy_conversion_is_no_library_target(self): + assert "xbox" not in LIBRARY_TARGETS_BY_PLATFORM + assert resolve_operation("xbox", "xiso", "game.iso") is not None + assert resolve_operation("xbox", "xiso", "game.iso", lossless=True) is None + + +class TestDownloadFormats: + def test_maps_each_input_extension_to_its_targets(self): + formats = DOWNLOAD_FORMATS + + assert formats["psp"][".chd"] == ["cso", "iso", "zso"] + assert formats["xbox"][".iso"] == ["xiso"] + + def test_no_decrypt_or_encrypt_targets(self): + targets = { + target + for exts in DOWNLOAD_FORMATS.values() + for ext_targets in exts.values() + for target in ext_targets + } + assert not targets & {"decrypted", "encrypted"} diff --git a/backend/tests/config/test_config_loader.py b/backend/tests/config/test_config_loader.py index 6338b5fac..4e43f6d1e 100644 --- a/backend/tests/config/test_config_loader.py +++ b/backend/tests/config/test_config_loader.py @@ -11,6 +11,7 @@ from config.config_manager import ( DEFAULT_EXCLUDED_MULTI_FILE_DIRS, DEFAULT_EXCLUDED_PLATFORM_DIRS, ConfigManager, + ConvertoConfig, parse_firmware_template, parse_platform_templates, parse_structure_template, @@ -188,6 +189,9 @@ def test_empty_config_loader(): assert loader.config.SCAN_REGION_MODE == "prefer_rom_tags" assert loader.config.GAMELIST_MEDIA_THUMBNAIL == "box2d" assert loader.config.GAMELIST_MEDIA_IMAGE == "screenshot" + assert not loader.config.CONVERTO.download_conversion_enabled + assert loader.config.CONVERTO.cache_ttl_hours == 24 + assert loader.config.CONVERTO.platform_formats == {} assert loader.config.STRUCTURE_TEMPLATES == {} @@ -677,6 +681,72 @@ def test_reparenting_a_folder_replaces_the_existing_version(tmp_path): assert reloaded.config.PLATFORMS_VERSIONS == {"naomi": "dc"} +def test_converto_config_from_yaml(tmp_path): + config_file = tmp_path / "config.yml" + config_file.write_text( + "converto:\n" + " download_conversion_enabled: true\n" + " cache_ttl_hours: 48\n" + " cache_max_size_gb: 0\n" + " platform_formats:\n" + " PSP: iso\n" + " ngc: rvz\n" + ) + loader = ConfigManager(str(config_file)) + + # Slugs and targets are normalized to lowercase. + assert loader.config.CONVERTO == ConvertoConfig( + download_conversion_enabled=True, + cache_ttl_hours=48, + cache_max_size_gb=0, + platform_formats={"psp": "iso", "ngc": "rvz"}, + ) + + +def test_null_converto_platform_formats_means_empty(tmp_path): + config_file = tmp_path / "config.yml" + config_file.write_text("converto:\n platform_formats:\n") + + assert ConfigManager(str(config_file)).config.CONVERTO.platform_formats == {} + + +def test_update_converto_settings_round_trip(tmp_path): + config_file = tmp_path / "config.yml" + config_file.write_text("converto:\n cache_ttl_hours: 12\n") + loader = ConfigManager(str(config_file)) + converto = ConvertoConfig( + download_conversion_enabled=True, + cache_ttl_hours=72, + cache_max_size_gb=50, + platform_formats={"psp": "iso", "ngc": "rvz"}, + ) + + loader.update_converto_settings(converto) + + assert ConfigManager(str(config_file)).config.CONVERTO == converto + + +@pytest.mark.parametrize( + "block", + [ + " platform_formats:\n psvita: iso\n", + " platform_formats:\n psp: rvz\n", + " platform_formats:\n xbox: xiso\n", + " cache_ttl_hours: 0\n", + " cache_max_size_gb: -1\n", + " cache_max_size_gb: true\n", + ], +) +def test_invalid_converto_setting_exits(tmp_path, block): + config_file = tmp_path / "config.yml" + config_file.write_text(f"converto:\n{block}") + + with pytest.raises(SystemExit) as excinfo: + ConfigManager(str(config_file)) + + assert excinfo.value.code == 3 + + def _write_emulatorjs_config(tmp_path: Path, emulatorjs_block: str) -> ConfigManager: config_file = tmp_path / "config.yml" config_file.write_text(f"emulatorjs:\n{emulatorjs_block}") diff --git a/backend/tests/endpoints/roms/test_converted_download.py b/backend/tests/endpoints/roms/test_converted_download.py new file mode 100644 index 000000000..4506fbd2d --- /dev/null +++ b/backend/tests/endpoints/roms/test_converted_download.py @@ -0,0 +1,311 @@ +import asyncio +import os +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from urllib.parse import unquote + +import pytest +from fastapi import status +from fastapi.testclient import TestClient + +from adapters.services.rom_converto import resolve_operation +from config.config_manager import ConvertoConfig +from models.rom import Rom +from utils import conversion_cache +from utils.conversion_cache import FAILED_FILE, RETRY_AFTER_SECONDS + + +@pytest.fixture +def conversion(tmp_path: Path, mocker): + """Enable download conversion with every rom-converto step mocked. + + Any listed target resolves to a CHD under a temporary cache, and a + conversion writes it there. + """ + converto = ConvertoConfig(download_conversion_enabled=True) + mocker.patch( + "utils.conversion_cache.cm.get_config", + return_value=SimpleNamespace(CONVERTO=converto), + ) + mocker.patch( + "utils.conversion_cache.rom_converto_service.is_enabled", + AsyncMock(return_value=True), + ) + final = tmp_path / "cache/converts/1-abc/test_rom.chd" + final.parent.mkdir(parents=True) + resolved = resolve_operation("psp", "chd", "test_rom.iso") + assert resolved is not None + lookup = mocker.patch.object( + conversion_cache, "_lookup", return_value=(resolved[0], final) + ) + mocker.patch.object(conversion_cache, "ROMM_BASE_PATH", str(tmp_path)) + mocker.patch.object( + conversion_cache, "ROM_CONVERTO_CACHE_PATH", str(final.parent.parent) + ) + + async def convert(*args, **kwargs) -> Path: + final.write_bytes(b"x" * 120) + return final + + return SimpleNamespace( + converto=converto, + final=final, + lookup=lookup, + convert=mocker.patch.object( + conversion_cache, "get_or_convert", AsyncMock(side_effect=convert) + ), + ) + + +@pytest.fixture +def cached_copy(conversion) -> Path: + final: Path = conversion.final + final.write_bytes(b"x" * 120) + served = time.time() - 3600 + os.utime(final, (served, served)) + return final + + +def _redirect(response) -> str: + return unquote(response.headers["X-Accel-Redirect"]) + + +def _download(client: TestClient, rom: Rom, *, token: str | None, **params): + return client.get( + f"/api/roms/{rom.id}/content/test_rom.zip", + headers={"Authorization": f"Bearer {token}"} if token else {}, + params=params, + follow_redirects=False, + ) + + +def _as_kiosk(mocker) -> None: + mocker.patch("handler.auth.hybrid_auth.KIOSK_MODE", True) + mocker.patch("handler.auth.permissions.KIOSK_MODE", True) + + +def test_download_is_untouched_without_a_format( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + response = _download(client, rom, token=access_token) + + assert response.status_code == status.HTTP_200_OK + assert _redirect(response) == f"/library/{rom_file.full_path}" + conversion.lookup.assert_not_called() + conversion.convert.assert_not_called() + + +@pytest.mark.parametrize("formats", ["zip", "chd,zip", " ZIP "]) +def test_a_listed_stored_format_serves_the_stored_file( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion, formats +): + response = _download(client, rom, token=access_token, format=formats) + + assert response.status_code == status.HTTP_200_OK + assert _redirect(response) == f"/library/{rom_file.full_path}" + conversion.convert.assert_not_called() + + +def test_a_listed_stored_format_is_served_with_conversion_disabled( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + conversion.converto.download_conversion_enabled = False + + response = _download(client, rom, token=access_token, format="zip") + + assert _redirect(response) == f"/library/{rom_file.full_path}" + + +def test_a_cached_copy_is_served_without_converting( + client: TestClient, access_token: str, rom: Rom, rom_file, cached_copy, conversion +): + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_200_OK + assert _redirect(response) == "/cache/converts/1-abc/test_rom.chd" + assert "test_rom.chd" in response.headers["Content-Disposition"] + conversion.convert.assert_not_called() + + +def test_a_quick_conversion_is_served_by_the_same_request( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_200_OK + assert _redirect(response) == "/cache/converts/1-abc/test_rom.chd" + conversion.convert.assert_awaited_once() + + +def test_a_slow_conversion_answers_202_and_keeps_running( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion, mocker +): + mocker.patch.object(conversion_cache, "CONVERSION_WAIT_SECONDS", 0.01) + + final: Path = conversion.final + + async def slow_convert(*args, **kwargs) -> Path: + await asyncio.sleep(0.2) + return final + + conversion.convert.side_effect = slow_convert + + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_202_ACCEPTED + assert response.headers["Retry-After"] == str(RETRY_AFTER_SECONDS) + assert "X-Accel-Redirect" not in response.headers + conversion.convert.assert_called_once() + + +def test_a_running_conversion_answers_202_without_starting_another( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion, mocker +): + mocker.patch.object(conversion_cache, "_is_converting", return_value=True) + + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_202_ACCEPTED + conversion.convert.assert_not_called() + + +def test_head_reports_202_or_the_cached_copy_but_never_converts( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + headers = {"Authorization": f"Bearer {access_token}"} + url = f"/api/roms/{rom.id}/content/test_rom.zip" + + uncached = client.head(url, headers=headers, params={"format": "chd"}) + conversion.final.write_bytes(b"x") + cached = client.head(url, headers=headers, params={"format": "chd"}) + + assert uncached.status_code == status.HTTP_202_ACCEPTED + assert _redirect(cached) == "/cache/converts/1-abc/test_rom.chd" + conversion.convert.assert_not_called() + + +def test_a_kiosk_visitor_never_starts_a_conversion( + client: TestClient, rom: Rom, rom_file, conversion, mocker +): + _as_kiosk(mocker) + + response = _download(client, rom, token=None, format="chd") + + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + conversion.convert.assert_not_called() + + +def test_a_kiosk_visitor_still_gets_a_cached_copy( + client: TestClient, rom: Rom, rom_file, cached_copy, conversion, mocker +): + _as_kiosk(mocker) + + response = _download(client, rom, token=None, format="chd") + + assert _redirect(response) == "/cache/converts/1-abc/test_rom.chd" + + +def test_no_producible_format_answers_406( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + conversion.lookup.return_value = None + + response = _download(client, rom, token=access_token, format="rvz,wbfs") + + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + assert "rvz, wbfs" in response.json()["detail"] + conversion.convert.assert_not_called() + + +def test_disabled_conversion_answers_406_for_another_format( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + conversion.converto.download_conversion_enabled = False + + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + conversion.lookup.assert_not_called() + + +def test_a_failed_conversion_is_not_retried( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + (conversion.final.parent / FAILED_FILE).touch() + + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + conversion.convert.assert_not_called() + + +def test_a_conversion_that_fails_answers_406( + client: TestClient, access_token: str, rom: Rom, rom_file, conversion +): + conversion.convert.side_effect = None + conversion.convert.return_value = None + + response = _download(client, rom, token=access_token, format="chd") + + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + + +def test_a_format_needs_a_single_file( + client: TestClient, access_token: str, multi_file_rom: Rom, conversion +): + response = client.head( + f"/api/roms/{multi_file_rom.id}/content/test_multi_file_rom.zip", + headers={"Authorization": f"Bearer {access_token}"}, + params={"format": "chd"}, + ) + + assert response.status_code == status.HTTP_406_NOT_ACCEPTABLE + + +@pytest.mark.parametrize( + ("method", "visitor"), + [ + pytest.param("get", "authenticated", id="authenticated-download"), + pytest.param("get", "kiosk", id="kiosk-download"), + pytest.param("get", "public", id="download-auth-disabled"), + pytest.param("head", "authenticated", id="metadata"), + ], +) +def test_get_refreshes_a_cached_copy_but_head_does_not( + client: TestClient, + access_token: str, + rom: Rom, + rom_file, + conversion, + cached_copy: Path, + mocker, + method, + visitor, +): + headers = {"Authorization": f"Bearer {access_token}"} + if visitor == "kiosk": + _as_kiosk(mocker) + headers = {} + elif visitor == "public": + # Download scopes are fixed when the route decorators are evaluated. + mocker.patch("decorators.auth.has_required_scope", return_value=True) + headers = {} + conversion.converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + + response = client.request( + method, + f"/api/roms/{rom.id}/content/test_rom.zip", + headers=headers, + params={"format": "chd"}, + follow_redirects=False, + ) + + assert response.status_code == status.HTTP_200_OK + assert _redirect(response) == "/cache/converts/1-abc/test_rom.chd" + assert (cached_copy.stat().st_mtime > time.time() - 60) is (method == "get") + assert conversion_cache.cleanup_stale_conversions() == (0 if method == "get" else 1) + assert cached_copy.exists() is (method == "get") + conversion.convert.assert_not_called() diff --git a/backend/tests/endpoints/test_config.py b/backend/tests/endpoints/test_config.py index 8ea1ddf27..ef4415e5b 100644 --- a/backend/tests/endpoints/test_config.py +++ b/backend/tests/endpoints/test_config.py @@ -1,5 +1,7 @@ +from types import SimpleNamespace from unittest.mock import AsyncMock, patch +import pytest from fastapi import status from config.config_manager import ( @@ -7,6 +9,7 @@ from config.config_manager import ( DEFAULT_EXCLUDED_FILES, DEFAULT_EXCLUDED_MULTI_FILE_DIRS, DEFAULT_EXCLUDED_PLATFORM_DIRS, + ConvertoConfig, ExclusionType, ) from config.config_manager import config_manager as cm @@ -261,3 +264,66 @@ def test_update_scan_settings_normalizes_codes(client, access_token: str): _, kwargs = update_scan_settings.call_args assert kwargs["region_priority"] == ["us", "eu"] assert kwargs["language_priority"] == ["en"] + + +def _converto_payload(**overrides): + payload = { + "download_conversion_enabled": True, + "cache_max_size_gb": 20, + "platform_formats": {"psp": "cso"}, + } + payload.update(overrides) + return payload + + +def test_update_converto_settings_normalizes_formats_and_keeps_the_ttl( + client, access_token: str +): + current = SimpleNamespace(CONVERTO=ConvertoConfig(cache_ttl_hours=72)) + with ( + patch.object(cm, "get_config", return_value=current), + patch.object(cm, "update_converto_settings") as update_converto_settings, + ): + response = client.put( + "/api/config/converto_settings", + headers={"Authorization": f"Bearer {access_token}"}, + json=_converto_payload(platform_formats={" PSP ": "CSO"}), + ) + + assert response.status_code == status.HTTP_200_OK + update_converto_settings.assert_called_once_with( + ConvertoConfig( + download_conversion_enabled=True, + cache_ttl_hours=72, + cache_max_size_gb=20, + platform_formats={"psp": "cso"}, + ) + ) + + +def test_update_converto_settings_requires_auth(client): + response = client.put("/api/config/converto_settings", json=_converto_payload()) + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +@pytest.mark.parametrize( + "overrides", + [ + {"platform_formats": {"psp": "rvz"}}, + {"platform_formats": {"psvita": "iso"}}, + {"platform_formats": {"xbox": "xiso"}}, + {"cache_max_size_gb": -1}, + ], +) +def test_update_converto_settings_rejects_invalid_values( + client, access_token: str, overrides +): + with patch.object(cm, "update_converto_settings") as update_converto_settings: + response = client.put( + "/api/config/converto_settings", + headers={"Authorization": f"Bearer {access_token}"}, + json=_converto_payload(**overrides), + ) + + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + update_converto_settings.assert_not_called() diff --git a/backend/tests/tasks/test_cleanup_conversion_cache.py b/backend/tests/tasks/test_cleanup_conversion_cache.py new file mode 100644 index 000000000..e2631a1b9 --- /dev/null +++ b/backend/tests/tasks/test_cleanup_conversion_cache.py @@ -0,0 +1,28 @@ +from dataclasses import replace + +from tasks.registry import CLEANUP_CONVERSION_CACHE_SPEC +from tasks.scheduled.cleanup_conversion_cache import CleanupConversionCacheTask + + +class TestCleanupConversionCacheTask: + def test_configuration(self): + assert CLEANUP_CONVERSION_CACHE_SPEC.enabled is True + assert CLEANUP_CONVERSION_CACHE_SPEC.cron_string == "0 4 * * *" + + async def test_run_calls_cleanup(self, mocker): + task = CleanupConversionCacheTask() + mock_cleanup = mocker.patch( + "tasks.scheduled.cleanup_conversion_cache.cleanup_stale_conversions", + return_value=2, + ) + await task.run() + mock_cleanup.assert_called_once_with() + + async def test_run_disabled_skips_the_cleanup(self, mocker): + task = CleanupConversionCacheTask() + task.spec = replace(task.spec, enabled=False) + mock_cleanup = mocker.patch( + "tasks.scheduled.cleanup_conversion_cache.cleanup_stale_conversions", + ) + await task.run() + mock_cleanup.assert_not_called() diff --git a/backend/tests/tasks/test_convert_library.py b/backend/tests/tasks/test_convert_library.py new file mode 100644 index 000000000..9d3e3689e --- /dev/null +++ b/backend/tests/tasks/test_convert_library.py @@ -0,0 +1,400 @@ +import os +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy.exc import SQLAlchemyError +from tests.factories import make_rom + +from adapters.services.rom_converto import ( + RomConvertoOperationError, + rom_converto_service, +) +from config.config_manager import ConvertoConfig +from config.config_manager import config_manager as cm +from handler.database import db_platform_handler, db_rom_handler +from handler.filesystem import fs_rom_handler +from models.platform import Platform +from models.rom import Rom, RomFile, RomFileCategory +from models.user import User +from tasks.manual.convert_library import ConvertLibraryTask + + +@pytest.fixture +def library(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + lib = tmp_path / "library" + lib.mkdir() + monkeypatch.setattr(fs_rom_handler, "base_path", lib.resolve()) + return lib + + +@pytest.fixture +def converto(mocker) -> ConvertoConfig: + config = ConvertoConfig(platform_formats={"psp": "chd", "psx": "chd"}) + # The file refresh after a conversion reads the rest of the same config. + real = cm.get_config() + mocker.patch.object(real, "CONVERTO", config) + mocker.patch.object(cm, "get_config", return_value=real) + return config + + +@pytest.fixture +def converted(mocker) -> list[tuple[str, Path]]: + """Fake rom-converto: each conversion writes a small output and is recorded.""" + calls: list[tuple[str, Path]] = [] + + def convert(operation, src: Path, out: Path) -> None: + calls.append((src.name, out)) + out.write_bytes(b"c") + + mocker.patch.object( + rom_converto_service, "is_enabled", AsyncMock(return_value=True) + ) + mocker.patch.object(rom_converto_service, "convert", AsyncMock(side_effect=convert)) + return calls + + +def _platform(slug: str) -> Platform: + return db_platform_handler.add_platform( + Platform(name=slug, slug=slug, fs_slug=slug) + ) + + +def _rom( + library: Path, + admin_user: User, + platform: Platform, + files: dict[str, bytes], + *, + folder: bool = False, + matched: bool = True, +) -> Rom: + """A rom with `files` on disk: a single file, or a folder holding them all.""" + name = "game" + rom = make_rom( + platform, + name, + fs_extension="" if folder else Path(next(iter(files))).suffix.lstrip("."), + igdb_id=1 if matched else None, + ) + db_rom_handler.add_rom_user(rom_id=rom.id, user_id=admin_user.id) + file_path = f"{rom.fs_path}/{rom.fs_name}" if folder else rom.fs_path + for file_name, data in files.items(): + disk = library / file_path / file_name + disk.parent.mkdir(parents=True, exist_ok=True) + disk.write_bytes(data) + db_rom_handler.add_rom_file( + RomFile( + rom_id=rom.id, + file_name=file_name, + file_path=file_path, + file_size_bytes=len(data), + last_modified=disk.stat().st_mtime, + category=RomFileCategory.GAME, + ) + ) + refreshed = db_rom_handler.get_rom(rom.id) + assert refreshed is not None + return refreshed + + +def _reload(rom: Rom) -> Rom: + refreshed = db_rom_handler.get_rom(rom.id) + assert refreshed is not None + return refreshed + + +async def _run(platform: Platform | None = None) -> dict[str, int]: + return await ConvertLibraryTask().run(platform.id if platform else None) + + +async def test_a_single_file_rom_is_replaced_and_keeps_its_id( + library: Path, admin_user: User, converto, converted +): + psp = _platform("psp") + rom = _rom(library, admin_user, psp, {"game.iso": b"x" * 100}) + folder = library / rom.fs_path + + stats = await _run() + + assert stats["converted"] == 1 + assert stats["bytes_saved"] == 99 + assert sorted(p.name for p in folder.iterdir()) == ["game.chd"] + after = _reload(rom) + assert after.fs_name == "game.chd" + assert after.fs_extension == "chd" + assert [f.file_name for f in after.files] == ["game.chd"] + + +async def test_a_cue_folder_becomes_one_chd_and_its_playlist_follows( + library: Path, admin_user: User, converto, converted +): + psx = _platform("psx") + cue = 'FILE "game (Track 1).bin" BINARY\nFILE "game (Track 2).bin" BINARY\n' + rom = _rom( + library, + admin_user, + psx, + { + "game.cue": cue.encode(), + "game (Track 1).bin": b"x" * 50, + "game (Track 2).bin": b"x" * 50, + "game.m3u": b"game.cue\n", + }, + folder=True, + ) + folder = library / rom.full_path + + stats = await _run() + + assert stats["converted"] == 1 + assert sorted(p.name for p in folder.iterdir()) == ["game.chd", "game.m3u"] + assert (folder / "game.m3u").read_text() == "game.chd\n" + after = _reload(rom) + assert after.fs_name == rom.fs_name + assert sorted(f.file_name for f in after.files) == ["game.chd", "game.m3u"] + + +async def test_a_bom_cue_and_playlist_still_follow_the_conversion( + library: Path, admin_user: User, converto, converted +): + bom = "\ufeff".encode() + rom = _rom( + library, + admin_user, + _platform("psx"), + { + "game.cue": bom + b'FILE "game.bin" BINARY\n', + "game.bin": b"x" * 50, + "game.m3u": bom + b"./game.cue\n", + }, + folder=True, + ) + folder = library / rom.full_path + + await _run() + + assert sorted(p.name for p in folder.iterdir()) == ["game.chd", "game.m3u"] + assert (folder / "game.m3u").read_text() == "game.chd\n" + # Staged beside the rom folder, whose dot-prefixed subfolders count as its files. + assert converted[0][1].parent.parent == library / rom.fs_path + + +async def test_one_roms_failure_does_not_stop_the_run( + library: Path, admin_user: User, converto, converted, mocker +): + _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + _rom(library, admin_user, _platform("psx"), {"game.cue": b""}, folder=True) + mocker.patch( + "tasks.manual.convert_library.refresh_rom_files", + AsyncMock(side_effect=[RuntimeError("lock timeout"), None]), + ) + + stats = await _run() + + assert stats["failed"] == 1 + assert sorted(src for src, _ in converted) == ["game.cue", "game.iso"] + + +async def test_a_file_already_in_the_library_format_is_left_alone( + library: Path, admin_user: User, converto, converted +): + _rom(library, admin_user, _platform("psp"), {"game.chd": b"x"}) + + stats = await _run() + + assert stats["already_converted"] == 1 + assert converted == [] + + +async def test_an_unmatched_rom_is_not_converted( + library: Path, admin_user: User, converto, converted +): + rom = _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}, matched=False) + + stats = await _run() + + assert stats["unmatched"] == 1 + assert converted == [] + assert (library / rom.full_path).exists() + + +async def test_a_platform_without_a_library_format_is_skipped( + library: Path, admin_user: User, converto, converted +): + converto.platform_formats = {} + _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + + stats = await _run() + + assert stats["converted"] == 0 + assert converted == [] + + +async def test_only_the_requested_platform_is_converted( + library: Path, admin_user: User, converto, converted +): + psp = _platform("psp") + _rom(library, admin_user, psp, {"game.iso": b"x" * 10}) + _rom(library, admin_user, _platform("psx"), {"game.cue": b""}, folder=True) + + await _run(psp) + + assert [src for src, _ in converted] == ["game.iso"] + + +async def test_a_failed_conversion_leaves_the_original_and_no_staged_output( + library: Path, admin_user: User, converto, converted, mocker +): + rom = _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + mocker.patch.object( + rom_converto_service, + "convert", + AsyncMock(side_effect=RomConvertoOperationError("boom")), + ) + + stats = await _run() + + assert stats["failed"] == 1 + assert sorted(p.name for p in (library / rom.fs_path).iterdir()) == ["game.iso"] + assert _reload(rom).fs_name == "game.iso" + + +async def test_an_existing_output_is_never_overwritten( + library: Path, admin_user: User, converto, converted +): + rom = _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + (library / rom.fs_path / "game.chd").write_bytes(b"theirs") + + stats = await _run() + + assert stats["failed"] == 1 + assert converted == [] + assert (library / rom.fs_path / "game.chd").read_bytes() == b"theirs" + assert (library / rom.full_path).exists() + + +async def test_an_output_appearing_during_conversion_is_never_overwritten( + library: Path, admin_user: User, converto, converted, mocker +): + rom = _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + theirs = library / rom.fs_path / "game.chd" + + def convert(operation, src: Path, out: Path) -> None: + theirs.write_bytes(b"theirs") + out.write_bytes(b"c") + + mocker.patch.object(rom_converto_service, "convert", AsyncMock(side_effect=convert)) + + stats = await _run() + + assert stats["failed"] == 1 + assert theirs.read_bytes() == b"theirs" + assert (library / rom.full_path).exists() + + +async def test_a_failed_rom_update_keeps_the_original( + library: Path, admin_user: User, converto, converted, mocker +): + rom = _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + mocker.patch.object( + db_rom_handler, "update_rom", side_effect=SQLAlchemyError("conflict") + ) + + stats = await _run() + + assert stats["failed"] == 1 + assert sorted(p.name for p in (library / rom.fs_path).iterdir()) == ["game.iso"] + + +async def test_a_cue_track_held_by_another_rom_is_not_deleted( + library: Path, admin_user: User, converto, converted +): + psx = _platform("psx") + rom = _rom(library, admin_user, psx, {"game.cue": b'FILE "game.bin" BINARY\n'}) + (library / rom.fs_path / "game.bin").write_bytes(b"x") + make_rom(psx, "track", fs_stem="game", fs_extension="bin") + + stats = await _run(psx) + + assert stats["failed"] == 1 + assert converted == [] + assert (library / rom.fs_path / "game.bin").exists() + + +async def test_a_track_another_cue_shares_is_refused( + library: Path, admin_user: User, converto, converted +): + track = b'FILE "game.bin" BINARY\n' + rom = _rom( + library, + admin_user, + _platform("psx"), + {"disc 1.cue": track, "disc 2.cue": track, "game.bin": b"x"}, + folder=True, + ) + + stats = await _run() + + assert stats["failed"] == 2 + assert converted == [] + assert (library / rom.full_path / "game.bin").exists() + + +async def test_a_stage_left_by_a_killed_conversion_is_removed( + library: Path, admin_user: User, converto, converted +): + rom = _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + stale = library / rom.fs_path / ".romm_tmp_convert_old" + stale.mkdir() + os.utime(stale, (0, 0)) + + await _run() + + assert not stale.exists() + + +async def test_a_rom_row_already_holding_the_output_name_is_left_alone( + library: Path, admin_user: User, converto, converted +): + psp = _platform("psp") + rom = _rom(library, admin_user, psp, {"game.iso": b"x"}) + make_rom(psp, "other", fs_stem="game", fs_extension="chd") + + stats = await _run() + + assert stats["failed"] == 1 + assert converted == [] + assert (library / rom.full_path).exists() + + +async def test_a_cue_pointing_outside_its_folder_is_refused( + library: Path, admin_user: User, converto, converted +): + rom = _rom( + library, + admin_user, + _platform("psx"), + {"game.cue": b'FILE "../other.bin" BINARY\n'}, + folder=True, + ) + + stats = await _run() + + assert stats["failed"] == 1 + assert converted == [] + assert (library / rom.full_path / "game.cue").exists() + + +async def test_nothing_runs_without_rom_converto( + library: Path, admin_user: User, converto, converted, mocker +): + mocker.patch.object( + rom_converto_service, "is_enabled", AsyncMock(return_value=False) + ) + _rom(library, admin_user, _platform("psp"), {"game.iso": b"x"}) + + stats = await _run() + + assert stats["converted"] == 0 + assert converted == [] diff --git a/backend/tests/utils/test_conversion_cache.py b/backend/tests/utils/test_conversion_cache.py new file mode 100644 index 000000000..1f32172aa --- /dev/null +++ b/backend/tests/utils/test_conversion_cache.py @@ -0,0 +1,986 @@ +import asyncio +import fcntl +import os +import shutil +import time +from collections.abc import Iterator +from contextlib import ExitStack, contextmanager, nullcontext +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import AsyncMock + +import pytest + +from adapters.services.rom_converto import ( + RomConvertoBinaryNotFoundError, + RomConvertoOperationError, + RomConvertoTimeoutError, + resolve_operation, +) +from config import ROMM_BASE_PATH +from config.config_manager import ConvertoConfig +from models.rom import Rom, RomFile +from utils import conversion_cache +from utils.conversion_cache import ( + FAILED_FILE, + PARTIAL_DIR, + SERVE_GRACE_SECONDS, + FormatOutcome, + FormatResolution, + converted_file_path, + get_cached_converted, + get_or_convert, + get_redirect_path, + parse_formats, + resolve_format_download, +) + + +def _rom_file(**overrides: Any) -> RomFile: + defaults: dict[str, Any] = { + "rom_id": 1, + "file_name": "game.iso", + "file_path": "psp/roms", + "file_size_bytes": 1000, + "last_modified": 1700000000.0, + } + defaults.update(overrides) + return RomFile(**defaults) + + +def _final_path(rom_file: RomFile) -> Path: + resolved = resolve_operation("psp", "chd", rom_file.file_name) + assert resolved is not None + return converted_file_path(1, rom_file, *resolved) + + +def _partial_path(final: Path) -> Path: + return final.parent / PARTIAL_DIR / final.name + + +def _seed_cached(rom_file: RomFile, *, age_seconds: float = 0) -> Path: + """Write a converted copy for `rom_file`, last served `age_seconds` ago.""" + final = _final_path(rom_file) + final.parent.mkdir(parents=True, exist_ok=True) + final.write_bytes(b"cached") + served = time.time() - age_seconds + os.utime(final, (served, served)) + return final + + +@contextmanager +def _held_lock(key_dir: Path, *, shared: bool = False) -> Iterator[None]: + """Hold a key dir lock while a request or worker uses it.""" + fd = os.open(key_dir, os.O_RDONLY) + fcntl.flock(fd, fcntl.LOCK_SH if shared else fcntl.LOCK_EX) + try: + yield + finally: + os.close(fd) + + +@pytest.fixture(autouse=True) +def converto(mocker) -> ConvertoConfig: + """The converto.* settings every cache function reads, editable per test.""" + config = ConvertoConfig() + mocker.patch( + "utils.conversion_cache.cm.get_config", + return_value=SimpleNamespace(CONVERTO=config), + ) + return config + + +@pytest.fixture +def cache_root(tmp_path, mocker): + mocker.patch.object(conversion_cache, "ROM_CONVERTO_CACHE_PATH", str(tmp_path)) + return tmp_path + + +@pytest.fixture +def convert_outputs(mocker) -> list[Path]: + """Fake the adapter: write each requested output and record its path.""" + outputs: list[Path] = [] + + async def convert(operation, src, out) -> None: + outputs.append(out) + out.write_bytes(b"converted") + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + return outputs + + +class TestGetOrConvert: + async def test_unresolvable_returns_none_without_touching_disk( + self, cache_root, convert_outputs + ): + f = _rom_file(file_name="game.zip") + + assert await get_or_convert(1, f, "psp", "chd") is None + assert convert_outputs == [] + assert not any(cache_root.iterdir()) + + async def test_cache_hit_returns_path_and_refreshes_mtime( + self, cache_root, convert_outputs + ): + f = _rom_file() + final = _seed_cached(f, age_seconds=3600) + + assert await get_or_convert(1, f, "psp", "chd") == final + assert final.stat().st_mtime > time.time() - 60 + assert convert_outputs == [] + + async def test_oversized_conversion_returns_none_without_converting( + self, cache_root, converto, convert_outputs, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + + assert ( + await get_or_convert(1, _rom_file(file_size_bytes=101), "psp", "chd") + is None + ) + assert convert_outputs == [] + + async def test_admission_reserves_room_for_a_decompression( + self, cache_root, converto, convert_outputs, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + f = _rom_file(file_name="game.chd", file_size_bytes=30) + + assert await get_or_convert(1, f, "psp", "iso") is None + assert convert_outputs == [] + + async def test_admission_evicts_an_older_copy_to_make_room( + self, cache_root, converto, convert_outputs, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + older = _seed_cached(_rom_file(file_name="older.iso")) + older.write_bytes(b"x" * 40) + served = time.time() - 300 + os.utime(older, (served, served)) + f = _rom_file(file_size_bytes=70) + + assert await get_or_convert(1, f, "psp", "chd") == _final_path(f) + assert not older.parent.exists() + assert _final_path(f).read_bytes() == b"converted" + + async def test_admission_does_not_evict_a_recently_served_copy( + self, cache_root, converto, convert_outputs, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + recent = _seed_cached(_rom_file(file_name="recent.iso")) + recent.write_bytes(b"x" * 80) + f = _rom_file(file_size_bytes=50) + + assert await get_or_convert(1, f, "psp", "chd") is None + assert recent.read_bytes() == b"x" * 80 + assert convert_outputs == [] + + @pytest.mark.parametrize( + ("size_bytes", "blocked"), + [ + pytest.param(101, "none", id="larger-than-cap"), + pytest.param(70, "grace", id="insufficient-outside-grace"), + pytest.param(70, "exclusive", id="conversion-locked"), + pytest.param(70, "shared", id="download-locked"), + ], + ) + async def test_impossible_admission_preserves_other_copies( + self, cache_root, converto, convert_outputs, mocker, size_bytes, blocked + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + older = _seed_cached(_rom_file(file_name="older.iso")) + protected = _seed_cached(_rom_file(file_name="protected.iso")) + older.write_bytes(b"x" * 40) + protected.write_bytes(b"x" * 50) + now = time.time() + os.utime(older, (now - 300, now - 300)) + protected_age = 0 if blocked == "grace" else SERVE_GRACE_SECONDS * 2 + os.utime(protected, (now - protected_age, now - protected_age)) + holder = ( + _held_lock(protected.parent, shared=blocked == "shared") + if blocked in ("exclusive", "shared") + else nullcontext() + ) + + with holder: + assert ( + await get_or_convert( + 1, _rom_file(file_size_bytes=size_bytes), "psp", "chd" + ) + is None + ) + + assert older.read_bytes() == b"x" * 40 + assert protected.read_bytes() == b"x" * 50 + assert convert_outputs == [] + + async def test_conversion_evicts_the_least_recently_served_other_copy( + self, cache_root, converto, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + oldest = _seed_cached(_rom_file(file_name="old.iso"), age_seconds=300) + newer = _seed_cached(_rom_file(file_name="newer.iso"), age_seconds=200) + oldest.write_bytes(b"x" * 40) + newer.write_bytes(b"x" * 40) + now = time.time() + os.utime(oldest, (now - 300, now - 300)) + os.utime(newer, (now - 200, now - 200)) + f = _rom_file(file_size_bytes=10) + + async def convert(op, src, out) -> None: + out.write_bytes(b"converted" * 6) + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + + final = await get_or_convert(1, f, "psp", "chd") + + assert final == _final_path(f) + assert final is not None + assert final.read_bytes() == b"converted" * 6 + assert not oldest.parent.exists() + assert newer.exists() + assert conversion_cache.cache_size_bytes() <= 100 + + async def test_post_conversion_cleanup_failure_still_serves_the_copy( + self, cache_root, convert_outputs, mocker + ): + f = _rom_file() + final = _final_path(f) + cleanup = conversion_cache.cleanup_stale_conversions + + def fail_after_publication(*args, **kwargs): + if final.exists(): + raise RuntimeError("cleanup failed") + return cleanup(*args, **kwargs) + + mocker.patch.object( + conversion_cache, + "cleanup_stale_conversions", + side_effect=fail_after_publication, + ) + + assert await get_or_convert(1, f, "psp", "chd") == final + assert final.read_bytes() == b"converted" + + async def test_locked_key_dir_returns_none(self, cache_root, convert_outputs): + f = _rom_file() + key_dir = _final_path(f).parent + key_dir.mkdir(parents=True) + + with _held_lock(key_dir): + assert await get_or_convert(1, f, "psp", "chd") is None + assert convert_outputs == [] + + async def test_concurrent_request_returns_none_while_one_converts( + self, cache_root, mocker + ): + f = _rom_file() + started = asyncio.Event() + release = asyncio.Event() + + async def convert(op, src, out) -> None: + started.set() + await release.wait() + out.write_bytes(b"converted") + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + + first = asyncio.create_task(get_or_convert(1, f, "psp", "chd")) + await started.wait() + assert await get_or_convert(1, f, "psp", "chd") is None + release.set() + assert await first == _final_path(f) + + async def test_queued_conversion_checks_budget_after_a_slot_opens( + self, cache_root, converto, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + mocker.patch.object( + conversion_cache, "_convert_semaphore", asyncio.Semaphore(1) + ) + first_file = _rom_file(file_size_bytes=10) + second_file = _rom_file(file_name="second.iso", file_size_bytes=10) + second_dir = _final_path(second_file).parent + started = asyncio.Event() + release = asyncio.Event() + + async def convert(op, src, out) -> None: + started.set() + await release.wait() + out.write_bytes(b"x" * 100) + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + + first = asyncio.create_task(get_or_convert(1, first_file, "psp", "chd")) + await started.wait() + second = asyncio.create_task(get_or_convert(1, second_file, "psp", "chd")) + try: + while not second.done(): + if second_dir.exists(): + with conversion_cache._try_lock(second_dir) as locked: + if not locked: + break + await asyncio.sleep(0) + queued = not second.done() + finally: + release.set() + results = await asyncio.gather(first, second) + + assert queued + assert list(results) == [_final_path(first_file), None] + assert _final_path(first_file).read_bytes() == b"x" * 100 + assert not _final_path(second_file).exists() + + async def test_writes_in_partial_dir_then_renames_into_place( + self, cache_root, convert_outputs + ): + f = _rom_file() + final = _final_path(f) + + assert await get_or_convert(1, f, "psp", "chd") == final + assert convert_outputs == [_partial_path(final)] + assert final.read_bytes() == b"converted" + assert [p.name for p in final.parent.iterdir()] == [final.name] + + async def test_leading_dot_copy_survives_cleanup(self, cache_root, convert_outputs): + f = _rom_file(file_name=".hack - Infection (USA).iso") + final = _final_path(f) + + assert await get_or_convert(1, f, "psp", "chd") == final + assert final.name == ".hack - Infection (USA).chd" + assert conversion_cache.cleanup_stale_conversions() == 0 + assert final.read_bytes() == b"converted" + assert get_cached_converted(1, f, "psp", "chd") == final + + async def test_removes_all_a_crashed_runs_partial_outputs( + self, cache_root, convert_outputs + ): + f = _rom_file() + final = _final_path(f) + final.parent.mkdir(parents=True) + partial = _partial_path(final) + partial.parent.mkdir() + for path in ( + partial, + partial.with_name(f".{partial.name}.ABCDEF.tmp"), + partial.with_suffix(".bin"), + ): + path.write_bytes(b"leftover") + + assert await get_or_convert(1, f, "psp", "chd") == final + assert final.read_bytes() == b"converted" + assert list(final.parent.iterdir()) == [final] + + @pytest.mark.parametrize( + ("error", "writes_partial"), + [ + pytest.param(RomConvertoOperationError, False, id="nonzero-exit-no-output"), + pytest.param(RomConvertoOperationError, True, id="nonzero-exit-partials"), + pytest.param(RomConvertoTimeoutError, True, id="timeout-partials"), + ], + ) + async def test_failed_conversion_retries_only_after_marker_expires( + self, cache_root, converto, mocker, error, writes_partial + ): + f = _rom_file() + final = _final_path(f) + marker = final.parent / FAILED_FILE + + async def convert(op, src, out) -> None: + if writes_partial: + out.write_bytes(b"partial") + out.with_name(f".{out.name}.ABCDEF.tmp").write_bytes(b"scratch") + out.with_suffix(".bin").write_bytes(b"data") + raise error("boom") + + convert_mock = mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + + assert await get_or_convert(1, f, "psp", "chd") is None + assert list(final.parent.iterdir()) == [marker] + assert await get_or_convert(1, f, "psp", "chd") is None + convert_mock.assert_awaited_once() + expired = time.time() - converto.cache_ttl_hours * 3600 - 1 + os.utime(marker, (expired, expired)) + assert conversion_cache.cleanup_stale_conversions() == 1 + + async def succeed(op, src, out) -> None: + out.write_bytes(b"converted") + + convert_mock.side_effect = succeed + assert await get_or_convert(1, f, "psp", "chd") == final + assert final.read_bytes() == b"converted" + assert not marker.exists() + + @pytest.mark.parametrize( + "stage", + [ + pytest.param("mkdir", id="partial-dir-error"), + pytest.param("replace", id="publication-error"), + pytest.param("convert-os-error", id="subprocess-os-error"), + pytest.param("missing-binary", id="binary-not-found"), + ], + ) + async def test_transient_failure_retries_the_next_download( + self, cache_root, mocker, stage + ): + f = _rom_file() + final = _final_path(f) + failed = False + + async def convert(op, src, out) -> None: + nonlocal failed + if not failed and stage in ("convert-os-error", "missing-binary"): + failed = True + error = ( + RomConvertoBinaryNotFoundError + if stage == "missing-binary" + else OSError + ) + raise error("temporary failure") + out.write_bytes(b"converted") + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + if stage in ("mkdir", "replace"): + write = Path.mkdir if stage == "mkdir" else os.replace + + def fail_write_once(*args, **kwargs): + nonlocal failed + if not failed and (stage == "replace" or args[0].name == PARTIAL_DIR): + failed = True + raise OSError("temporary filesystem failure") + return write(*args, **kwargs) + + mocker.patch( + "pathlib.Path.mkdir" if stage == "mkdir" else "os.replace", + side_effect=fail_write_once, + autospec=True, + ) + + assert await get_or_convert(1, f, "psp", "chd") is None + assert not (final.parent / FAILED_FILE).exists() + assert not (final.parent / PARTIAL_DIR).exists() + assert await get_or_convert(1, f, "psp", "chd") == final + assert final.read_bytes() == b"converted" + + async def test_cancellation_removes_every_partial_output(self, cache_root, mocker): + f = _rom_file() + final = _final_path(f) + started = asyncio.Event() + + async def convert(op, src, out) -> None: + out.write_bytes(b"partial") + out.with_name(f".{out.name}.ABCDEF.tmp").write_bytes(b"scratch") + out.with_suffix(".bin").write_bytes(b"data") + started.set() + await asyncio.Event().wait() + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + + conversion = asyncio.create_task(get_or_convert(1, f, "psp", "chd")) + await started.wait() + conversion.cancel() + with pytest.raises(asyncio.CancelledError): + await conversion + + assert not final.exists() + assert list(final.parent.iterdir()) == [] + + +class TestGetCachedConverted: + @pytest.mark.parametrize("file_name", ["game.zip", "game.iso"]) + def test_returns_none_when_unresolvable_or_not_cached(self, cache_root, file_name): + f = _rom_file(file_name=file_name) + + assert get_cached_converted(1, f, "psp", "chd") is None + + @pytest.mark.parametrize("touch", [False, True]) + def test_returns_the_copy_refreshing_mtime_only_when_asked(self, cache_root, touch): + f = _rom_file() + final = _seed_cached(f, age_seconds=3600) + + assert get_cached_converted(1, f, "psp", "chd", touch=touch) == final + assert (final.stat().st_mtime > time.time() - 60) is touch + + @pytest.mark.parametrize( + "touch", + [ + pytest.param(False, id="lookup"), + pytest.param(True, id="download"), + ], + ) + def test_returns_none_while_key_dir_is_locked(self, cache_root, touch): + f = _rom_file() + final = _seed_cached(f) + + with _held_lock(final.parent): + assert get_cached_converted(1, f, "psp", "chd", touch=touch) is None + + @pytest.mark.parametrize( + "error", + [ + pytest.param(FileNotFoundError, id="touch-not-found"), + pytest.param(PermissionError, id="permission-denied"), + ], + ) + def test_touch_failure_still_serves_the_copy(self, cache_root, mocker, error): + f = _rom_file() + final = _seed_cached(f) + mocker.patch("os.utime", side_effect=error) + + assert get_cached_converted(1, f, "psp", "chd", touch=True) == final + assert final.read_bytes() == b"cached" + + def test_lock_open_failure_returns_none(self, cache_root, mocker): + f = _rom_file() + _seed_cached(f) + mocker.patch("os.open", side_effect=PermissionError) + + assert get_cached_converted(1, f, "psp", "chd") is None + + @pytest.mark.parametrize( + "recreate", + [ + pytest.param(False, id="removed"), + pytest.param(True, id="recreated"), + ], + ) + def test_does_not_serve_a_replaced_key_dir(self, cache_root, mocker, recreate): + f = _rom_file() + final = _seed_cached(f) + flock = fcntl.flock + + def replace_dir(fd, mode): + flock(fd, mode) + shutil.rmtree(final.parent) + if recreate: + final.parent.mkdir() + final.write_bytes(b"replacement") + + mocker.patch("fcntl.flock", side_effect=replace_dir) + + assert get_cached_converted(1, f, "psp", "chd") is None + + +class TestHasRoomFor: + def test_unbounded_when_cap_is_zero(self, cache_root, converto): + converto.cache_max_size_gb = 0 + + assert conversion_cache.has_room_for(10 * conversion_cache.BYTES_PER_GB) + + def test_counts_partial_output_against_the_cap(self, cache_root, converto, mocker): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + key_dir = cache_root / "1-a" + key_dir.mkdir() + (key_dir / "game.chd").write_bytes(b"x" * 40) + partial_dir = key_dir / PARTIAL_DIR + partial_dir.mkdir() + (partial_dir / "other.chd").write_bytes(b"x" * 20) + + assert conversion_cache.has_room_for(40) + assert not conversion_cache.has_room_for(41) + + @pytest.mark.parametrize( + "vanishes", + [ + pytest.param("file", id="vanishing-file"), + pytest.param("dir", id="vanishing-dir"), + ], + ) + def test_size_walk_tolerates_vanishing_entries(self, cache_root, mocker, vanishes): + vanished = _seed_cached(_rom_file(file_name="vanished.iso")) + survivor = _seed_cached(_rom_file(file_name="survivor.iso")) + stat = os.stat + scandir = os.scandir + + if vanishes == "file": + + def stat_after_removal(path, *args, **kwargs): + if path == str(vanished): + vanished.unlink(missing_ok=True) + return stat(path, *args, **kwargs) + + mocker.patch("os.stat", side_effect=stat_after_removal) + else: + + def scan_after_removal(path): + if path == str(vanished.parent): + shutil.rmtree(vanished.parent) + return scandir(path) + + mocker.patch("os.scandir", side_effect=scan_after_removal) + + assert conversion_cache.cache_size_bytes() == len(b"cached") + assert survivor.read_bytes() == b"cached" + + +class TestGetRedirectPath: + def test_relative_to_romm_base_path(self): + converted_path = Path(ROMM_BASE_PATH) / "cache/converts/1-abc/Game.chd" + assert get_redirect_path(converted_path) == Path( + "/cache/converts/1-abc/Game.chd" + ) + + +class TestCleanupStaleConversions: + def test_missing_root_is_noop(self, tmp_path, mocker): + mocker.patch.object( + conversion_cache, "ROM_CONVERTO_CACHE_PATH", str(tmp_path / "nope") + ) + assert conversion_cache.cleanup_stale_conversions() == 0 + + def test_deletes_expired_and_keeps_fresh(self, cache_root): + expired = _seed_cached(_rom_file(file_name="old.iso"), age_seconds=25 * 3600) + fresh = _seed_cached(_rom_file(file_name="new.iso")) + + assert conversion_cache.cleanup_stale_conversions() == 1 + assert not expired.parent.exists() + assert fresh.exists() + + def test_removes_partial_outputs_from_a_kept_dir(self, cache_root): + final = _seed_cached(_rom_file()) + partial = _partial_path(final) + partial.parent.mkdir() + for path in ( + partial, + partial.with_name(f".{partial.name}.ABCDEF.tmp"), + partial.with_suffix(".bin"), + ): + path.write_bytes(b"leftover") + + assert conversion_cache.cleanup_stale_conversions() == 0 + assert final.read_bytes() == b"cached" + assert list(final.parent.iterdir()) == [final] + + @pytest.mark.parametrize( + "leftover", + [ + pytest.param(False, id="empty"), + pytest.param(True, id="partial"), + ], + ) + def test_deletes_dirs_without_a_served_copy(self, cache_root, leftover): + key_dir = cache_root / "1-e" + key_dir.mkdir() + if leftover: + partial_dir = key_dir / PARTIAL_DIR + partial_dir.mkdir() + (partial_dir / "game.chd").write_bytes(b"partial") + + assert conversion_cache.cleanup_stale_conversions() == 1 + assert not key_dir.exists() + + def test_skips_a_dir_whose_conversion_is_running(self, cache_root): + key_dir = cache_root / "1-busy" + key_dir.mkdir() + partial_dir = key_dir / PARTIAL_DIR + partial_dir.mkdir() + (partial_dir / "game.chd").write_bytes(b"partial") + + with _held_lock(key_dir): + assert conversion_cache.cleanup_stale_conversions() == 0 + assert key_dir.exists() + + def test_kept_copy_remains_available_during_cleanup(self, cache_root, mocker): + f = _rom_file() + final = _seed_cached(f) + files = conversion_cache._files + + def serve_during_listing(key_dir): + listed = files(key_dir) + assert get_cached_converted(1, f, "psp", "chd") == final + return listed + + mocker.patch.object( + conversion_cache, "_files", side_effect=serve_during_listing + ) + + assert conversion_cache.cleanup_stale_conversions() == 0 + assert final.read_bytes() == b"cached" + + def test_evicts_least_recently_served_over_the_cap( + self, cache_root, converto, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + dirs = {} + for name, age in (("old", 300), ("mid", 200), ("new", 100)): + final = _seed_cached(_rom_file(file_name=f"{name}.iso")) + final.write_bytes(b"x" * 40) + os.utime(final, (time.time() - age, time.time() - age)) + dirs[name] = final.parent + + assert conversion_cache.cleanup_stale_conversions() == 1 + assert not dirs["old"].exists() + assert dirs["mid"].exists() + assert dirs["new"].exists() + + @pytest.mark.parametrize( + "reserve_bytes", + [ + pytest.param(0, id="post-conversion"), + pytest.param(70, id="admission"), + ], + ) + def test_size_eviction_preserves_failure_markers( + self, cache_root, converto, mocker, reserve_bytes + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + marker_dir = cache_root / "1-failed" + marker_dir.mkdir() + marker = marker_dir / FAILED_FILE + marker.touch() + served = time.time() - 3600 + os.utime(marker, (served, served)) + recent = _seed_cached(_rom_file()) + recent.write_bytes(b"x" * 120) + + assert conversion_cache.cleanup_stale_conversions(reserve_bytes) == 0 + assert marker.exists() + assert recent.read_bytes() == b"x" * 120 + + @pytest.mark.parametrize( + ("age_seconds", "kept"), + [ + pytest.param(SERVE_GRACE_SECONDS // 2, True, id="within-grace"), + pytest.param(SERVE_GRACE_SECONDS * 2, False, id="after-grace"), + ], + ) + def test_size_eviction_respects_serve_grace( + self, cache_root, converto, mocker, age_seconds, kept + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + final = _seed_cached(_rom_file()) + final.write_bytes(b"x" * 120) + served = time.time() - age_seconds + os.utime(final, (served, served)) + + assert conversion_cache.cleanup_stale_conversions() == (0 if kept else 1) + assert final.exists() is kept + + def test_size_eviction_rechecks_a_copy_served_after_the_ttl_pass( + self, cache_root, converto, mocker + ): + converto.cache_max_size_gb = 1 + mocker.patch.object(conversion_cache, "BYTES_PER_GB", 100) + f = _rom_file() + final = _seed_cached(f) + final.write_bytes(b"x" * 120) + served = time.time() - 120 + os.utime(final, (served, served)) + iterdir = Path.iterdir + + def serve_after_listing(path): + yield from iterdir(path) + if path == cache_root: + assert get_cached_converted(1, f, "psp", "chd", touch=True) == final + + mocker.patch.object( + Path, "iterdir", side_effect=serve_after_listing, autospec=True + ) + + assert conversion_cache.cleanup_stale_conversions() == 0 + assert final.read_bytes() == b"x" * 120 + + +class TestParseFormats: + @pytest.mark.parametrize( + ("raw", "expected"), + [ + pytest.param("zso,iso", ("zso", "iso"), id="in-order"), + pytest.param(" ZSO , iso,zso ", ("zso", "iso"), id="trimmed-deduplicated"), + pytest.param(",,", (), id="empty"), + pytest.param("3ds,GCM", ("cci", "iso"), id="aliases"), + ], + ) + def test_parses_the_list(self, raw: str, expected: tuple[str, ...]): + assert parse_formats(raw) == expected + + +class TestResolveFormatDownload: + @pytest.fixture(autouse=True) + def enabled(self, converto, mocker): + converto.download_conversion_enabled = True + mocker.patch( + "utils.conversion_cache.rom_converto_service.is_enabled", + AsyncMock(return_value=True), + ) + + @staticmethod + def _rom() -> Rom: + return cast(Rom, SimpleNamespace(id=1, platform_slug="psp")) + + async def test_a_listed_stored_format_wins_over_a_cached_copy(self, cache_root): + f = _rom_file() + _seed_cached(f) + + resolution = await resolve_format_download( + self._rom(), f, ("chd", "iso"), allowed=True, start=True, touch=True + ) + + assert resolution == FormatResolution(FormatOutcome.ORIGINAL) + + async def test_an_aliased_files_own_extension_serves_the_original( + self, cache_root, convert_outputs + ): + rom = cast(Rom, SimpleNamespace(id=1, platform_slug="3ds")) + + resolution = await resolve_format_download( + rom, + _rom_file(file_name="game.3ds"), + parse_formats("3ds,cia"), + allowed=True, + start=True, + touch=True, + ) + + assert resolution == FormatResolution(FormatOutcome.ORIGINAL) + assert convert_outputs == [] + + async def test_a_cached_copy_wins_over_an_earlier_uncached_format( + self, cache_root, convert_outputs + ): + f = _rom_file() + final = _seed_cached(f) + + resolution = await resolve_format_download( + self._rom(), f, ("cso", "chd"), allowed=True, start=True, touch=True + ) + + assert resolution == FormatResolution(FormatOutcome.CONVERTED, final) + assert convert_outputs == [] + + async def test_converts_to_the_first_listed_format_it_can_produce( + self, cache_root, convert_outputs + ): + f = _rom_file() + + resolution = await resolve_format_download( + self._rom(), f, ("rvz", "cso", "chd"), allowed=True, start=True, touch=True + ) + + assert resolution.outcome == FormatOutcome.CONVERTED + assert resolution.path is not None and resolution.path.suffix == ".cso" + assert [out.suffix for out in convert_outputs] == [".cso"] + + async def test_skips_a_format_whose_conversion_failed( + self, cache_root, convert_outputs + ): + f = _rom_file() + failed = _final_path(f).parent + failed.mkdir(parents=True) + (failed / FAILED_FILE).touch() + + resolution = await resolve_format_download( + self._rom(), f, ("chd", "cso"), allowed=True, start=True, touch=True + ) + + assert resolution.path is not None and resolution.path.suffix == ".cso" + + async def test_reports_pending_without_starting(self, cache_root, convert_outputs): + resolution = await resolve_format_download( + self._rom(), _rom_file(), ("chd",), allowed=True, start=False, touch=False + ) + + assert resolution.outcome == FormatOutcome.PENDING + assert convert_outputs == [] + + async def test_falls_back_to_a_later_format_when_a_conversion_fails( + self, cache_root, mocker + ): + async def convert(operation, src, out) -> None: + if out.suffix == ".cso": + raise RomConvertoOperationError("bad image") + out.write_bytes(b"converted") + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + + resolution = await resolve_format_download( + self._rom(), + _rom_file(), + ("cso", "chd"), + allowed=True, + start=True, + touch=True, + ) + + assert resolution.path is not None and resolution.path.suffix == ".chd" + + async def test_a_locked_key_dir_without_partial_output_is_pending( + self, cache_root, convert_outputs + ): + key_dir = _final_path(_rom_file()).parent + key_dir.mkdir(parents=True) + + with _held_lock(key_dir): + resolution = await resolve_format_download( + self._rom(), _rom_file(), ("chd",), allowed=True, start=True, touch=True + ) + + assert resolution.outcome == FormatOutcome.PENDING + assert convert_outputs == [] + + async def test_a_finished_conversion_is_served_while_a_download_holds_its_dir( + self, cache_root, mocker + ): + f = _rom_file() + final = _final_path(f) + + async def convert(operation, src, out) -> None: + out.write_bytes(b"converted") + + async def convert_then_serve(*args) -> Path | None: + converted = await get_or_convert(*args) + stack.enter_context(_held_lock(final.parent, shared=True)) + return converted + + mocker.patch( + "utils.conversion_cache.rom_converto_service.convert", side_effect=convert + ) + mocker.patch.object( + conversion_cache, "get_or_convert", side_effect=convert_then_serve + ) + with ExitStack() as stack: + resolution = await resolve_format_download( + self._rom(), f, ("chd",), allowed=True, start=True, touch=True + ) + + assert resolution == FormatResolution(FormatOutcome.CONVERTED, final) + + async def test_is_pending_without_starting_once_too_many_conversions_run( + self, cache_root, convert_outputs, mocker + ): + mocker.patch.object(conversion_cache, "MAX_STARTED_CONVERSIONS", 0) + + resolution = await resolve_format_download( + self._rom(), _rom_file(), ("chd",), allowed=True, start=True, touch=True + ) + + assert resolution.outcome == FormatOutcome.PENDING + assert convert_outputs == [] diff --git a/backend/utils/conversion_cache.py b/backend/utils/conversion_cache.py new file mode 100644 index 000000000..c16373f33 --- /dev/null +++ b/backend/utils/conversion_cache.py @@ -0,0 +1,415 @@ +from __future__ import annotations + +import asyncio +import contextlib +import enum +import fcntl +import hashlib +import os +import shutil +import time +from collections.abc import Generator, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from adapters.services.rom_converto import ( + UNCOMPRESSED_TARGETS, + Operation, + RomConvertoOperationError, + RomConvertoTimeoutError, + canonical_format, + file_format, + resolve_operation, + rom_converto_service, +) +from config import ( + LIBRARY_BASE_PATH, + ROM_CONVERTO_CACHE_PATH, + ROM_CONVERTO_MAX_CONCURRENCY, + ROMM_BASE_PATH, +) +from config.config_manager import config_manager as cm +from logger.formatter import highlight as hl +from logger.logger import log +from utils.background_tasks import fire_and_forget +from utils.zip_cache import CACHE_KEY_LENGTH, SECONDS_PER_HOUR + +if TYPE_CHECKING: + from models.rom import Rom, RomFile + +BYTES_PER_GB = 1024**3 +# How long a download waits on a conversion it started before answering 202. +CONVERSION_WAIT_SECONDS: Final = 15 +RETRY_AFTER_SECONDS: Final = 30 +PARTIAL_DIR: Final = ".partial" +FAILED_FILE: Final = ".failed" +# Give nginx time to open a copy after the X-Accel-Redirect response. +SERVE_GRACE_SECONDS: Final = 60 +# Room a conversion to an uncompressed target reserves per input byte; the +# cleanup pass after it evicts any overshoot. +EXPANSION_RESERVE_FACTOR: Final = 4 +# Downloads may start conversions until this many wait for or hold the semaphore. +MAX_STARTED_CONVERSIONS: Final = ROM_CONVERTO_MAX_CONCURRENCY * 4 + +# Each conversion reads and writes whole disc images. +_convert_semaphore = asyncio.Semaphore(ROM_CONVERTO_MAX_CONCURRENCY) +_started_conversions: set[asyncio.Task[Path | None]] = set() + + +def converted_file_path( + rom_id: int, rom_file: RomFile, operation: Operation, input_ext: str +) -> Path: + """Deterministic cache path for a converted ROM file.""" + key = hashlib.sha1( + f"{rom_file.file_path}/{rom_file.file_name}|{rom_file.last_modified}|{rom_file.file_size_bytes}|{operation.target}".encode(), + usedforsecurity=False, + ).hexdigest()[:CACHE_KEY_LENGTH] + key_dir = Path(ROM_CONVERTO_CACHE_PATH) / f"{rom_id}-{key}" + return key_dir / operation.output_name(Path(rom_file.file_name), input_ext) + + +def get_redirect_path(converted_path: Path) -> Path: + """The nginx-internal path for a converted file (`/cache/` aliases `${ROMM_BASE_PATH}/cache/`).""" + return Path("/") / converted_path.relative_to(ROMM_BASE_PATH) + + +def _dir_size(path: Path) -> int: + total = 0 + for root, _, files in os.walk(path): + for name in files: + with contextlib.suppress(OSError): + total += os.stat(os.path.join(root, name)).st_size + return total + + +def _files(key_dir: Path) -> list[Path]: + return [p for p in key_dir.iterdir() if p.is_file()] + + +@contextlib.contextmanager +def _try_lock(key_dir: Path, *, shared: bool = False) -> Generator[bool]: + """Lock the key dir without blocking, yielding False if it is busy, removed or replaced.""" + fd = os.open(key_dir, os.O_RDONLY) + try: + try: + mode = fcntl.LOCK_SH if shared else fcntl.LOCK_EX + fcntl.flock(fd, mode | fcntl.LOCK_NB) + except BlockingIOError: + yield False + return + try: + current_dir = os.path.samestat(os.fstat(fd), os.stat(key_dir)) + except FileNotFoundError: + current_dir = False + yield current_dir + finally: + os.close(fd) + + +def _stat_files(key_dir: Path) -> tuple[dict[str, os.stat_result], float]: + """The stats of `key_dir`'s files and the newest mtime among them.""" + stats = {p.name: p.stat() for p in _files(key_dir)} + return stats, max((st.st_mtime for st in stats.values()), default=0.0) + + +def cache_size_bytes() -> int: + """Bytes every key dir holds on disk, partial output included.""" + return _dir_size(Path(ROM_CONVERTO_CACHE_PATH)) + + +def has_room_for(size_bytes: int) -> bool: + """Whether adding `size_bytes` keeps the cache under `converto.cache_max_size_gb`.""" + max_bytes = cm.get_config().CONVERTO.cache_max_size_gb * BYTES_PER_GB + return not max_bytes or cache_size_bytes() + size_bytes <= max_bytes + + +def _lookup( + rom_id: int, rom_file: RomFile, platform_slug: str, target: str +) -> tuple[Operation, Path] | None: + """The operation bringing `rom_file` to `target` and its cache path.""" + resolved = resolve_operation(platform_slug, target, rom_file.file_name) + if resolved is None: + return None + operation, input_ext = resolved + return operation, converted_file_path(rom_id, rom_file, operation, input_ext) + + +def _serve_cached(final_path: Path, *, touch: bool) -> Path | None: + """`final_path` if it is cached, or None while cleanup or a conversion holds its dir.""" + with ( + contextlib.suppress(OSError), + _try_lock(final_path.parent, shared=True) as locked, + ): + if locked and final_path.exists(): + if touch: + # Keep a served file fresh so TTL cleanup measures demand. + with contextlib.suppress(OSError): + os.utime(final_path) + return final_path + return None + + +def get_cached_converted( + rom_id: int, + rom_file: RomFile, + platform_slug: str, + target: str, + *, + touch: bool = False, +) -> Path | None: + """The converted file if it is already cached, without converting. + + Args: + touch: Refresh its mtime so TTL cleanup measures demand. + """ + found = _lookup(rom_id, rom_file, platform_slug, target) + return _serve_cached(found[1], touch=touch) if found else None + + +class FormatOutcome(enum.StrEnum): + ORIGINAL = "original" + CONVERTED = "converted" + PENDING = "pending" + UNAVAILABLE = "unavailable" + + +@dataclass(frozen=True) +class FormatResolution: + outcome: FormatOutcome + # The cached copy to serve when `outcome` is CONVERTED. + path: Path | None = None + + +_PENDING: Final = FormatResolution(FormatOutcome.PENDING) +_UNAVAILABLE: Final = FormatResolution(FormatOutcome.UNAVAILABLE) + + +def parse_formats(raw: str) -> tuple[str, ...]: + """The comma-separated `?format=` list, canonical, lowercased and deduplicated in order.""" + formats = (canonical_format(f.strip().lower()) for f in raw.split(",")) + return tuple(dict.fromkeys(f for f in formats if f)) + + +def _is_converting(final_path: Path) -> bool: + """Whether a worker holds `final_path`'s key dir, which it locks before converting.""" + # A shared probe leaves downloads serving a cached copy undisturbed. + with ( + contextlib.suppress(OSError), + _try_lock(final_path.parent, shared=True) as locked, + ): + return not locked + return False + + +async def resolve_format_download( + rom: Rom, + file: RomFile, + formats: Sequence[str], + *, + allowed: bool, + start: bool, + touch: bool, +) -> FormatResolution: + """What a `?format=` download serves: the stored file if listed, else the first cached, then the first convertible format. + + Args: + allowed: The caller may start a conversion. + start: Start it now, rather than only report that one would run. + touch: Refresh a served copy's mtime so TTL cleanup measures demand. + """ + if file_format(file.file_name) in formats: + return FormatResolution(FormatOutcome.ORIGINAL) + + converto = cm.get_config().CONVERTO + if ( + not converto.download_conversion_enabled + or not await rom_converto_service.is_enabled() + ): + return _UNAVAILABLE + + candidates = [ + (target, found[1]) + for target in formats + if (found := _lookup(rom.id, file, rom.platform_slug, target)) + ] + for _, final_path in candidates: + if cached := _serve_cached(final_path, touch=touch): + return FormatResolution(FormatOutcome.CONVERTED, cached) + if not allowed: + return _UNAVAILABLE + + for target, final_path in candidates: + if (final_path.parent / FAILED_FILE).exists(): + continue + if ( + not start + or _is_converting(final_path) + or len(_started_conversions) >= MAX_STARTED_CONVERSIONS + ): + return _PENDING + # The conversion finishes into the cache even when this request stops waiting. + conversion = fire_and_forget( + get_or_convert(rom.id, file, rom.platform_slug, target) + ) + _started_conversions.add(conversion) + conversion.add_done_callback(_started_conversions.discard) + done, _ = await asyncio.wait({conversion}, timeout=CONVERSION_WAIT_SECONDS) + if done and (converted := conversion.result()): + return FormatResolution(FormatOutcome.CONVERTED, converted) + if not done or _is_converting(final_path): + return _PENDING + return _UNAVAILABLE + + +async def get_or_convert( + rom_id: int, + rom_file: RomFile, + platform_slug: str, + target: str, +) -> Path | None: + """The converted file, converting it under a lock on its key dir, or None when it can't be.""" + found = _lookup(rom_id, rom_file, platform_slug, target) + if found is None: + return None + operation, final_path = found + + try: + if cached := _serve_cached(final_path, touch=True): + return cached + + final_path.parent.mkdir(parents=True, exist_ok=True) + with _try_lock(final_path.parent) as locked: + if not locked: + # Another worker is converting it. + return None + # A conversion may have finished between the check above and the lock. + if final_path.exists(): + with contextlib.suppress(OSError): + os.utime(final_path) + return final_path + if (final_path.parent / FAILED_FILE).exists(): + return None + async with _convert_semaphore: + await asyncio.to_thread( + shutil.rmtree, final_path.parent / PARTIAL_DIR, True + ) + size_bytes = (rom_file.file_size_bytes or 0) * ( + EXPANSION_RESERVE_FACTOR + if operation.target in UNCOMPRESSED_TARGETS + else 1 + ) + await asyncio.to_thread(cleanup_stale_conversions, size_bytes) + if not await asyncio.to_thread(has_room_for, size_bytes): + log.info( + f"Conversion cache is full, not converting ROM {rom_id} (target {hl(target)})" + ) + return None + converted = await _convert( + rom_id, rom_file, target, operation, final_path + ) + if converted: + try: + await asyncio.to_thread(cleanup_stale_conversions) + except Exception as e: + log.warning( + f"Conversion cache cleanup failed for ROM {rom_id}: {e}; serving converted copy" + ) + return converted + except Exception as e: + log.warning( + f"Conversion cache unavailable for ROM {rom_id} (target {hl(target)}): {e}" + ) + return None + + +async def _convert( + rom_id: int, rom_file: RomFile, target: str, operation: Operation, final_path: Path +) -> Path | None: + """Convert into a partial file and rename it into place; the caller holds the key dir lock.""" + partial_dir = final_path.parent / PARTIAL_DIR + produced = partial_dir / final_path.name + try: + partial_dir.mkdir() + await rom_converto_service.convert( + operation, src=Path(LIBRARY_BASE_PATH) / rom_file.full_path, out=produced + ) + os.replace(produced, final_path) + except Exception as e: + log.warning(f"Conversion failed for ROM {rom_id} (target {hl(target)}): {e}") + if isinstance(e, (RomConvertoOperationError, RomConvertoTimeoutError)): + (final_path.parent / FAILED_FILE).touch() + return None + finally: + await asyncio.to_thread(shutil.rmtree, partial_dir, True) + return final_path + + +def cleanup_stale_conversions(reserve_bytes: int = 0) -> int: + """Remove expired or leaked key dirs, then evict least recently served copies to make room.""" + cache_root = Path(ROM_CONVERTO_CACHE_PATH) + if not cache_root.exists(): + return 0 + + converto = cm.get_config().CONVERTO + now = time.time() + expired_before = now - converto.cache_ttl_hours * SECONDS_PER_HOUR + deleted = 0 + # (last served, size) of the final copies in each dir that survives the TTL pass. + kept: list[tuple[float, int, Path]] = [] + + for key_dir in cache_root.iterdir(): + if not key_dir.is_dir(): + continue + with contextlib.suppress(FileNotFoundError): + stats, last_served = _stat_files(key_dir) + partial_dir = key_dir / PARTIAL_DIR + if last_served < expired_before or partial_dir.exists(): + with _try_lock(key_dir) as locked: + if locked: + stats, last_served = _stat_files(key_dir) + if last_served < expired_before: + shutil.rmtree(key_dir, ignore_errors=True) + deleted += 1 + continue + shutil.rmtree(partial_dir, ignore_errors=True) + # A failure marker frees nothing, and evicting it would retry the conversion. + stats.pop(FAILED_FILE, None) + if stats: + kept.append( + ( + max(st.st_mtime for st in stats.values()), + sum(st.st_size for st in stats.values()), + key_dir, + ) + ) + + max_bytes = converto.cache_max_size_gb * BYTES_PER_GB + if not max_bytes: + return deleted + used = cache_size_bytes() + candidates = sorted(c for c in kept if c[0] < now - SERVE_GRACE_SECONDS) + if reserve_bytes and used + reserve_bytes > max_bytes: + evictable = 0 + for _, size, key_dir in candidates: + # A dir a download or conversion holds right now can't be evicted. + with contextlib.suppress(FileNotFoundError), _try_lock(key_dir) as locked: + evictable += size if locked else 0 + if used + reserve_bytes - evictable > max_bytes: + # Evicting every candidate would still not make room, so keep them. + return deleted + for _, _, key_dir in candidates: + if used + reserve_bytes <= max_bytes: + break + with contextlib.suppress(FileNotFoundError), _try_lock(key_dir) as locked: + # A download may have touched the copy since the pass above. + if not locked or any( + p.stat().st_mtime >= now - SERVE_GRACE_SECONDS for p in _files(key_dir) + ): + continue + size = _dir_size(key_dir) + shutil.rmtree(key_dir, ignore_errors=True) + used -= size + deleted += 1 + + return deleted diff --git a/docker/Dockerfile b/docker/Dockerfile index 4e09483ef..79c03ec11 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,6 +5,7 @@ # - backend-dev-build: Similar to `backend-build`, but also compiles and installs development dependencies # - rahasher-build: Build RAHasher # - sigil-build: Build sigil Python bindings (title ID extraction) +# - rom-converto-fetch: Fetch the rom-converto CLI (ROM conversion and metadata) # - emulator-stage: Fetch and extract emulators # - nginx-build: Build nginx modules # - production-stage: Setup frontend and backend @@ -144,6 +145,22 @@ RUN git clone --filter=blob:none https://github.com/rommapp/argosy-sigil.git && cp ./sigil/*.py ./sigil/_sigil.*.so /sigil-pkg/ +# ROM-CONVERTO CLI +# Keep the version and both sums in sync with the dev image's copy in ../Dockerfile. +FROM alpine:${ALPINE_VERSION}@sha256:${ALPINE_SHA256} AS rom-converto-fetch +ARG TARGETARCH +ARG ROM_CONVERTO_VERSION=v0.23.0 +RUN case "${TARGETARCH}" in \ + amd64) rc="linux-x64-musl"; sum="37d03a67ad97fc02768890e763b4c80072fdfd15f2c5b97e44f9a2b50fc40876" ;; \ + arm64) rc="linux-arm64-musl"; sum="8abe37ca2d1a0e3d719fb8e795e011c672e3f1c088521cf0bb1e467dc658dc3c" ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + wget -qO /rom-converto \ + "https://github.com/DevYukine/rom-converto/releases/download/${ROM_CONVERTO_VERSION}/rom-converto-cli-${rc}" && \ + echo "${sum} /rom-converto" | sha256sum -c - && \ + chmod 0755 /rom-converto + + # FETCH EMULATORJS, RUFFLE, JS-DOS AND FAKE-08 # Keep the pins below in sync with the dev image's copies in ../Dockerfile. # checkov:skip=CKV_DOCKER_4:every ADD below is a remote URL pinned by --checksum @@ -258,6 +275,7 @@ COPY --from=python-runtime /usr/local/lib/libpython* /usr/local/lib/ COPY --from=python-runtime /usr/local/lib/python${PYTHON_VERSION} /usr/local/lib/python${PYTHON_VERSION} COPY --from=rahasher-build /RALibretro/bin64/RAHasher /usr/bin/RAHasher +COPY --from=rom-converto-fetch /rom-converto /usr/bin/rom-converto COPY --from=nginx-build ./nginx/objs/ngx_http_zip_module.so /usr/lib/nginx/modules/ COPY --from=frontend-build /front/dist ${WEBSERVER_FOLDER} COPY --from=backend-node-build /backend-node/rom_patcher/node_modules/rom-patcher/rom-patcher-js /backend/utils/rom_patcher/rom-patcher-js diff --git a/docs/BACKEND_ARCHITECTURE.md b/docs/BACKEND_ARCHITECTURE.md index 6331fba82..0b229440c 100644 --- a/docs/BACKEND_ARCHITECTURE.md +++ b/docs/BACKEND_ARCHITECTURE.md @@ -321,6 +321,7 @@ backend/ │ │ └── reap_streaming_sessions.py # Stop abandoned streaming sessions │ └── manual/ # On-demand tasks │ ├── cleanup_missing_roms.py # Drop DB entries for missing files +│ ├── convert_library.py # Store matched ROMs in each platform's library format │ ├── cleanup_orphaned_resources.py # Remove unreferenced artwork │ └── sync_folder_scan.py # Scan sync folder for new saves │ @@ -908,8 +909,11 @@ A reset link is emailed when SMTP is set up, the user has an address and `ROMM_B | DELETE | `/{id}` | ROMS_WRITE | Delete ROM | | POST | `/delete` | ROMS_WRITE | Bulk delete | | POST | `/download/{id}/{file_name}` | ROMS_READ | Download ROM | +| GET | `/{id}/content/{file_name}` | ROMS_READ | Download ROM, in a listed `?format=` when asked | | POST | `/unidentified` | ROMS_READ | Get unidentified ROMs | +With rom-converto enabled (`ROM_CONVERTO_ENABLED`, `converto.download_conversion_enabled`), `?format=zso,iso` lists the formats a client can read. A single-file download whose stored format is listed is served as-is. Otherwise the first listed format with a copy cached under `/romm/cache/converts` is served, then the first one rom-converto can produce: answered by the same request if it converts within a few seconds, else `202` with `Retry-After` while it finishes. `406` means no listed format can be served, and `HEAD` reports the same without starting a conversion. Only signed-in, non-kiosk users start conversions. `CONVERTO_DOWNLOAD_FORMATS` in `/api/config` lists the formats each input extension converts to. + #### ROM Upload (Chunked) | Method | Path | Scope | Description | @@ -1679,29 +1683,33 @@ failure callback and a scan needs one to report a worker that died mid-scan. Toggled via environment variables: -| Task | Env Toggle | Default Cron | Description | -| --------------------------------- | ----------------------------------------------------- | ------------------ | ---------------------- | -| `scan_library` | `ENABLE_SCHEDULED_RESCAN` | `0 3 * * *` (3 AM) | Full library rescan | -| `update_switch_titledb` | `ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB` | `0 4 * * *` | Update Switch game DB | -| `update_launchbox_metadata` | `ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA` | `0 4 * * *` | Refresh LaunchBox data | -| `convert_images_to_webp` | `ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP` | `0 4 * * *` | Image optimization | -| `sync_retroachievements_progress` | `ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC` | `0 4 * * *` | Sync RA user progress | -| `cleanup_orphaned_resources` | `ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES` | `0 5 * * *` | Remove unused artwork | -| `cleanup_netplay` | `ENABLE_SCHEDULED_CLEANUP_NETPLAY` (default on) | `*/30 * * * *` | Clean empty rooms | -| `cleanup_upload_tmp` | `ENABLE_SCHEDULED_CLEANUP_UPLOAD_TMP` (default on) | `0 * * * *` | Drop stale uploads | -| `cleanup_zip_cache` | `ENABLE_SCHEDULED_CLEANUP_ZIP_CACHE` (default on) | `0 4 * * *` | Drop stale cached ZIPs | -| `cleanup_sync_sessions` | `ENABLE_SCHEDULED_CLEANUP_SYNC_SESSIONS` (default on) | `23 * * * *` | Fail abandoned syncs | -| `reap_streaming_sessions` | `streaming.enabled` in config, read at startup | `* * * * *` | Stop abandoned streams | -| `cleanup_audit_log` | `AUDIT_LOG_RETENTION_DAYS` above 0 (default 90) | `30 4 * * *` | Prune old audit events | +| Task | Env Toggle | Default Cron | Description | +| --------------------------------- | ----------------------------------------------------- | ------------------ | ------------------------------ | +| `scan_library` | `ENABLE_SCHEDULED_RESCAN` | `0 3 * * *` (3 AM) | Full library rescan | +| `update_switch_titledb` | `ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB` | `0 4 * * *` | Update Switch game DB | +| `update_launchbox_metadata` | `ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA` | `0 4 * * *` | Refresh LaunchBox data | +| `convert_images_to_webp` | `ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP` | `0 4 * * *` | Image optimization | +| `sync_retroachievements_progress` | `ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC` | `0 4 * * *` | Sync RA user progress | +| `cleanup_orphaned_resources` | `ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES` | `0 5 * * *` | Remove unused artwork | +| `cleanup_netplay` | `ENABLE_SCHEDULED_CLEANUP_NETPLAY` (default on) | `*/30 * * * *` | Clean empty rooms | +| `cleanup_upload_tmp` | `ENABLE_SCHEDULED_CLEANUP_UPLOAD_TMP` (default on) | `0 * * * *` | Drop stale uploads | +| `cleanup_zip_cache` | `ENABLE_SCHEDULED_CLEANUP_ZIP_CACHE` (default on) | `0 4 * * *` | Drop stale cached ZIPs | +| `cleanup_conversion_cache` | Always on | `0 4 * * *` | Drop stale converted downloads | +| `cleanup_sync_sessions` | `ENABLE_SCHEDULED_CLEANUP_SYNC_SESSIONS` (default on) | `23 * * * *` | Fail abandoned syncs | +| `reap_streaming_sessions` | `streaming.enabled` in config, read at startup | `* * * * *` | Stop abandoned streams | +| `cleanup_audit_log` | `AUDIT_LOG_RETENTION_DAYS` above 0 (default 90) | `30 4 * * *` | Prune old audit events | ### Manual Tasks Triggered via `POST /api/tasks/run/{task_name}`, which enqueues on `low_prio_queue` and answers 503 when no live worker is listening on it: -| Task | Description | -| ---------------------- | --------------------------------------------- | -| `cleanup_missing_roms` | Remove DB entries for files no longer on disk | -| `sync_folder_scan` | Scan sync folder for new device saves | +| Task | Description | +| ---------------------- | ------------------------------------------------------------------------------------- | +| `cleanup_missing_roms` | Remove DB entries for files no longer on disk | +| `sync_folder_scan` | Scan sync folder for new device saves | +| `convert_library` | Convert matched ROMs to `converto.platform_formats` in place, replacing the originals | + +`convert_library` only converts losslessly (no xiso) and only identified ROMs, since a converted file no longer hash-matches a DAT. It stages each output under a `.romm_tmp_` directory beside the ROM, deletes the originals (and a cue's tracks) once the output is in place, rewrites `.m3u` entries in folder ROMs, and refreshes the ROM's files, so the ROM keeps its id, saves and collections. `cleanup_orphaned_resources` is also runnable this way; it is listed under Scheduled Tasks because it additionally supports an opt-in cron schedule. It diff --git a/env.template b/env.template index 465c5a1e8..1f2edf8f2 100644 --- a/env.template +++ b/env.template @@ -150,6 +150,11 @@ DISABLE_RUFFLE_RS=false # Disable in-browser Flash playback via RuffleRS DISABLE_JSDOS=false # Disable in-browser Win3.x and Win9.x playback via js-dos DISABLE_PICO8=false # Disable in-browser PICO-8 playback via FAKE-08 +# ROM conversion +ROM_CONVERTO_ENABLED=false # Enable rom-converto for the "Convert library" task and ?format= download conversions +ROM_CONVERTO_TIMEOUT=600 # Timeout for each rom-converto operation in seconds +ROM_CONVERTO_MAX_CONCURRENCY=2 # Concurrent conversions per web or task worker + # Integrations YOUTUBE_BASE_URL=https://www.youtube.com # Base URL for alternate YouTube frontends (Piped, Invidious, etc.) TINFOIL_WELCOME_MESSAGE="RomM Switch Library" # Welcome message shown in Tinfoil Switch clients diff --git a/examples/config.example.yml b/examples/config.example.yml index 200114249..cc07cd7c4 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -253,3 +253,24 @@ # core: parallel_n64 # # Lets a core listed as broken by the broker run anyway. # # experimental_cores: true + +# converto: +# # Convert a single-file download into a format the client lists with +# # ?format=, e.g. ?format=zso,iso (needs ROM_CONVERTO_ENABLED) +# download_conversion_enabled: false +# # Hours a converted download stays cached after it was last downloaded +# cache_ttl_hours: 24 +# # Cache size before the least recently downloaded copies are evicted (0: unbounded) +# cache_max_size_gb: 20 +# # Platform slug -> the format the "Convert library" task stores its matched +# # roms in, replacing the original files. Formats per platform: +# # 3ds: cci, cia, z3ds +# # psp, ps2: chd, cso, iso, zso +# # psx, saturn, segacd, dc: chd +# # ngc: iso, rvz +# # wii: iso, rvz, wbfs +# # switch, switch-2: nsp, nsz, xci, xcz +# # xbox360: zar +# platform_formats: +# psx: chd +# ngc: rvz diff --git a/frontend/src/__generated__/index.ts b/frontend/src/__generated__/index.ts index ee12376d0..f0a190edc 100644 --- a/frontend/src/__generated__/index.ts +++ b/frontend/src/__generated__/index.ts @@ -82,6 +82,8 @@ export type { ContainerSessionSchema } from './models/ContainerSessionSchema'; export type { ConversionStats } from './models/ConversionStats'; export type { ConversionTaskMeta } from './models/ConversionTaskMeta'; export type { ConversionTaskStatusResponse } from './models/ConversionTaskStatusResponse'; +export type { ConvertoConfig } from './models/ConvertoConfig'; +export type { ConvertoSettingsPayload } from './models/ConvertoSettingsPayload'; export type { CoverResource } from './models/CoverResource'; export type { CsdbCredit } from './models/CsdbCredit'; export type { CustomLimitOffsetPage_SimpleRomSchema_ } from './models/CustomLimitOffsetPage_SimpleRomSchema_'; diff --git a/frontend/src/__generated__/models/ConfigResponse.ts b/frontend/src/__generated__/models/ConfigResponse.ts index 850221e01..3f0e6c4e8 100644 --- a/frontend/src/__generated__/models/ConfigResponse.ts +++ b/frontend/src/__generated__/models/ConfigResponse.ts @@ -2,6 +2,7 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ +import type { ConvertoConfig } from './ConvertoConfig'; import type { EjsControls } from './EjsControls'; import type { MetadataMediaType } from './MetadataMediaType'; import type { NetplayICEServer } from './NetplayICEServer'; @@ -42,5 +43,8 @@ export type ConfigResponse = { GAMELIST_MEDIA_THUMBNAIL: MetadataMediaType; GAMELIST_MEDIA_IMAGE: MetadataMediaType; PEGASUS_AUTO_EXPORT_ON_SCAN: boolean; + CONVERTO: ConvertoConfig; + CONVERTO_LIBRARY_TARGETS: Record>; + CONVERTO_DOWNLOAD_FORMATS: Record>>; }; diff --git a/frontend/src/__generated__/models/ConvertoConfig.ts b/frontend/src/__generated__/models/ConvertoConfig.ts new file mode 100644 index 000000000..0029b7e5f --- /dev/null +++ b/frontend/src/__generated__/models/ConvertoConfig.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ConvertoConfig = { + download_conversion_enabled?: boolean; + cache_ttl_hours?: number; + cache_max_size_gb?: number; + platform_formats?: Record; +}; + diff --git a/frontend/src/__generated__/models/ConvertoSettingsPayload.ts b/frontend/src/__generated__/models/ConvertoSettingsPayload.ts new file mode 100644 index 000000000..c6fc57ece --- /dev/null +++ b/frontend/src/__generated__/models/ConvertoSettingsPayload.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +/** + * The converto.* settings editable at runtime; `cache_ttl_hours` stays config.yml-only. + */ +export type ConvertoSettingsPayload = { + download_conversion_enabled: boolean; + cache_max_size_gb: number; + platform_formats: Record; +}; + diff --git a/frontend/src/stores/config.ts b/frontend/src/stores/config.ts index cb9520038..0524f26e8 100644 --- a/frontend/src/stores/config.ts +++ b/frontend/src/stores/config.ts @@ -51,6 +51,14 @@ const defaultConfig = { GAMELIST_MEDIA_THUMBNAIL: "box2d", GAMELIST_MEDIA_IMAGE: "screenshot", PEGASUS_AUTO_EXPORT_ON_SCAN: false, + CONVERTO: { + download_conversion_enabled: false, + cache_ttl_hours: 24, + cache_max_size_gb: 20, + platform_formats: {}, + }, + CONVERTO_LIBRARY_TARGETS: {}, + CONVERTO_DOWNLOAD_FORMATS: {}, } as ConfigResponse; export default defineStore("config", {