mirror of
https://github.com/Heretek-AI/Asset-Manager.git
synced 2026-11-07 17:16:08 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e76623a4f1 | ||
|
|
745bdc9e73 | ||
|
|
89b3b4eed5 | ||
|
|
e830f75efd | ||
|
|
78e9bf7ceb | ||
|
|
47a5becb8e | ||
|
|
d1a3df592e | ||
|
|
149284121b | ||
|
|
e84058ce38 |
No files matched your search
+102
-1
@@ -236,10 +236,12 @@ def _cmd_library_push(args: argparse.Namespace) -> int:
|
||||
else:
|
||||
params = {}
|
||||
tags = [t for t in (args.tags or "").split(",") if t]
|
||||
chunk_above = 0 if args.chunked else args.chunk_above
|
||||
try:
|
||||
result = push(server, token, args.file, name=args.name,
|
||||
kind=args.kind, importer=args.importer,
|
||||
params=params, tags=tags)
|
||||
params=params, tags=tags,
|
||||
chunk_size=args.chunk_size, chunk_above=chunk_above)
|
||||
except LibraryError as exc:
|
||||
return _die(args, str(exc), 1)
|
||||
_emit(args, f"uploaded as asset {result['id']}", result)
|
||||
@@ -261,6 +263,71 @@ def _cmd_library_pull(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_library_thumbnail(args: argparse.Namespace) -> int:
|
||||
from library.client import LibraryError, pull_thumbnail
|
||||
|
||||
try:
|
||||
server, token = _library_endpoint(args)
|
||||
except SystemExit as exc:
|
||||
return _die(args, str(exc), 2)
|
||||
try:
|
||||
result = pull_thumbnail(server, token, args.asset_id, args.out)
|
||||
except LibraryError as exc:
|
||||
return _die(args, str(exc), 1)
|
||||
_emit(args, f"thumbnail saved to {result['path']}", result)
|
||||
return 0
|
||||
|
||||
|
||||
def _library_db(args: argparse.Namespace) -> str:
|
||||
return os.path.abspath(args.db or "library-data/library.db")
|
||||
|
||||
|
||||
def _cmd_library_token_create(args: argparse.Namespace) -> int:
|
||||
from library import db as library_db
|
||||
|
||||
conn = library_db.open_db(_library_db(args))
|
||||
try:
|
||||
plaintext = library_db.create_token(conn, args.name)
|
||||
except ValueError as exc:
|
||||
return _die(args, str(exc), 1)
|
||||
finally:
|
||||
conn.close()
|
||||
# Plaintext is shown once and never stored.
|
||||
_emit(args, plaintext, {"name": args.name, "token": plaintext})
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_library_token_list(args: argparse.Namespace) -> int:
|
||||
from library import db as library_db
|
||||
|
||||
conn = library_db.open_db(_library_db(args))
|
||||
try:
|
||||
rows = library_db.list_tokens(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
if args.json:
|
||||
print(json.dumps({"tokens": rows}))
|
||||
else:
|
||||
for row in rows:
|
||||
print(f"{row['name']} created={row['created_at']}")
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_library_token_revoke(args: argparse.Namespace) -> int:
|
||||
from library import db as library_db
|
||||
|
||||
conn = library_db.open_db(_library_db(args))
|
||||
try:
|
||||
revoked = library_db.revoke_token(conn, args.name)
|
||||
finally:
|
||||
conn.close()
|
||||
if not revoked:
|
||||
return _die(args, f"no such token: {args.name}", 1)
|
||||
_emit(args, f"revoked {args.name}", {"name": args.name,
|
||||
"revoked": True})
|
||||
return 0
|
||||
|
||||
|
||||
def _die(args: argparse.Namespace, message: str, code: int) -> int:
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps({"error": message}))
|
||||
@@ -343,6 +410,12 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="import params as a JSON object")
|
||||
p.add_argument("--tags", default="",
|
||||
help="comma-separated tags")
|
||||
p.add_argument("--chunked", action="store_true",
|
||||
help="force the resumable session protocol")
|
||||
p.add_argument("--chunk-size", type=int, default=8 << 20,
|
||||
help="session block size in bytes")
|
||||
p.add_argument("--chunk-above", type=int, default=64 << 20,
|
||||
help="auto-switch to sessions above this file size")
|
||||
_add_json(p)
|
||||
p.set_defaults(func=_cmd_library_push)
|
||||
|
||||
@@ -353,6 +426,34 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
p.add_argument("--token", default="")
|
||||
_add_json(p)
|
||||
p.set_defaults(func=_cmd_library_pull)
|
||||
|
||||
p = sub.add_parser("library-thumbnail",
|
||||
help="download an asset's thumbnail PNG")
|
||||
p.add_argument("asset_id", type=int)
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--server", default="")
|
||||
p.add_argument("--token", default="")
|
||||
_add_json(p)
|
||||
p.set_defaults(func=_cmd_library_thumbnail)
|
||||
|
||||
p = sub.add_parser("library-token-create",
|
||||
help="create a library API token (printed once)")
|
||||
p.add_argument("name")
|
||||
p.add_argument("--db", default="",
|
||||
help="metadata DB (default: library-data/library.db)")
|
||||
_add_json(p)
|
||||
p.set_defaults(func=_cmd_library_token_create)
|
||||
|
||||
p = sub.add_parser("library-token-list", help="list library API tokens")
|
||||
p.add_argument("--db", default="")
|
||||
_add_json(p)
|
||||
p.set_defaults(func=_cmd_library_token_list)
|
||||
|
||||
p = sub.add_parser("library-token-revoke", help="revoke a library API token")
|
||||
p.add_argument("name")
|
||||
p.add_argument("--db", default="")
|
||||
_add_json(p)
|
||||
p.set_defaults(func=_cmd_library_token_revoke)
|
||||
return parser
|
||||
|
||||
|
||||
|
||||
@@ -399,6 +399,37 @@ IMPORTER_PARAMS: dict[str, frozenset[str]] = {
|
||||
"edit/trim", "edit/normalize", "edit/loop_mode", "edit/loop_begin",
|
||||
"edit/loop_end", "compress/mode",
|
||||
}),
|
||||
"mp3": frozenset({
|
||||
"loop", "loop_offset", "bpm", "beat_count", "bar_beats",
|
||||
}),
|
||||
"oggvorbisstr": frozenset({
|
||||
"loop", "loop_offset", "bpm", "beat_count", "bar_beats",
|
||||
}),
|
||||
"font_data_dynamic": frozenset({
|
||||
"Rendering", "antialiasing", "generate_mipmaps",
|
||||
"disable_embedded_bitmaps", "multichannel_signed_distance_field",
|
||||
"msdf_pixel_range", "msdf_size", "allow_system_fallback",
|
||||
"force_autohinter", "hinting", "subpixel_positioning",
|
||||
"oversampling", "Fallbacks", "fallbacks", "Compress", "compress",
|
||||
"preload", "language_support", "script_support",
|
||||
"opentype_features",
|
||||
}),
|
||||
"wavefront_obj": frozenset({
|
||||
"generate_tangents", "scale_mesh", "offset_mesh", "optimize_mesh",
|
||||
"force_disable_mesh_compression",
|
||||
}),
|
||||
"scene": frozenset({
|
||||
"nodes/root_type", "nodes/root_name", "nodes/apply_root_scale",
|
||||
"nodes/root_scale", "nodes/import_as_skeleton_bones",
|
||||
"meshes/ensure_tangents", "meshes/generate_lods",
|
||||
"meshes/create_shadow_meshes", "meshes/light_baking",
|
||||
"meshes/lightmap_texel_size", "meshes/force_disable_compression",
|
||||
"skins/use_named_skins", "animation/import", "animation/fps",
|
||||
"animation/trimming", "animation/remove_immutable_tracks",
|
||||
"animation/import_rest_as_RESET", "import_script/path",
|
||||
"_subresources", "gltf/naming_version",
|
||||
"gltf/embedded_image_handling",
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
|
||||
+5
-1
@@ -74,5 +74,9 @@ curl -X POST localhost:8080/api/thumbnail \
|
||||
asset inventory) · `thumbnail SRC [--out PNG]` (exit 3 = no engine) ·
|
||||
`library-init DIR` · `library-push FILE --server URL --token T [--name N]
|
||||
[--kind K] [--importer I] [--params JSON] [--tags a,b]` ·
|
||||
`library-pull ID --out PATH --server URL --token T`
|
||||
`library-pull ID --out PATH --server URL --token T` ·
|
||||
`library-thumbnail ID --out PNG --server URL --token T` (exit 1 when absent) ·
|
||||
`library-token-create NAME [--db PATH]` (prints once) ·
|
||||
`library-token-list` · `library-token-revoke NAME`
|
||||
(server/token also via `ASSETMGR_LIBRARY_URL` / `ASSETMGR_LIBRARY_TOKEN`).
|
||||
Files above 64 MiB (or `--chunked`) stream via resumable upload sessions.
|
||||
+43
-8
@@ -19,10 +19,26 @@ The API is token-authenticated (`Authorization: Bearer <token>`):
|
||||
- `POST /api/v1/assets?name=&kind=&importer=¶ms={}&tags=` (raw body)
|
||||
— publish; replies `{"id", "blob_sha256"}` (dedupe is by content hash)
|
||||
- `GET /api/v1/assets/<id>/blob` — download bytes
|
||||
- `GET /api/v1/assets/<id>/thumbnail` — 320px PNG preview, or 404
|
||||
`{"error": "no thumbnail"}` when the asset has none
|
||||
|
||||
Thumbnails render server-side at publish time (either ingest path) via
|
||||
ffmpeg/ImageMagick when a suitable engine is installed — best effort:
|
||||
unsupported formats, or a server with no engine at all, simply get no
|
||||
thumbnail and everything else works. Asset listings carry
|
||||
`thumbnail_sha256` (null when absent). Databases from older releases
|
||||
gain the column automatically on first open; existing rows keep serving
|
||||
with no thumbnail until re-published.
|
||||
|
||||
The same operations exist offline-first on the CLI:
|
||||
`assetmgr library-init|library-push|library-pull` (see `docs/API.md`).
|
||||
Uploads are capped at 2 GiB (`MAX_UPLOAD_BYTES`); no chunked upload yet.
|
||||
`assetmgr library-init|library-push|library-pull|library-thumbnail`
|
||||
(see `docs/API.md`).
|
||||
Uploads are capped at 2 GiB (`MAX_UPLOAD_BYTES`); for anything large use
|
||||
the session protocol — the CLI switches automatically above 64 MiB
|
||||
(`--chunk-above BYTES`), or force it with `library-push --chunked
|
||||
[--chunk-size BYTES]`. Sessions stream in 8 MiB blocks (the server never
|
||||
holds a whole file in RAM), each attempt gets a fresh session id, and
|
||||
abandoned sessions are swept on server start.
|
||||
|
||||
## Connect Godot
|
||||
|
||||
@@ -30,7 +46,13 @@ Uploads are capped at 2 GiB (`MAX_UPLOAD_BYTES`); no chunked upload yet.
|
||||
2. Project → Project Settings → Plugins → enable **Asset Library**.
|
||||
3. Open the dock (left), enter server URL + token, Search, select, Download.
|
||||
Files land in `res://<name>` and the editor filesystem rescans, so
|
||||
`.import` sidecars generate on the spot.
|
||||
`.import` sidecars generate on the spot. Selecting a result also shows
|
||||
its server-rendered thumbnail in the preview pane (blank when the
|
||||
asset has none).
|
||||
4. The web UI (native binary / Docker) has a Library section that
|
||||
searches the same catalog over HTTP (the server sends
|
||||
`Access-Control-Allow-Origin: *`; auth still required) and shows the
|
||||
thumbnails inline.
|
||||
|
||||
Publish the other way with the `godot::publish-to-library` action
|
||||
(importer + params travel in the upload), or plain
|
||||
@@ -38,10 +60,23 @@ Publish the other way with the `godot::publish-to-library` action
|
||||
|
||||
## Security notes (read before exposing this)
|
||||
|
||||
- Token auth only, **no TLS, no users, no rate limiting**. Safe on a LAN /
|
||||
VPN or behind a reverse proxy that adds TLS — do not put 8081 directly
|
||||
on the internet.
|
||||
- Tokens live in `.env` (git-ignored; see `.env.example`). Rotate by
|
||||
changing the value and recreating the container.
|
||||
- Token auth: one token per teammate, created locally against the DB file
|
||||
(plaintext shown once, only digests stored).
|
||||
- Revocation is instant (`library-token-revoke`); revoked tokens 401.
|
||||
There are no roles — every live token reads and writes every file.
|
||||
- TLS is built in, off by default. Pass `--tls-cert/--tls-key`
|
||||
(or `ASSETMGR_LIBRARY_TLS_CERT/KEY`) with a PEM pair — self-signed is
|
||||
fine on a LAN, use a real certificate (or a reverse proxy that
|
||||
terminates for you) for anything internet-facing. Never serve plain
|
||||
HTTP past your own network: clients fail closed against a TLS port
|
||||
and vice versa.
|
||||
- Tokens live in `.env` (git-ignored; see `.env.example`) only as the
|
||||
*bootstrap* secret: on first start it seeds the `default` token, and is
|
||||
ignored once any token exists. Prefer per-user tokens after that.
|
||||
- Compose with TLS: mount your PEM pair and override the command, e.g.
|
||||
volumes `["./certs:/certs:ro"]`, command
|
||||
`["-m", "library", "--host", "0.0.0.0", "--port", "8081",
|
||||
"--tls-cert", "/certs/fullchain.pem", "--tls-key", "/certs/key.pem"]`.
|
||||
- Rotate by revoking the name and creating a fresh one.
|
||||
- The server trusts its operator: anyone with the token can read and
|
||||
write every asset.
|
||||
@@ -3,11 +3,20 @@ extends VBoxContainer
|
||||
## Dock panel: search the remote library, download into res://.
|
||||
|
||||
const CONFIG_PATH := "user://asset_library.cfg"
|
||||
const Client := preload("res://addons/asset_library/asset_client.gd")
|
||||
|
||||
var _items: Array = []
|
||||
var _http: HTTPRequest
|
||||
var _dl: HTTPRequest
|
||||
var _dl_name := ""
|
||||
var _thumb: HTTPRequest
|
||||
var _dl_name: String = ""
|
||||
var _url_field: LineEdit
|
||||
var _token_field: LineEdit
|
||||
var _query_field: LineEdit
|
||||
var _go_button: Button
|
||||
var _results: ItemList
|
||||
var _preview: TextureRect
|
||||
var _download_button: Button
|
||||
|
||||
|
||||
func _ready() -> void:
|
||||
@@ -17,38 +26,52 @@ func _ready() -> void:
|
||||
_dl = HTTPRequest.new()
|
||||
add_child(_dl)
|
||||
_dl.request_completed.connect(_on_blob_done)
|
||||
%Go.pressed.connect(_on_search)
|
||||
%Download.pressed.connect(_on_download)
|
||||
_thumb = HTTPRequest.new()
|
||||
add_child(_thumb)
|
||||
_thumb.request_completed.connect(_on_thumb_done)
|
||||
# Plain relative paths, not %-unique names: % resolution depends on
|
||||
# scene-owner bookkeeping that is absent when the panel is instanced
|
||||
# outside the editor (headless probes, runtime tools).
|
||||
_url_field = get_node("URL") as LineEdit
|
||||
_token_field = get_node("Token") as LineEdit
|
||||
_query_field = get_node("SearchRow/Query") as LineEdit
|
||||
_go_button = get_node("SearchRow/Go") as Button
|
||||
_results = get_node("Results") as ItemList
|
||||
_preview = get_node("Preview") as TextureRect
|
||||
_download_button = get_node("Download") as Button
|
||||
_go_button.pressed.connect(_on_search)
|
||||
_results.item_selected.connect(_on_preview_index)
|
||||
_download_button.pressed.connect(_on_download)
|
||||
_load_config()
|
||||
|
||||
|
||||
func _load_config() -> void:
|
||||
var cfg := ConfigFile.new()
|
||||
if cfg.load(CONFIG_PATH) == OK:
|
||||
%URL.text = str(cfg.get_value("server", "url", ""))
|
||||
%Token.text = str(cfg.get_value("server", "token", ""))
|
||||
_url_field.text = str(cfg.get_value("server", "url", ""))
|
||||
_token_field.text = str(cfg.get_value("server", "token", ""))
|
||||
|
||||
|
||||
func _save_config() -> void:
|
||||
var cfg := ConfigFile.new()
|
||||
cfg.set_value("server", "url", %URL.text)
|
||||
cfg.set_value("server", "token", %Token.text)
|
||||
cfg.set_value("server", "url", _url_field.text)
|
||||
cfg.set_value("server", "token", _token_field.text)
|
||||
cfg.save(CONFIG_PATH)
|
||||
|
||||
|
||||
func _api(path: String) -> String:
|
||||
return %URL.text.strip_edges().trim_suffix("/") + path
|
||||
return _url_field.text.strip_edges().trim_suffix("/") + path
|
||||
|
||||
|
||||
func _auth() -> PackedStringArray:
|
||||
return PackedStringArray(
|
||||
["Authorization: Bearer " + %Token.text])
|
||||
["Authorization: Bearer " + _token_field.text])
|
||||
|
||||
|
||||
func _on_search() -> void:
|
||||
_save_config()
|
||||
var url := AssetLibraryClient.search_url(
|
||||
_api("/api/v1/assets"), {"q": %Query.text})
|
||||
var url := Client.search_url(
|
||||
_api("/api/v1/assets"), {"q": _query_field.text})
|
||||
if _http.request(url, _auth()) != OK:
|
||||
push_error("asset-library: search request failed to start")
|
||||
|
||||
@@ -58,14 +81,14 @@ func _on_list_done(result: int, _code: int, _headers: PackedStringArray,
|
||||
if result != HTTPRequest.RESULT_SUCCESS:
|
||||
push_error("asset-library: search failed")
|
||||
return
|
||||
_items = AssetLibraryClient.parse_assets(body.get_string_from_utf8())
|
||||
%Results.clear()
|
||||
_items = Client.parse_assets(body.get_string_from_utf8())
|
||||
_results.clear()
|
||||
for item: Variant in _items:
|
||||
%Results.add_item(str((item as Dictionary).get("name", "?")))
|
||||
_results.add_item(str((item as Dictionary).get("name", "?")))
|
||||
|
||||
|
||||
func _on_download() -> void:
|
||||
var selected: PackedInt32Array = %Results.get_selected_items()
|
||||
var selected: PackedInt32Array = _results.get_selected_items()
|
||||
if selected.is_empty():
|
||||
return
|
||||
var item: Dictionary = _items[selected[0]]
|
||||
@@ -75,6 +98,28 @@ func _on_download() -> void:
|
||||
push_error("asset-library: download request failed to start")
|
||||
|
||||
|
||||
func _on_preview_index(index: int) -> void:
|
||||
if index < 0 or index >= _items.size():
|
||||
return
|
||||
var item: Dictionary = _items[index]
|
||||
var url := _api("/api/v1/assets/%d/thumbnail" % int(item.get("id", 0)))
|
||||
if _thumb.request(url, _auth()) != OK:
|
||||
push_error("asset-library: thumbnail request failed to start")
|
||||
|
||||
|
||||
func _on_thumb_done(result: int, code: int, _headers: PackedStringArray,
|
||||
body: PackedByteArray) -> void:
|
||||
if result != HTTPRequest.RESULT_SUCCESS or code != 200:
|
||||
_preview.texture = null
|
||||
return
|
||||
var image := Image.new()
|
||||
if image.load_png_from_buffer(body) != OK:
|
||||
push_warning("asset-library: thumbnail is not a PNG")
|
||||
_preview.texture = null
|
||||
return
|
||||
_preview.texture = ImageTexture.create_from_image(image)
|
||||
|
||||
|
||||
func _on_blob_done(result: int, code: int, _headers: PackedStringArray,
|
||||
body: PackedByteArray) -> void:
|
||||
if result != HTTPRequest.RESULT_SUCCESS or code != 200:
|
||||
|
||||
@@ -24,5 +24,10 @@ text = "Search"
|
||||
[node name="Results" type="ItemList" parent="." unique_name_in_owner=true]
|
||||
size_flags_vertical = 3
|
||||
|
||||
[node name="Preview" type="TextureRect" parent="." unique_name_in_owner=true]
|
||||
custom_minimum_size = Vector2(0, 128)
|
||||
expand_mode = 1
|
||||
stretch_mode = 5
|
||||
|
||||
[node name="Download" type="Button" parent="." unique_name_in_owner=true]
|
||||
text = "Download selected"
|
||||
+25
-8
@@ -1,6 +1,10 @@
|
||||
"""Run the self-hosted asset library server.
|
||||
|
||||
Token via ``--token`` or ``ASSETMGR_LIBRARY_TOKEN`` (required).
|
||||
First run: create a token, then start (or pass --token once to bootstrap
|
||||
the ``default`` token)::
|
||||
|
||||
python -m assetmgr.cli library-token-create alice --db library-data/library.db
|
||||
python -m library --store library-data/store --db library-data/library.db
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -22,21 +26,34 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--db", default="library-data/library.db",
|
||||
help="metadata database file")
|
||||
parser.add_argument("--token", default="",
|
||||
help="bearer token (or ASSETMGR_LIBRARY_TOKEN)")
|
||||
help="bootstrap the 'default' token on first run "
|
||||
"(or ASSETMGR_LIBRARY_TOKEN)")
|
||||
parser.add_argument("--max-upload-bytes", type=int,
|
||||
default=DEFAULT_MAX_UPLOAD_BYTES)
|
||||
parser.add_argument("--tls-cert", default="",
|
||||
help="TLS certificate file (enables HTTPS;"
|
||||
" or ASSETMGR_LIBRARY_TLS_CERT)")
|
||||
parser.add_argument("--tls-key", default="",
|
||||
help="TLS private key file"
|
||||
" (or ASSETMGR_LIBRARY_TLS_KEY)")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
token = args.token or os.environ.get("ASSETMGR_LIBRARY_TOKEN", "")
|
||||
if not token:
|
||||
print("error: library token required (--token or ASSETMGR_LIBRARY_TOKEN)",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
server = LibraryServer(args.store, args.db, token,
|
||||
max_upload_bytes=args.max_upload_bytes)
|
||||
tls_cert = args.tls_cert or os.environ.get("ASSETMGR_LIBRARY_TLS_CERT", "")
|
||||
tls_key = args.tls_key or os.environ.get("ASSETMGR_LIBRARY_TLS_KEY", "")
|
||||
server = LibraryServer(args.store, args.db, token or None,
|
||||
max_upload_bytes=args.max_upload_bytes,
|
||||
tls_cert=tls_cert or None,
|
||||
tls_key=tls_key or None)
|
||||
from library import db as library_db
|
||||
names = [t["name"] for t in library_db.list_tokens(server.conn)]
|
||||
if not names:
|
||||
print("warning: no API tokens exist; create one with "
|
||||
"assetmgr library-token-create <name> --db "
|
||||
+ args.db, file=sys.stderr)
|
||||
print(f"asset-library on http://{args.host}:{args.port}"
|
||||
f" (store {args.store})", flush=True)
|
||||
server.serve_forever(args.host, args.port)
|
||||
|
||||
+34
-3
@@ -1,19 +1,31 @@
|
||||
"""Content-addressed blob store (SHA-256, `ab/cd/rest` fan-out)."""
|
||||
"""Content-addressed blob store (SHA-256, ``ab/cd/rest`` fan-out)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
|
||||
__all__ = ["put", "get_path"]
|
||||
__all__ = ["put", "put_stream", "get_path"]
|
||||
|
||||
|
||||
def _shard(root: Path, digest: str) -> Path:
|
||||
return root / digest[:2] / digest[2:4] / digest[4:]
|
||||
|
||||
|
||||
def _finalize(root: Path, digest: str, source: Path) -> Path:
|
||||
target = _shard(root, digest)
|
||||
if not target.exists():
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
source.replace(target)
|
||||
else:
|
||||
source.unlink()
|
||||
return target
|
||||
|
||||
|
||||
def put(root: str | Path, data: bytes) -> str:
|
||||
"""Store *data*, returning its SHA-256 hex digest (idempotent)."""
|
||||
"""Store *data*; returns its SHA-256 hex digest (idempotent)."""
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
target = _shard(Path(root), digest)
|
||||
if not target.exists():
|
||||
@@ -22,6 +34,25 @@ def put(root: str | Path, data: bytes) -> str:
|
||||
return digest
|
||||
|
||||
|
||||
def put_stream(root: str | Path, stream: BinaryIO) -> str:
|
||||
"""Store stream contents without holding them in RAM; returns digest.
|
||||
|
||||
Streams through a temp file, then moves it into place atomically.
|
||||
"""
|
||||
root = Path(root)
|
||||
digestor = hashlib.sha256()
|
||||
with tempfile.NamedTemporaryFile(dir=str(root), delete=False) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
while True:
|
||||
block = stream.read(1 << 20)
|
||||
if not block:
|
||||
break
|
||||
digestor.update(block)
|
||||
tmp.write(block)
|
||||
_finalize(root, digestor.hexdigest(), tmp_path)
|
||||
return digestor.hexdigest()
|
||||
|
||||
|
||||
def get_path(root: str | Path, digest: str) -> Path:
|
||||
"""Filesystem path of a stored blob; raises FileNotFoundError if absent."""
|
||||
target = _shard(Path(root), digest)
|
||||
|
||||
+79
-3
@@ -8,7 +8,13 @@ import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
__all__ = ["LibraryError", "push", "pull"]
|
||||
__all__ = ["LibraryError", "push", "pull", "pull_thumbnail",
|
||||
"upload_status"]
|
||||
|
||||
#: Files at or above this size upload through the session protocol.
|
||||
CHUNK_ABOVE_BYTES = 64 << 20
|
||||
#: Streaming block size for session uploads.
|
||||
CHUNK_SIZE_BYTES = 8 << 20
|
||||
|
||||
|
||||
class LibraryError(RuntimeError):
|
||||
@@ -47,10 +53,22 @@ def _request(server: str, token: str, method: str, path: str, *,
|
||||
def push(server: str, token: str, file_path: str, *,
|
||||
name: str | None = None, kind: str = "raw",
|
||||
importer: str | None = None, params: dict | None = None,
|
||||
tags: list[str] | tuple = ()) -> dict:
|
||||
"""Upload *file_path*; returns the server's ``{"id", "blob_sha256"}``."""
|
||||
tags: list[str] | tuple = (),
|
||||
chunk_size: int = CHUNK_SIZE_BYTES,
|
||||
chunk_above: int = CHUNK_ABOVE_BYTES) -> dict:
|
||||
"""Upload *file_path*; returns the server's ``{"id", "blob_sha256"}``.
|
||||
|
||||
Files above *chunk_above* bytes stream through the resumable session
|
||||
protocol instead of one in-memory body.
|
||||
"""
|
||||
if not os.path.isfile(file_path):
|
||||
raise LibraryError(f"no such file: {file_path}")
|
||||
size = os.path.getsize(file_path)
|
||||
meta = {"kind": kind, "importer": importer,
|
||||
"params": params or {}, "tags": list(tags)}
|
||||
if size > chunk_above:
|
||||
return _push_session(server, token, file_path, size, meta,
|
||||
chunk_size)
|
||||
with open(file_path, "rb") as handle:
|
||||
data = handle.read()
|
||||
query: dict[str, str] = {"name": name or os.path.basename(file_path),
|
||||
@@ -77,3 +95,61 @@ def pull(server: str, token: str, asset_id: int, out_path: str) -> dict:
|
||||
with open(out_path, "wb") as handle:
|
||||
handle.write(body)
|
||||
return {"path": os.path.abspath(out_path), "size": len(body)}
|
||||
|
||||
|
||||
def pull_thumbnail(server: str, token: str, asset_id: int,
|
||||
out_path: str) -> dict:
|
||||
"""Download an asset's thumbnail PNG; raises LibraryError when absent."""
|
||||
status, body = _request(server, token, "GET",
|
||||
f"/api/v1/assets/{asset_id}/thumbnail")
|
||||
if status != 200:
|
||||
raise LibraryError(f"no thumbnail: HTTP {status}", status=status)
|
||||
with open(out_path, "wb") as handle:
|
||||
handle.write(body)
|
||||
return {"path": os.path.abspath(out_path), "size": len(body)}
|
||||
|
||||
|
||||
def upload_status(server: str, token: str, upload_id: str) -> dict:
|
||||
"""Current ``{"upload_id", "offset", "size"}`` for a session."""
|
||||
status, body = _request(server, token, "GET",
|
||||
f"/api/v1/uploads/{upload_id}")
|
||||
if status != 200:
|
||||
raise LibraryError(f"no such upload: {upload_id}", status=status)
|
||||
return json.loads(body.decode())
|
||||
|
||||
|
||||
def _push_session(server: str, token: str, file_path: str, size: int,
|
||||
meta: dict, chunk_size: int) -> dict:
|
||||
"""Stream *file_path* through init/chunks/complete; returns the asset."""
|
||||
name = os.path.basename(file_path)
|
||||
query = {"name": name, "kind": meta.get("kind") or "raw"}
|
||||
if meta.get("importer"):
|
||||
query["importer"] = meta["importer"]
|
||||
if meta.get("params"):
|
||||
query["params"] = json.dumps(meta["params"])
|
||||
if meta.get("tags"):
|
||||
query["tags"] = ",".join(meta["tags"])
|
||||
status, body = _request(server, token, "POST", "/api/v1/uploads",
|
||||
params=query, data=b"")
|
||||
if status != 201:
|
||||
raise LibraryError(f"upload init rejected: HTTP {status}",
|
||||
status=status)
|
||||
upload_id = json.loads(body.decode())["upload_id"]
|
||||
with open(file_path, "rb") as handle:
|
||||
while True:
|
||||
block = handle.read(max(1, chunk_size))
|
||||
if not block:
|
||||
break
|
||||
status, body = _request(
|
||||
server, token, "POST",
|
||||
f"/api/v1/uploads/{upload_id}/chunks", data=block)
|
||||
if status != 200:
|
||||
raise LibraryError(
|
||||
f"chunk rejected: HTTP {status}", status=status)
|
||||
status, body = _request(server, token, "POST",
|
||||
f"/api/v1/uploads/{upload_id}/complete")
|
||||
if status != 201:
|
||||
raise LibraryError(f"complete rejected: HTTP {status}", status=status)
|
||||
result = json.loads(body.decode())
|
||||
result["chunked"] = True
|
||||
return result
|
||||
+80
-8
@@ -1,13 +1,17 @@
|
||||
"""SQLite metadata for library assets (stdlib only)."""
|
||||
"""SQLite metadata for library assets and API tokens (stdlib only)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
__all__ = ["open_db", "add_asset", "get_asset", "list_assets"]
|
||||
__all__ = ["open_db", "add_asset", "get_asset", "list_assets",
|
||||
"create_token", "revoke_token", "list_tokens", "check_token"]
|
||||
|
||||
SCHEMA = """\
|
||||
CREATE TABLE IF NOT EXISTS assets(
|
||||
@@ -19,9 +23,14 @@ CREATE TABLE IF NOT EXISTS assets(
|
||||
godot_importer TEXT,
|
||||
params_json TEXT NOT NULL DEFAULT '{}',
|
||||
tags_json TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
thumbnail_sha256 TEXT NOT NULL DEFAULT '');
|
||||
CREATE INDEX IF NOT EXISTS idx_assets_kind ON assets(kind);
|
||||
CREATE TABLE IF NOT EXISTS api_tokens(
|
||||
token_sha256 TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
revoked INTEGER NOT NULL DEFAULT 0);
|
||||
"""
|
||||
|
||||
|
||||
@@ -31,9 +40,19 @@ def open_db(path: str | Path, *,
|
||||
conn = sqlite3.connect(str(path), check_same_thread=check_same_thread)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA)
|
||||
_migrate(conn)
|
||||
return conn
|
||||
|
||||
|
||||
def _migrate(conn: sqlite3.Connection) -> None:
|
||||
"""Bring older databases up to the current schema (additive only)."""
|
||||
cols = [row[1] for row in conn.execute("PRAGMA table_info(assets)")]
|
||||
if "thumbnail_sha256" not in cols:
|
||||
conn.execute("ALTER TABLE assets ADD COLUMN"
|
||||
" thumbnail_sha256 TEXT NOT NULL DEFAULT ''")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
@@ -45,6 +64,7 @@ def _row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
|
||||
"params": json.loads(row["params_json"]),
|
||||
"tags": json.loads(row["tags_json"]),
|
||||
"created_at": row["created_at"],
|
||||
"thumbnail_sha256": row["thumbnail_sha256"] or None,
|
||||
}
|
||||
|
||||
|
||||
@@ -52,15 +72,17 @@ def add_asset(conn: sqlite3.Connection, *, name: str, blob_sha256: str,
|
||||
size_bytes: int, kind: str = "raw",
|
||||
godot_importer: str | None = None,
|
||||
params: dict | None = None,
|
||||
tags: list | None = None) -> int:
|
||||
tags: list | None = None,
|
||||
thumbnail_sha256: str = "") -> int:
|
||||
"""Insert one asset row; returns its id."""
|
||||
placeholders = ",".join(["?"] * 7)
|
||||
placeholders = ",".join(["?"] * 8)
|
||||
cur = conn.execute(
|
||||
"INSERT INTO assets(name, blob_sha256, size_bytes, kind,"
|
||||
" godot_importer, params_json, tags_json)"
|
||||
" godot_importer, params_json, tags_json, thumbnail_sha256)"
|
||||
f" VALUES ({placeholders})",
|
||||
(name, blob_sha256, size_bytes, kind, godot_importer,
|
||||
json.dumps(params or {}), json.dumps(tags or [])))
|
||||
json.dumps(params or {}), json.dumps(tags or []),
|
||||
thumbnail_sha256))
|
||||
conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
@@ -93,3 +115,53 @@ def list_assets(conn: sqlite3.Connection, *, q: str = "",
|
||||
if tag:
|
||||
rows = [r for r in rows if tag in r["tags"]]
|
||||
return rows
|
||||
|
||||
|
||||
def _token_digest(plaintext: str) -> str:
|
||||
return hashlib.sha256(plaintext.encode()).hexdigest()
|
||||
|
||||
|
||||
def create_token(conn: sqlite3.Connection, name: str) -> str:
|
||||
"""Create an API token for *name*; returns the plaintext (shown once).
|
||||
|
||||
Only the SHA-256 digest is stored. Raises ValueError on duplicate name.
|
||||
"""
|
||||
plaintext = secrets.token_urlsafe(32)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO api_tokens(token_sha256, name) VALUES (?, ?)",
|
||||
(_token_digest(plaintext), name))
|
||||
except sqlite3.IntegrityError:
|
||||
raise ValueError(f"token {name!r} already exists") from None
|
||||
conn.commit()
|
||||
return plaintext
|
||||
|
||||
|
||||
def revoke_token(conn: sqlite3.Connection, name: str) -> bool:
|
||||
"""Revoke *name*'s token; True when something was revoked."""
|
||||
cur = conn.execute("UPDATE api_tokens SET revoked=1 WHERE name=?",
|
||||
(name,))
|
||||
conn.commit()
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
def check_token(conn: sqlite3.Connection, presented: str) -> bool:
|
||||
"""Whether *presented* matches a live (non-revoked) token."""
|
||||
if not presented:
|
||||
return False
|
||||
digest = _token_digest(presented)
|
||||
stored = [r[0] for r in
|
||||
conn.execute("SELECT token_sha256 FROM api_tokens"
|
||||
" WHERE revoked=0")]
|
||||
return any(hmac.compare_digest(digest, candidate) for candidate in stored)
|
||||
|
||||
|
||||
def list_tokens(conn: sqlite3.Connection,
|
||||
include_revoked: bool = False) -> list[dict[str, Any]]:
|
||||
"""Token metadata (digests, never plaintext), oldest first."""
|
||||
sql = "SELECT name, token_sha256, created_at, revoked FROM api_tokens"
|
||||
if not include_revoked:
|
||||
sql += " WHERE revoked=0"
|
||||
return [{"name": r[0], "token_sha256": r[1], "created_at": r[2],
|
||||
"revoked": bool(r[3])}
|
||||
for r in conn.execute(sql + " ORDER BY rowid")]
|
||||
+354
-16
@@ -7,20 +7,36 @@ Every route except ``GET /health`` requires
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import ssl
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from apsync import generate_thumbnail as _render_thumbnail
|
||||
from library import blobs, db
|
||||
|
||||
__all__ = ["LibraryServer", "DEFAULT_MAX_UPLOAD_BYTES"]
|
||||
__all__ = ["LibraryServer", "DEFAULT_MAX_UPLOAD_BYTES", "UploadTooLargeError"]
|
||||
|
||||
#: Refuse single uploads above this (documented cap, not chunked upload).
|
||||
DEFAULT_MAX_UPLOAD_BYTES = 2 ** 31
|
||||
|
||||
#: Sessions untouched this long are deleted on server start.
|
||||
STALE_UPLOAD_SECONDS = 24 * 3600
|
||||
|
||||
#: Upload ids are URL-safe tokens minted by the server.
|
||||
_ID_RE = re.compile(r"[A-Za-z0-9_-]{1,64}\Z")
|
||||
|
||||
|
||||
class UploadTooLargeError(Exception):
|
||||
"""A chunk would push the upload over the server cap."""
|
||||
|
||||
|
||||
def _make_handler(lib: "LibraryServer") -> type[BaseHTTPRequestHandler]:
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
@@ -29,6 +45,19 @@ def _make_handler(lib: "LibraryServer") -> type[BaseHTTPRequestHandler]:
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
pass # keep test output clean; front with a real server for logs
|
||||
|
||||
def end_headers(self) -> None:
|
||||
# Wide-open CORS: this is a self-hosted LAN tool whose web UI
|
||||
# lives on another origin (assetmgrd). Auth still required.
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Headers",
|
||||
"Authorization, Content-Type")
|
||||
super().end_headers()
|
||||
|
||||
def do_OPTIONS(self) -> None:
|
||||
self.send_response(204)
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.end_headers()
|
||||
|
||||
# -- helpers ----------------------------------------------------
|
||||
def _json(self, code: int, payload: object) -> None:
|
||||
body = json.dumps(payload).encode()
|
||||
@@ -42,8 +71,9 @@ def _make_handler(lib: "LibraryServer") -> type[BaseHTTPRequestHandler]:
|
||||
if self.path == "/health" or self.command == "OPTIONS":
|
||||
return True
|
||||
presented = self.headers.get("Authorization", "")
|
||||
expected = f"Bearer {lib.token}"
|
||||
if hmac.compare_digest(presented, expected):
|
||||
if presented.startswith("Bearer "):
|
||||
presented = presented[len("Bearer "):]
|
||||
if lib.check_token(presented):
|
||||
return True
|
||||
self._json(401, {"error": "unauthorized"})
|
||||
return False
|
||||
@@ -86,25 +116,61 @@ def _make_handler(lib: "LibraryServer") -> type[BaseHTTPRequestHandler]:
|
||||
self.end_headers()
|
||||
self.wfile.write(blob)
|
||||
return
|
||||
if path.startswith("/api/v1/assets/") and path.endswith("/thumbnail"):
|
||||
self._serve_thumbnail(path)
|
||||
return
|
||||
parts = path.strip("/").split("/")
|
||||
if (len(parts) == 4 and parts[:3] == ["api", "v1", "uploads"]
|
||||
and _ID_RE.fullmatch(parts[3] or "")):
|
||||
try:
|
||||
self._json(200, lib.upload_status(parts[3]))
|
||||
except KeyError:
|
||||
self._json(404, {"error": "no such upload"})
|
||||
return
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if not self._authorized():
|
||||
return
|
||||
if urlparse(self.path).path != "/api/v1/assets":
|
||||
self._json(404, {"error": "not found"})
|
||||
path = urlparse(self.path).path
|
||||
if path == "/api/v1/assets":
|
||||
self._handle_single_upload()
|
||||
return
|
||||
if path == "/api/v1/uploads":
|
||||
self._handle_upload_init()
|
||||
return
|
||||
parts = path.strip("/").split("/")
|
||||
if (len(parts) == 5 and parts[:3] == ["api", "v1", "uploads"]
|
||||
and _ID_RE.fullmatch(parts[3])):
|
||||
if parts[4] == "chunks":
|
||||
self._handle_chunk(parts[3])
|
||||
elif parts[4] == "complete":
|
||||
self._handle_complete(parts[3])
|
||||
else:
|
||||
self._json(404, {"error": "not found"})
|
||||
return
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
def _body(self, cap: int | None = None) -> bytes:
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
length = 0
|
||||
if length <= 0:
|
||||
if length < 0:
|
||||
length = 0
|
||||
if cap is not None and length > cap:
|
||||
self._json(413, {"error": "upload too large"})
|
||||
raise _BodyTooLarge
|
||||
return self.rfile.read(length)
|
||||
|
||||
def _handle_single_upload(self) -> None:
|
||||
try:
|
||||
data = self._body(cap=lib.max_upload_bytes)
|
||||
except _BodyTooLarge:
|
||||
return
|
||||
if not data:
|
||||
self._json(400, {"error": "empty body"})
|
||||
return
|
||||
if length > lib.max_upload_bytes:
|
||||
self._json(413, {"error": "upload too large"})
|
||||
return
|
||||
data = self.rfile.read(length)
|
||||
query = self._query()
|
||||
name = query.get("name") or "upload.bin"
|
||||
try:
|
||||
@@ -116,33 +182,194 @@ def _make_handler(lib: "LibraryServer") -> type[BaseHTTPRequestHandler]:
|
||||
return
|
||||
tags = [t for t in query.get("tags", "").split(",") if t]
|
||||
digest = blobs.put(lib.store_root, data)
|
||||
thumb = lib.thumbnail_for_bytes(data, name)
|
||||
asset_id = lib.add_asset(
|
||||
name=name, blob_sha256=digest,
|
||||
size_bytes=len(data), kind=query.get("kind", "raw"),
|
||||
godot_importer=query.get("importer") or None,
|
||||
params=params, tags=tags)
|
||||
params=params, tags=tags,
|
||||
thumbnail_sha256=thumb or "")
|
||||
self._json(201, {"id": asset_id, "blob_sha256": digest})
|
||||
|
||||
def _upload_meta(self) -> tuple[dict, dict | None]:
|
||||
query = self._query()
|
||||
try:
|
||||
params = json.loads(query.get("params", "{}"))
|
||||
if not isinstance(params, dict):
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
params = None
|
||||
return query, params
|
||||
|
||||
def _serve_thumbnail(self, path: str) -> None:
|
||||
try:
|
||||
asset_id = int(path.rsplit("/", 2)[-2])
|
||||
row = lib.get_asset(asset_id)
|
||||
except (ValueError, KeyError):
|
||||
self._json(404, {"error": "no such asset"})
|
||||
return
|
||||
digest = row.get("thumbnail_sha256")
|
||||
if not digest:
|
||||
self._json(404, {"error": "no thumbnail"})
|
||||
return
|
||||
try:
|
||||
blob = blobs.get_path(lib.store_root,
|
||||
str(digest)).read_bytes()
|
||||
except FileNotFoundError:
|
||||
self._json(404, {"error": "thumbnail missing"})
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "image/png")
|
||||
self.send_header("Content-Length", str(len(blob)))
|
||||
self.end_headers()
|
||||
self.wfile.write(blob)
|
||||
|
||||
def _handle_upload_init(self) -> None:
|
||||
query, params = self._upload_meta()
|
||||
if params is None:
|
||||
self._json(400, {"error": "params must be a JSON object"})
|
||||
return
|
||||
try:
|
||||
upload_id = lib.create_upload(
|
||||
name=query.get("name") or "upload.bin",
|
||||
kind=query.get("kind", "raw"),
|
||||
importer=query.get("importer") or None,
|
||||
params=params,
|
||||
tags=[t for t in query.get("tags", "").split(",") if t])
|
||||
except OSError as exc:
|
||||
self._json(500, {"error": str(exc)})
|
||||
return
|
||||
self._json(201, {"upload_id": upload_id})
|
||||
|
||||
def _handle_chunk(self, upload_id: str) -> None:
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
length = 0
|
||||
if length < 0:
|
||||
length = 0
|
||||
try:
|
||||
if lib.upload_would_exceed(upload_id, length):
|
||||
lib.abort_upload(upload_id)
|
||||
self._json(413, {"error": "upload too large"})
|
||||
return
|
||||
data = self.rfile.read(length) if length else b""
|
||||
try:
|
||||
offset = lib.append_chunk(upload_id, data)
|
||||
except UploadTooLargeError:
|
||||
self._json(413, {"error": "upload too large"})
|
||||
return
|
||||
except KeyError:
|
||||
self._json(404, {"error": "no such upload"})
|
||||
return
|
||||
self._json(200, {"upload_id": upload_id, "offset": offset})
|
||||
|
||||
def _handle_complete(self, upload_id: str) -> None:
|
||||
try:
|
||||
result = lib.complete_upload(upload_id)
|
||||
except KeyError:
|
||||
self._json(404, {"error": "no such upload"})
|
||||
return
|
||||
self._json(201, result)
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
class _BodyTooLarge(Exception):
|
||||
"""Internal control flow: body already rejected with 413."""
|
||||
|
||||
|
||||
class LibraryServer:
|
||||
"""A self-hosted asset library: blob store + metadata + HTTP API."""
|
||||
|
||||
def __init__(self, store_root, db_path, token: str,
|
||||
max_upload_bytes: int = DEFAULT_MAX_UPLOAD_BYTES) -> None:
|
||||
def __init__(self, store_root, db_path, token: str | None = None,
|
||||
max_upload_bytes: int = DEFAULT_MAX_UPLOAD_BYTES,
|
||||
tls_cert: str | None = None,
|
||||
tls_key: str | None = None) -> None:
|
||||
if bool(tls_cert) != bool(tls_key):
|
||||
raise ValueError("tls_cert and tls_key must be given together")
|
||||
for path in (tls_cert, tls_key):
|
||||
if path and not os.path.isfile(path):
|
||||
raise FileNotFoundError(f"TLS file not found: {path}")
|
||||
self.tls_cert = tls_cert
|
||||
self.tls_key = tls_key
|
||||
os.makedirs(store_root, exist_ok=True)
|
||||
os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True)
|
||||
self.store_root = store_root
|
||||
self.store_root = str(store_root)
|
||||
self.uploads_dir = os.path.join(
|
||||
os.path.dirname(os.path.abspath(store_root)), "uploads")
|
||||
os.makedirs(self.uploads_dir, exist_ok=True)
|
||||
self._sweep_stale_uploads()
|
||||
# One shared connection: handler threads are serialized by this lock.
|
||||
self.conn = db.open_db(db_path, check_same_thread=False)
|
||||
self._lock = threading.Lock()
|
||||
self.token = token
|
||||
if token:
|
||||
self._ensure_bootstrap_token(token)
|
||||
self.max_upload_bytes = max_upload_bytes
|
||||
self.ready = threading.Event()
|
||||
self.port = 0
|
||||
self._httpd: ThreadingHTTPServer | None = None
|
||||
|
||||
def _ensure_bootstrap_token(self, token: str) -> None:
|
||||
"""Seed the ``default`` token on first run (back-compat single secret)."""
|
||||
import hashlib as _hashlib
|
||||
|
||||
with self._lock:
|
||||
exists = self.conn.execute(
|
||||
"SELECT 1 FROM api_tokens WHERE name='default'").fetchone()
|
||||
if exists is None:
|
||||
self.conn.execute(
|
||||
"INSERT INTO api_tokens(token_sha256, name)"
|
||||
" VALUES (?, 'default')",
|
||||
(_hashlib.sha256(token.encode()).hexdigest(),))
|
||||
self.conn.commit()
|
||||
|
||||
def check_token(self, presented: str) -> bool:
|
||||
"""Thread-safe :func:`library.db.check_token`."""
|
||||
with self._lock:
|
||||
return db.check_token(self.conn, presented)
|
||||
|
||||
def thumbnail_for_file(self, source_path: str) -> str | None:
|
||||
"""Best-effort PNG thumbnail stored as a blob; None when impossible.
|
||||
|
||||
Requires an external engine (ffmpeg/ImageMagick); anything failing
|
||||
— missing engine, unsupported format — yields None, never raises.
|
||||
"""
|
||||
out = ""
|
||||
try:
|
||||
fd, out = tempfile.mkstemp(prefix="thumb-",
|
||||
suffix=".png",
|
||||
dir=self.uploads_dir)
|
||||
os.close(fd)
|
||||
_render_thumbnail(source_path, out)
|
||||
with open(out, "rb") as handle:
|
||||
digest = blobs.put_stream(self.store_root, handle)
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if out:
|
||||
try:
|
||||
os.unlink(out)
|
||||
except OSError:
|
||||
pass
|
||||
return digest
|
||||
|
||||
def thumbnail_for_bytes(self, data: bytes, name_hint: str) -> str | None:
|
||||
"""Like :meth:`thumbnail_for_file` for in-memory uploads."""
|
||||
_fd, src = tempfile.mkstemp(
|
||||
prefix="upload-",
|
||||
suffix=os.path.splitext(name_hint)[1][:8] or ".bin",
|
||||
dir=self.uploads_dir)
|
||||
try:
|
||||
with open(src, "wb") as handle:
|
||||
handle.write(data)
|
||||
return self.thumbnail_for_file(src)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(src)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def list_assets(self, **filters) -> list:
|
||||
"""Thread-safe :func:`library.db.list_assets`."""
|
||||
with self._lock:
|
||||
@@ -158,10 +385,121 @@ class LibraryServer:
|
||||
with self._lock:
|
||||
return db.add_asset(self.conn, **fields)
|
||||
|
||||
# -- chunked upload sessions --------------------------------------
|
||||
def _session_paths(self, upload_id: str) -> tuple[str, str]:
|
||||
if not _ID_RE.fullmatch(upload_id or ""):
|
||||
raise KeyError(upload_id)
|
||||
session = os.path.join(self.uploads_dir, upload_id)
|
||||
if not os.path.isdir(session):
|
||||
raise KeyError(upload_id)
|
||||
return session, os.path.join(session, "data.bin")
|
||||
|
||||
@staticmethod
|
||||
def _read_meta(session: str) -> dict:
|
||||
with open(os.path.join(session, "meta.json"),
|
||||
encoding="utf-8") as handle:
|
||||
meta = json.load(handle)
|
||||
return meta if isinstance(meta, dict) else {}
|
||||
|
||||
def _sweep_stale_uploads(self) -> None:
|
||||
"""Delete abandoned session dirs older than a day (startup only)."""
|
||||
now = time.time()
|
||||
try:
|
||||
entries = os.listdir(self.uploads_dir)
|
||||
except OSError:
|
||||
return
|
||||
for entry in entries:
|
||||
path = os.path.join(self.uploads_dir, entry)
|
||||
try:
|
||||
if (os.path.isdir(path) and not os.path.islink(path)
|
||||
and now - os.path.getmtime(path) > STALE_UPLOAD_SECONDS):
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
def create_upload(self, *, name: str, kind: str = "raw",
|
||||
importer: str | None = None,
|
||||
params: dict | None = None,
|
||||
tags: list | None = None) -> str:
|
||||
"""Begin a session; returns its id. Raises OSError on FS trouble."""
|
||||
upload_id = secrets.token_urlsafe(12)
|
||||
session = os.path.join(self.uploads_dir, upload_id)
|
||||
os.makedirs(session)
|
||||
with open(os.path.join(session, "meta.json"), "w",
|
||||
encoding="utf-8") as handle:
|
||||
json.dump({"name": name, "kind": kind, "importer": importer,
|
||||
"params": params or {}, "tags": tags or []}, handle)
|
||||
open(os.path.join(session, "data.bin"), "wb").close()
|
||||
return upload_id
|
||||
|
||||
def _session_size(self, upload_id: str) -> int:
|
||||
_session, data = self._session_paths(upload_id)
|
||||
return os.path.getsize(data)
|
||||
|
||||
def upload_would_exceed(self, upload_id: str, incoming: int) -> bool:
|
||||
"""Whether appending *incoming* bytes would breach the cap."""
|
||||
return self._session_size(upload_id) + incoming > self.max_upload_bytes
|
||||
|
||||
def append_chunk(self, upload_id: str, data: bytes) -> int:
|
||||
"""Append *data*; returns the new offset. KeyError if unknown."""
|
||||
with self._lock:
|
||||
_session, path = self._session_paths(upload_id)
|
||||
if os.path.getsize(path) + len(data) > self.max_upload_bytes:
|
||||
raise UploadTooLargeError(upload_id)
|
||||
with open(path, "ab") as handle:
|
||||
handle.write(data)
|
||||
os.utime(self._session_paths(upload_id)[0], None)
|
||||
return os.path.getsize(path)
|
||||
|
||||
def upload_status(self, upload_id: str) -> dict:
|
||||
"""Current ``{"upload_id", "offset", "size"}``; KeyError if unknown."""
|
||||
with self._lock:
|
||||
session, path = self._session_paths(upload_id)
|
||||
size = os.path.getsize(path)
|
||||
os.utime(session, None)
|
||||
return {"upload_id": upload_id, "offset": size, "size": size}
|
||||
|
||||
def abort_upload(self, upload_id: str) -> None:
|
||||
"""Drop a session and its bytes; unknown ids are ignored."""
|
||||
with self._lock:
|
||||
try:
|
||||
session, _data = self._session_paths(upload_id)
|
||||
except KeyError:
|
||||
return
|
||||
shutil.rmtree(session, ignore_errors=True)
|
||||
|
||||
def complete_upload(self, upload_id: str) -> dict:
|
||||
"""Assemble, store, index (+ best-effort thumbnail); returns the asset."""
|
||||
with self._lock:
|
||||
session, path = self._session_paths(upload_id)
|
||||
meta = self._read_meta(session)
|
||||
size = os.path.getsize(path)
|
||||
# Assemble outside the lock: session paths are per-upload, and
|
||||
# thumbnailing shells out to ffmpeg (slow). The DB write re-locks.
|
||||
with open(path, "rb") as stream:
|
||||
digest = blobs.put_stream(self.store_root, stream)
|
||||
thumb = self.thumbnail_for_file(path)
|
||||
with self._lock:
|
||||
asset_id = db.add_asset(
|
||||
self.conn, name=meta.get("name", "upload.bin"),
|
||||
blob_sha256=digest, size_bytes=size,
|
||||
kind=meta.get("kind", "raw"),
|
||||
godot_importer=meta.get("importer"),
|
||||
params=meta.get("params") or {},
|
||||
tags=meta.get("tags") or [],
|
||||
thumbnail_sha256=thumb or "")
|
||||
shutil.rmtree(session, ignore_errors=True)
|
||||
return {"id": asset_id, "blob_sha256": digest, "size": size}
|
||||
|
||||
def serve_forever(self, host: str = "127.0.0.1", port: int = 0) -> None:
|
||||
"""Block serving; ``port=0`` picks an ephemeral port (see .port)."""
|
||||
self._httpd = ThreadingHTTPServer((host, port),
|
||||
_make_handler(self))
|
||||
if self.tls_cert and self.tls_key:
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(self.tls_cert, self.tls_key)
|
||||
self._httpd.socket = context.wrap_socket(self._httpd.socket,
|
||||
server_side=True)
|
||||
self.port = self._httpd.server_address[1]
|
||||
self.ready.set()
|
||||
self._httpd.serve_forever()
|
||||
|
||||
@@ -48,6 +48,16 @@ pre { background: #0c0e11; padding: 12px; border-radius: 6px; overflow: auto; }
|
||||
<table><thead><tr><th>Path</th><th>Size</th><th>Import</th><th>Details</th><th>Preview</th></tr></thead>
|
||||
<tbody id="scanrows"><tr><td colspan="5" class="muted">No scan yet.</td></tr></tbody></table>
|
||||
|
||||
<h2>Library</h2>
|
||||
<div class="row">
|
||||
<input id="libserver" placeholder="http://library:8081" size="26">
|
||||
<input id="libtoken" placeholder="token" size="20" type="password">
|
||||
<input id="libquery" placeholder="search..." size="20">
|
||||
<button onclick="searchLibrary()">Search</button>
|
||||
</div>
|
||||
<table><thead><tr><th></th><th>ID</th><th>Name</th><th>Kind</th><th>Size</th><th>Tags</th></tr></thead>
|
||||
<tbody id="librows"><tr><td colspan="6" class="muted">Not connected.</td></tr></tbody></table>
|
||||
|
||||
<h2>Status</h2>
|
||||
<div class="row">
|
||||
<input id="statuspath" placeholder="/path/to/project" size="34">
|
||||
@@ -159,6 +169,72 @@ async function showThumb(root, resPath, cell, btn) {
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
function libBase() {
|
||||
return document.getElementById('libserver').value.replace(/\/+$/, '');
|
||||
}
|
||||
function libHeaders() {
|
||||
return {'Authorization':
|
||||
'Bearer ' + document.getElementById('libtoken').value};
|
||||
}
|
||||
async function searchLibrary() {
|
||||
const q = document.getElementById('libquery').value;
|
||||
const tb = document.getElementById('librows');
|
||||
tb.innerHTML = '';
|
||||
let r;
|
||||
try {
|
||||
r = await fetch(libBase() + '/api/v1/assets' +
|
||||
(q ? '?q=' + encodeURIComponent(q) : ''), {headers: libHeaders()});
|
||||
} catch (e) {
|
||||
tb.innerHTML = '<tr><td colspan="6" class="muted">Cannot reach library ' +
|
||||
'(check server URL; the library allows cross-origin requests, ' +
|
||||
'but https UI pages cannot fetch plain-http servers).</td></tr>';
|
||||
return;
|
||||
}
|
||||
if (!r.ok) {
|
||||
say({http: r.status, ...(await r.json().catch(() => ({})))});
|
||||
tb.innerHTML = '<tr><td colspan="6" class="muted">Request failed.</td></tr>';
|
||||
return;
|
||||
}
|
||||
const body = await r.json();
|
||||
for (const a of body.assets) {
|
||||
const tr = document.createElement('tr');
|
||||
const thumbTd = document.createElement('td');
|
||||
if (a.thumbnail_sha256) {
|
||||
loadLibThumb(a.id, thumbTd);
|
||||
} else {
|
||||
thumbTd.innerHTML = '<span class="muted">—</span>';
|
||||
}
|
||||
const idTd = document.createElement('td');
|
||||
idTd.textContent = a.id;
|
||||
const nameTd = document.createElement('td');
|
||||
nameTd.textContent = a.name;
|
||||
const kindTd = document.createElement('td');
|
||||
kindTd.textContent = a.kind;
|
||||
const sizeTd = document.createElement('td');
|
||||
sizeTd.textContent = fmtSize(a.size_bytes);
|
||||
sizeTd.style.whiteSpace = 'nowrap';
|
||||
const tagsTd = document.createElement('td');
|
||||
tagsTd.textContent = (a.tags || []).join(', ');
|
||||
tr.append(thumbTd, idTd, nameTd, kindTd, sizeTd, tagsTd);
|
||||
tb.appendChild(tr);
|
||||
}
|
||||
if (!body.assets.length) {
|
||||
tb.innerHTML = '<tr><td colspan="6" class="muted">No assets found.</td></tr>';
|
||||
}
|
||||
say({library: libBase(), assets: body.assets.length});
|
||||
}
|
||||
async function loadLibThumb(id, cell) {
|
||||
try {
|
||||
const r = await fetch(libBase() + '/api/v1/assets/' + id + '/thumbnail',
|
||||
{headers: libHeaders()});
|
||||
if (!r.ok) return;
|
||||
const img = document.createElement('img');
|
||||
img.style.maxWidth = '96px';
|
||||
img.loading = 'lazy';
|
||||
img.src = URL.createObjectURL(await r.blob());
|
||||
cell.replaceChildren(img);
|
||||
} catch (e) { /* leave the cell empty */ }
|
||||
}
|
||||
function fmtSize(n) {
|
||||
if (n < 1024) return n + ' B';
|
||||
if (n < 1048576) return (n / 1024).toFixed(1) + ' KB';
|
||||
|
||||
@@ -173,3 +173,70 @@ def test_library_push_needs_endpoint(tmp_path, capsys):
|
||||
code, payload = _run_json(capsys, "library-push", source)
|
||||
assert code == 2
|
||||
assert "server" in payload["error"]
|
||||
|
||||
|
||||
def test_library_push_chunked_roundtrip(tmp_path, capsys):
|
||||
from test_library_server import TOKEN, start_server
|
||||
_, base = start_server(tmp_path / "srv")
|
||||
source = tmp_path / "big.bin"
|
||||
source.write_bytes(b"q" * 100)
|
||||
code, payload = _run_json(
|
||||
capsys, "library-push", str(source), "--server", base,
|
||||
"--token", TOKEN, "--chunked", "--chunk-size", "7",
|
||||
"--kind", "raw")
|
||||
assert code == 0
|
||||
assert payload.get("chunked") is True
|
||||
dest = str(tmp_path / "back.bin")
|
||||
code, pulled = _run_json(capsys, "library-pull", str(payload["id"]),
|
||||
"--out", dest, "--server", base,
|
||||
"--token", TOKEN)
|
||||
assert code == 0 and pulled["size"] == 100
|
||||
assert open(dest, "rb").read() == b"q" * 100
|
||||
|
||||
|
||||
def test_library_thumbnail_roundtrip(tmp_path, capsys):
|
||||
import shutil
|
||||
from test_library_server import TOKEN, start_server
|
||||
if shutil.which("ffmpeg") is None and shutil.which("magick") is None:
|
||||
pytest.skip("no thumbnail engine on PATH")
|
||||
_, base = start_server(tmp_path / "srv")
|
||||
source = _write_png(tmp_path / "hero.png")
|
||||
code, created = _run_json(capsys, "library-push", source,
|
||||
"--server", base, "--token", TOKEN)
|
||||
assert code == 0
|
||||
out = str(tmp_path / "thumb.png")
|
||||
code, thumb = _run_json(capsys, "library-thumbnail",
|
||||
str(created["id"]), "--out", out,
|
||||
"--server", base, "--token", TOKEN)
|
||||
assert code == 0
|
||||
with open(out, "rb") as handle:
|
||||
assert handle.read(8) == b"\x89PNG\r\n\x1a\n"
|
||||
code, missing = _run_json(capsys, "library-thumbnail", "999",
|
||||
"--out", out, "--server", base,
|
||||
"--token", TOKEN)
|
||||
assert code == 1 and "404" in missing["error"]
|
||||
|
||||
|
||||
def test_library_token_lifecycle_against_server(tmp_path, capsys):
|
||||
from test_library_server import start_server
|
||||
db = str(tmp_path / "srv" / "lib.db")
|
||||
_, base = start_server(tmp_path / "srv")
|
||||
code, created = _run_json(capsys, "library-token-create", "carol",
|
||||
"--db", db)
|
||||
assert code == 0
|
||||
token = created["token"]
|
||||
code, listed = _run_json(capsys, "library-token-list", "--db", db)
|
||||
assert code == 0
|
||||
# "default" comes from the test server's bootstrap token
|
||||
assert {t["name"] for t in listed["tokens"]} == {"default", "carol"}
|
||||
# carol's token works against the live server sharing the same DB
|
||||
assert _run_json(capsys, "library-push", _write_png(tmp_path / "a.png"),
|
||||
"--server", base, "--token", token)[0] == 0
|
||||
code, _ = _run_json(capsys, "library-token-revoke", "carol",
|
||||
"--db", db)
|
||||
assert code == 0
|
||||
code, payload = _run_json(capsys, "library-push",
|
||||
_write_png(tmp_path / "b.png"),
|
||||
"--server", base, "--token", token)
|
||||
assert code == 1
|
||||
assert "401" in payload["error"]
|
||||
+30
-2
@@ -219,6 +219,20 @@ def test_array_with_quoted_commas():
|
||||
assert parsed["s"]["n"] == 2
|
||||
|
||||
|
||||
def test_font_import_value_shapes():
|
||||
parsed = parse_godot_config(
|
||||
"[params]\nRendering=null\nantialiasing=1\noversampling=0.0\n"
|
||||
"fallbacks=[]\npreload=[]\nlanguage_support={}\n"
|
||||
'scale_mesh=Vector3(1, 1, 1)\n')
|
||||
params = parsed["params"]
|
||||
assert params["Rendering"] == "null" # Godot null kept verbatim
|
||||
assert params["antialiasing"] == 1
|
||||
assert params["oversampling"] == 0.0
|
||||
assert params["fallbacks"] == []
|
||||
assert params["language_support"] == "{}"
|
||||
assert params["scale_mesh"] == "Vector3(1, 1, 1)"
|
||||
|
||||
|
||||
def test_quoted_key_lines_are_continuations():
|
||||
parsed = parse_godot_config('[s]\nmetadata={\n"a=b": true\n}\nnext=1\n')
|
||||
assert "a=b" in str(parsed["s"]["metadata"])
|
||||
@@ -241,7 +255,13 @@ def test_known_importer_params_are_harvested_not_guessed():
|
||||
wav = known_importer_params("wav")
|
||||
assert wav is not None
|
||||
assert len(wav) == 10 # real 4.3 output
|
||||
assert known_importer_params("scene") is None # unobserved: no verdict
|
||||
for importer, count in (("mp3", 5), ("oggvorbisstr", 5),
|
||||
("font_data_dynamic", 20),
|
||||
("wavefront_obj", 5), ("scene", 21)):
|
||||
params = known_importer_params(importer)
|
||||
assert params is not None
|
||||
assert len(params) == count, (importer, params)
|
||||
assert known_importer_params("not_a_real_importer") is None
|
||||
|
||||
|
||||
def test_validate_import_params():
|
||||
@@ -250,8 +270,16 @@ def test_validate_import_params():
|
||||
assert validate_import_params(
|
||||
"texture", {"compress/mode": 0, "loop_mode": 1}) == ["loop_mode"]
|
||||
assert validate_import_params("wav", {"edit/loop_mode": 1}) == []
|
||||
assert validate_import_params("mp3", {"loop": True}) == []
|
||||
assert validate_import_params("mp3", {"compress/mode": 0}) == ["compress/mode"]
|
||||
assert validate_import_params("font_data_dynamic", {"hinting": 2}) == []
|
||||
assert validate_import_params(
|
||||
"wavefront_obj", {"scale_mesh": "Vector3(1, 1, 1)"}) == []
|
||||
assert validate_import_params("scene", {"animation/fps": 30}) == []
|
||||
assert validate_import_params("scene", {"loop": True}) == ["loop"]
|
||||
# unknown importer: absence of data is not evidence of a bad key
|
||||
assert validate_import_params("scene", {"anything/at_all": 1}) == []
|
||||
assert validate_import_params(
|
||||
"not_a_real_importer", {"anything/at_all": 1}) == []
|
||||
|
||||
|
||||
def test_set_preset_warns_but_applies_unknown_keys(tmp_path, capsys):
|
||||
|
||||
@@ -70,3 +70,229 @@ def test_client_pure_functions_headless(tmp_path, godot_bin):
|
||||
assert "COUNT:1" in completed.stdout
|
||||
assert "NAME:hero.png" in completed.stdout
|
||||
assert "EMPTY:0" in completed.stdout
|
||||
|
||||
|
||||
# End-to-end: live library server -> Godot-side HTTP download into res://
|
||||
# (same HTTPClient + FileAccess stack the dock uses) -> headless --import
|
||||
# produces the editor cache. This is the project's core promise.
|
||||
E2E_DOWNLOAD_GD = """\
|
||||
extends SceneTree
|
||||
|
||||
func _fail(message: String) -> void:
|
||||
\tprint("E2E_FAIL:" + message)
|
||||
\tquit(1)
|
||||
|
||||
func _fetch(client: HTTPClient, host: String, port: int, path: String,
|
||||
\t\ttoken: String) -> PackedByteArray:
|
||||
\tvar err := client.connect_to_host(host, port)
|
||||
\tif err != OK:
|
||||
\t\t_fail("connect")
|
||||
\t\treturn PackedByteArray()
|
||||
\tvar deadline := Time.get_ticks_msec() + 15000
|
||||
\twhile client.get_status() == HTTPClient.STATUS_CONNECTING \\
|
||||
\t\t\tor client.get_status() == HTTPClient.STATUS_RESOLVING:
|
||||
\t\tclient.poll()
|
||||
\t\tOS.delay_msec(10)
|
||||
\t\tif Time.get_ticks_msec() > deadline:
|
||||
\t\t\t_fail("connect-timeout")
|
||||
\t\t\treturn PackedByteArray()
|
||||
\tif client.get_status() != HTTPClient.STATUS_CONNECTED:
|
||||
\t\t_fail("not-connected")
|
||||
\t\treturn PackedByteArray()
|
||||
\terr = client.request(HTTPClient.METHOD_GET, path,
|
||||
\t\tPackedStringArray(["Authorization: Bearer " + token]))
|
||||
\tif err != OK:
|
||||
\t\t_fail("request")
|
||||
\t\treturn PackedByteArray()
|
||||
\tdeadline = Time.get_ticks_msec() + 15000
|
||||
\twhile client.get_status() == HTTPClient.STATUS_REQUESTING:
|
||||
\t\tclient.poll()
|
||||
\t\tOS.delay_msec(10)
|
||||
\t\tif Time.get_ticks_msec() > deadline:
|
||||
\t\t\t_fail("request-timeout")
|
||||
\t\t\treturn PackedByteArray()
|
||||
\tif client.get_response_code() != 200:
|
||||
\t\t_fail("http-" + str(client.get_response_code()))
|
||||
\t\treturn PackedByteArray()
|
||||
\tvar body := PackedByteArray()
|
||||
\twhile client.get_status() == HTTPClient.STATUS_BODY:
|
||||
\t\tclient.poll()
|
||||
\t\tvar chunk := client.read_response_body_chunk()
|
||||
\t\tif chunk.size() > 0:
|
||||
\t\t\tbody.append_array(chunk)
|
||||
\t\tOS.delay_msec(10)
|
||||
\treturn body
|
||||
|
||||
func _init() -> void:
|
||||
\tvar args := OS.get_cmdline_user_args()
|
||||
\tif args.size() < 4:
|
||||
\t\t_fail("usage: host port token asset-id")
|
||||
\t\treturn
|
||||
\tvar host: String = args[0]
|
||||
\tvar port: int = int(args[1])
|
||||
\tvar token: String = args[2]
|
||||
\tvar wanted_id := int(args[3])
|
||||
\tvar client := HTTPClient.new()
|
||||
\tvar listing := _fetch(client, host, port, "/api/v1/assets", token)
|
||||
\tif listing.is_empty():
|
||||
\t\treturn # _fail already printed
|
||||
\tvar ClientScript = load("res://addons/asset_library/asset_client.gd")
|
||||
\tvar items: Array = ClientScript.parse_assets(listing.get_string_from_utf8())
|
||||
\tprint("E2E_FOUND:" + str(items.size()))
|
||||
\tvar target := {}
|
||||
\tfor item: Variant in items:
|
||||
\t\tif int((item as Dictionary).get("id", -1)) == wanted_id:
|
||||
\t\t\ttarget = item
|
||||
\tif target.is_empty():
|
||||
\t\t_fail("asset-missing")
|
||||
\t\treturn
|
||||
\tvar name: String = str(target.get("name", "download.bin"))
|
||||
\tvar blob := _fetch(client, host, port,
|
||||
\t\t"/api/v1/assets/%d/blob" % wanted_id, token)
|
||||
\tif blob.is_empty():
|
||||
\t\t_fail("empty-blob")
|
||||
\t\treturn
|
||||
\tvar dest := "res://" + name
|
||||
\tvar file := FileAccess.open(dest, FileAccess.WRITE)
|
||||
\tif file == null:
|
||||
\t\t_fail("write")
|
||||
\t\treturn
|
||||
\tfile.store_buffer(blob)
|
||||
\tfile.close()
|
||||
\tprint("E2E_SAVED:" + dest + ":" + str(blob.size()))
|
||||
\tquit()
|
||||
"""
|
||||
|
||||
|
||||
def test_full_loop_server_to_imported(tmp_path, godot_bin):
|
||||
import base64
|
||||
import sys
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from test_library_server import TOKEN, start_server
|
||||
|
||||
_, base = start_server(tmp_path / "srv")
|
||||
parts = urlparse(base)
|
||||
png = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==")
|
||||
source = tmp_path / "hero.png"
|
||||
source.write_bytes(png)
|
||||
|
||||
root = make_plugin_project(tmp_path)
|
||||
probe = os.path.join(root, "e2e_download.gd")
|
||||
with open(probe, "w", encoding="utf-8") as handle:
|
||||
handle.write(E2E_DOWNLOAD_GD)
|
||||
|
||||
# Publish through the real CLI (same path the action uses).
|
||||
pushed = subprocess.run(
|
||||
[sys.executable, "-m", "assetmgr.cli",
|
||||
"library-push", str(source), "--server", base, "--token", TOKEN,
|
||||
"--kind", "texture", "--importer", "texture", "--json"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
assert pushed.returncode == 0, pushed.stderr[-1000:]
|
||||
import json as _json
|
||||
asset_id = _json.loads(pushed.stdout)["id"]
|
||||
|
||||
# Godot side: search + download into res:// via Godot's own HTTP stack.
|
||||
dl = subprocess.run(
|
||||
[godot_bin, "--headless", "--path", root, "--script", probe, "--",
|
||||
parts.hostname, str(parts.port), TOKEN, str(asset_id)],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
assert dl.returncode == 0, dl.stderr[-2000:] + dl.stdout[-1000:]
|
||||
assert "E2E_FOUND:1" in dl.stdout, dl.stdout
|
||||
assert "E2E_SAVED:res://hero.png" in dl.stdout, dl.stdout
|
||||
assert "E2E_FAIL" not in dl.stdout
|
||||
|
||||
# Headless import must now produce the editor cache for the file.
|
||||
imported = subprocess.run(
|
||||
[godot_bin, "--headless", "--path", root, "--import"],
|
||||
capture_output=True, text=True, timeout=300)
|
||||
assert imported.returncode == 0, imported.stderr[-2000:]
|
||||
got = list((tmp_path / "plugproj").rglob("*.ctex"))
|
||||
assert got, "no imported output produced"
|
||||
assert os.path.isfile(os.path.join(root, "hero.png.import"))
|
||||
|
||||
|
||||
THUMB_PROBE_GD = """\
|
||||
extends SceneTree
|
||||
|
||||
var _panel
|
||||
var _phase := 0
|
||||
var _frames := 0
|
||||
var _base := ""
|
||||
var _token := ""
|
||||
var _want_id := 0
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
\tvar args := OS.get_cmdline_user_args()
|
||||
\t_base = str(args[0])
|
||||
\t_token = str(args[1])
|
||||
\t_want_id = int(args[2])
|
||||
\t_panel = load("res://addons/asset_library/asset_dock.tscn").instantiate()
|
||||
\troot.add_child(_panel)
|
||||
|
||||
|
||||
func _process(_delta: float) -> bool:
|
||||
\t_frames += 1
|
||||
\tif _frames > 900:
|
||||
\t\tprint("E2E_THUMB:timeout")
|
||||
\t\treturn true
|
||||
\tif _phase == 0:
|
||||
\t\tif _panel.get("_http") == null:
|
||||
\t\t\treturn false
|
||||
\t\t_panel.get_node("URL").text = _base
|
||||
\t\t_panel.get_node("Token").text = _token
|
||||
\t\t_panel.get_node("SearchRow/Query").text = ""
|
||||
\t\t_panel._on_search()
|
||||
\t\t_phase = 1
|
||||
\t\treturn false
|
||||
\tif _phase == 1 and _panel._items.size() > 0:
|
||||
\t\tvar first: Dictionary = _panel._items[0]
|
||||
\t\tif int(first.get("id", -1)) == _want_id:
|
||||
\t\t\t_panel._on_preview_index(0)
|
||||
\t\t\t_phase = 2
|
||||
\t\telse:
|
||||
\t\t\tprint("E2E_THUMB:wrong-id")
|
||||
\t\t\treturn true
|
||||
\telif _phase == 2:
|
||||
\t\tvar tex: Texture2D = _panel.get_node("Preview").texture
|
||||
\t\tif tex != null:
|
||||
\t\t\tprint("E2E_THUMB:ok %dx%d" % [tex.get_width(),
|
||||
\t\t\t\ttex.get_height()])
|
||||
\t\t\treturn true
|
||||
\treturn false
|
||||
"""
|
||||
|
||||
|
||||
def test_dock_shows_server_thumbnail(tmp_path, godot_bin):
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from test_library_server import TOKEN, start_server
|
||||
_, base = start_server(tmp_path / "srv")
|
||||
png = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==")
|
||||
source = tmp_path / "hero.png"
|
||||
source.write_bytes(png)
|
||||
root = make_plugin_project(tmp_path)
|
||||
probe = os.path.join(root, "thumb_probe.gd")
|
||||
with open(probe, "w", encoding="utf-8") as handle:
|
||||
handle.write(THUMB_PROBE_GD)
|
||||
# publish through the real CLI so importer metadata travels too
|
||||
done = subprocess.run(
|
||||
[sys.executable, "-m", "assetmgr.cli", "library-push",
|
||||
str(source), "--server", base, "--token", TOKEN,
|
||||
"--kind", "texture", "--importer", "texture", "--json"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
assert done.returncode == 0, done.stderr[-1000:]
|
||||
asset_id = json.loads(done.stdout)["id"]
|
||||
run = subprocess.run(
|
||||
[godot_bin, "--headless", "--path", root,
|
||||
"--script", probe, "--",
|
||||
base, TOKEN, str(asset_id)],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
assert run.returncode == 0, run.stderr[-2000:] + run.stdout[-1000:]
|
||||
assert "E2E_THUMB:ok" in run.stdout, run.stdout
|
||||
@@ -0,0 +1,35 @@
|
||||
"""API token tests (multi-user auth for the self-hosted library)."""
|
||||
|
||||
from library import db
|
||||
|
||||
|
||||
def test_create_and_check_token(tmp_path):
|
||||
conn = db.open_db(tmp_path / "lib.db")
|
||||
plaintext = db.create_token(conn, "alice")
|
||||
assert plaintext and isinstance(plaintext, str)
|
||||
assert db.check_token(conn, plaintext) is True
|
||||
assert db.check_token(conn, "wrong") is False
|
||||
assert db.check_token(conn, "") is False
|
||||
|
||||
|
||||
def test_tokens_are_stored_hashed(tmp_path):
|
||||
conn = db.open_db(tmp_path / "lib.db")
|
||||
plaintext = db.create_token(conn, "alice")
|
||||
rows = db.list_tokens(conn)
|
||||
assert [r["name"] for r in rows] == ["alice"]
|
||||
assert all(plaintext not in str(r) for r in rows)
|
||||
assert all("sha256" in r or len(r.get("token_sha256", "")) == 64
|
||||
for r in rows)
|
||||
|
||||
|
||||
def test_revoke_token(tmp_path):
|
||||
conn = db.open_db(tmp_path / "lib.db")
|
||||
alice = db.create_token(conn, "alice")
|
||||
bob = db.create_token(conn, "bob")
|
||||
assert db.revoke_token(conn, "alice") is True
|
||||
assert db.check_token(conn, alice) is False
|
||||
assert db.check_token(conn, bob) is True
|
||||
assert db.revoke_token(conn, "nobody") is False
|
||||
assert [r["name"] for r in db.list_tokens(conn, include_revoked=True)] == \
|
||||
["alice", "bob"]
|
||||
assert [r["name"] for r in db.list_tokens(conn)] == ["bob"]
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Library HTTP API tests (spins a real server on an ephemeral port)."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
@@ -52,6 +53,26 @@ def test_auth_required(tmp_path):
|
||||
assert call(base, "/api/v1/assets", token="wrong")[0] == 401
|
||||
|
||||
|
||||
def test_multiple_tokens_and_revocation(tmp_path):
|
||||
from library import db as library_db
|
||||
_, base = start_server(tmp_path)
|
||||
db_path = tmp_path / "lib.db"
|
||||
conn = library_db.open_db(str(db_path))
|
||||
try:
|
||||
second = library_db.create_token(conn, "second")
|
||||
finally:
|
||||
conn.close()
|
||||
assert call(base, "/api/v1/assets", token=TOKEN)[0] == 200
|
||||
assert call(base, "/api/v1/assets", token=second)[0] == 200
|
||||
conn = library_db.open_db(str(db_path))
|
||||
try:
|
||||
assert library_db.revoke_token(conn, "second") is True
|
||||
finally:
|
||||
conn.close()
|
||||
assert call(base, "/api/v1/assets", token=second)[0] == 401
|
||||
assert call(base, "/api/v1/assets", token=TOKEN)[0] == 200
|
||||
|
||||
|
||||
def test_upload_list_download_roundtrip(tmp_path):
|
||||
_, base = start_server(tmp_path)
|
||||
status, body = call(base, "/api/v1/assets", token=TOKEN, data=b"png-bytes",
|
||||
@@ -80,3 +101,42 @@ def test_list_filters(tmp_path):
|
||||
assert [a["name"] for a in json.loads(body)["assets"]] == ["hero.png"]
|
||||
_, body = call(base, "/api/v1/assets", token=TOKEN, params={"q": "zzz"})
|
||||
assert json.loads(body)["assets"] == []
|
||||
|
||||
|
||||
def _self_signed_cert(path):
|
||||
subprocess.run(
|
||||
["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes",
|
||||
"-keyout", str(path / "key.pem"), "-out", str(path / "cert.pem"),
|
||||
"-days", "1", "-subj", "/CN=localhost"],
|
||||
check=True, capture_output=True)
|
||||
return str(path / "cert.pem"), str(path / "key.pem")
|
||||
|
||||
|
||||
def test_tls_serves_https(tmp_path):
|
||||
import ssl as ssl_module
|
||||
|
||||
cert, key = _self_signed_cert(tmp_path)
|
||||
server = LibraryServer(tmp_path / "store", tmp_path / "lib.db",
|
||||
token=TOKEN, tls_cert=cert, tls_key=key)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
server.ready.wait(timeout=10)
|
||||
ctx = ssl_module._create_unverified_context()
|
||||
req = urllib.request.Request(f"https://127.0.0.1:{server.port}/health")
|
||||
with urllib.request.urlopen(req, context=ctx) as response:
|
||||
assert response.status == 200
|
||||
assert json.loads(response.read()) == {"ok": True}
|
||||
|
||||
|
||||
def test_tls_missing_files_raise(tmp_path):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
LibraryServer(tmp_path / "store", tmp_path / "lib.db",
|
||||
tls_cert=str(tmp_path / "nope.pem"),
|
||||
tls_key=str(tmp_path / "nope-key.pem"))
|
||||
|
||||
|
||||
def test_tls_half_configured_raises(tmp_path):
|
||||
_self_signed_cert(tmp_path)
|
||||
with pytest.raises(ValueError):
|
||||
LibraryServer(tmp_path / "store", tmp_path / "lib.db",
|
||||
tls_cert=str(tmp_path / "cert.pem"))
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Thumbnail column migration + server thumbnail tests."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
import library.db as db
|
||||
from library.server import LibraryServer
|
||||
|
||||
|
||||
def _old_schema(path):
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.execute("CREATE TABLE assets("
|
||||
"id INTEGER PRIMARY KEY, name TEXT NOT NULL,"
|
||||
" blob_sha256 TEXT NOT NULL, size_bytes INTEGER NOT NULL,"
|
||||
" kind TEXT NOT NULL DEFAULT 'raw',"
|
||||
" godot_importer TEXT,"
|
||||
" params_json TEXT NOT NULL DEFAULT '{}',"
|
||||
" tags_json TEXT NOT NULL DEFAULT '[]',"
|
||||
" created_at TEXT NOT NULL DEFAULT (datetime('now')))")
|
||||
conn.execute("INSERT INTO assets(name, blob_sha256, size_bytes)"
|
||||
" VALUES ('old.png', 'ab', 3)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_old_db_gains_thumbnail_column(tmp_path):
|
||||
path = tmp_path / "lib.db"
|
||||
_old_schema(path)
|
||||
conn = db.open_db(path)
|
||||
try:
|
||||
cols = [r[1] for r in conn.execute("PRAGMA table_info(assets)")]
|
||||
assert "thumbnail_sha256" in cols
|
||||
row = db.get_asset(conn, 1)
|
||||
assert row["thumbnail_sha256"] is None
|
||||
assert row["name"] == "old.png"
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_new_db_has_thumbnail_column(tmp_path):
|
||||
conn = db.open_db(tmp_path / "lib.db")
|
||||
try:
|
||||
cols = [r[1] for r in conn.execute("PRAGMA table_info(assets)")]
|
||||
assert "thumbnail_sha256" in cols
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _png_bytes() -> bytes:
|
||||
return base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==")
|
||||
|
||||
|
||||
def _start(tmp_path):
|
||||
server = LibraryServer(tmp_path / "store", tmp_path / "lib.db",
|
||||
token="secret")
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
server.ready.wait(timeout=10)
|
||||
return f"http://127.0.0.1:{server.port}"
|
||||
|
||||
|
||||
def _get(base, path, token="secret"):
|
||||
req = urllib.request.Request(base + path)
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req) as response:
|
||||
return response.status, dict(response.headers), response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, dict(exc.headers), exc.read()
|
||||
|
||||
|
||||
def _post(base, path, data=b"", params=None, token="secret"):
|
||||
url = base + path
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
req = urllib.request.Request(url, data=data, method="POST")
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req) as response:
|
||||
return response.status, response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read()
|
||||
|
||||
|
||||
def test_image_upload_generates_thumbnail(tmp_path):
|
||||
import shutil
|
||||
if shutil.which("ffmpeg") is None and shutil.which("magick") is None:
|
||||
pytest.skip("no thumbnail engine on PATH")
|
||||
base = _start(tmp_path)
|
||||
status, body = _post(base, "/api/v1/assets", data=_png_bytes(),
|
||||
params={"name": "hero.png", "kind": "texture"})
|
||||
assert status == 201, body
|
||||
asset_id = json.loads(body)["id"]
|
||||
listed = {a["id"]: a for a in json.loads(_list(base))["assets"]}
|
||||
assert listed[asset_id]["thumbnail_sha256"] is not None
|
||||
status, headers, blob = _get(base, f"/api/v1/assets/{asset_id}/thumbnail")
|
||||
assert status == 200, blob[:100]
|
||||
assert headers.get("Content-Type") == "image/png"
|
||||
assert headers.get("Access-Control-Allow-Origin") == "*"
|
||||
assert blob[:8] == b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def _list(base):
|
||||
status, _, body = _get(base, "/api/v1/assets")
|
||||
assert status == 200
|
||||
return body
|
||||
|
||||
|
||||
def test_text_upload_has_no_thumbnail(tmp_path):
|
||||
base = _start(tmp_path)
|
||||
status, body = _post(base, "/api/v1/assets", data=b"just text",
|
||||
params={"name": "note.txt"})
|
||||
assert status == 201, body
|
||||
asset_id = json.loads(body)["id"]
|
||||
status, _headers, body = _get(base, f"/api/v1/assets/{asset_id}/thumbnail")
|
||||
assert status == 404
|
||||
assert json.loads(body)["error"] == "no thumbnail"
|
||||
|
||||
|
||||
def test_thumbnail_unknown_asset_404(tmp_path):
|
||||
base = _start(tmp_path)
|
||||
status, _headers, body = _get(base, "/api/v1/assets/999/thumbnail")
|
||||
assert status == 404
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Chunked (session) upload tests: init -> chunks -> complete."""
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
from library.client import LibraryError, pull, push, upload_status
|
||||
from library.server import LibraryServer
|
||||
from test_library_server import TOKEN
|
||||
|
||||
CHUNK = b"01234567" # 8 bytes; tests force tiny chunks via chunk_size
|
||||
|
||||
|
||||
def start_server(tmp_path, **kwargs):
|
||||
import threading
|
||||
server = LibraryServer(tmp_path / "store", tmp_path / "lib.db",
|
||||
token=TOKEN, **kwargs)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
server.ready.wait(timeout=10)
|
||||
return server, f"http://127.0.0.1:{server.port}"
|
||||
|
||||
|
||||
def post(base, path, token=TOKEN, data=b"", params=None):
|
||||
url = base + path
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
req = urllib.request.Request(url, data=data, method="POST")
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req) as response:
|
||||
return response.status, response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read()
|
||||
|
||||
|
||||
def test_session_roundtrip_many_chunks(tmp_path):
|
||||
_, base = start_server(tmp_path)
|
||||
status, body = post(base, "/api/v1/uploads",
|
||||
params={"name": "big.bin", "kind": "raw"})
|
||||
assert status == 201, body
|
||||
upload_id = json.loads(body)["upload_id"]
|
||||
|
||||
payload = CHUNK * 100 # 800 bytes over 8-byte chunks
|
||||
for i in range(0, len(payload), 8):
|
||||
status, body = post(base, f"/api/v1/uploads/{upload_id}/chunks",
|
||||
data=payload[i:i + 8])
|
||||
assert status == 200, body
|
||||
assert json.loads(body)["offset"] == len(payload)
|
||||
|
||||
status, body = post(base, f"/api/v1/uploads/{upload_id}/complete")
|
||||
assert status == 201, body
|
||||
asset_id = json.loads(body)["id"]
|
||||
assert json.loads(body)["size"] == len(payload)
|
||||
|
||||
dest = tmp_path / "back.bin"
|
||||
assert pull(base, TOKEN, asset_id, str(dest))["size"] == len(payload)
|
||||
assert dest.read_bytes() == payload
|
||||
|
||||
|
||||
def test_status_reports_offset(tmp_path):
|
||||
_, base = start_server(tmp_path)
|
||||
_, body = post(base, "/api/v1/uploads", params={"name": "a.bin"})
|
||||
upload_id = json.loads(body)["upload_id"]
|
||||
post(base, f"/api/v1/uploads/{upload_id}/chunks", data=b"12345")
|
||||
status, body = post(base, f"/api/v1/uploads/{upload_id}/chunks",
|
||||
data=b"67890")
|
||||
assert json.loads(body)["offset"] == 10
|
||||
assert upload_status(base, TOKEN, upload_id)["offset"] == 10
|
||||
with pytest.raises(LibraryError):
|
||||
upload_status(base, TOKEN, "no-such-id")
|
||||
|
||||
|
||||
def test_unknown_upload_id_404s(tmp_path):
|
||||
_, base = start_server(tmp_path)
|
||||
assert post(base, "/api/v1/uploads/nope/chunks", data=b"x")[0] == 404
|
||||
assert post(base, "/api/v1/uploads/nope/complete")[0] == 404
|
||||
|
||||
|
||||
def test_cap_enforced_across_chunks(tmp_path):
|
||||
_, base = start_server(tmp_path, max_upload_bytes=16)
|
||||
_, body = post(base, "/api/v1/uploads", params={"name": "a.bin"})
|
||||
upload_id = json.loads(body)["upload_id"]
|
||||
assert post(base, f"/api/v1/uploads/{upload_id}/chunks",
|
||||
data=b"0" * 16)[0] == 200
|
||||
status, body = post(base, f"/api/v1/uploads/{upload_id}/chunks",
|
||||
data=b"overflow")
|
||||
assert status == 413
|
||||
# session is dead after exceeding the cap
|
||||
assert post(base, f"/api/v1/uploads/{upload_id}/complete")[0] == 404
|
||||
|
||||
|
||||
def test_push_auto_chunks_large_files(tmp_path):
|
||||
_, base = start_server(tmp_path)
|
||||
big = tmp_path / "big.bin"
|
||||
big.write_bytes(b"y" * 100)
|
||||
result = push(base, TOKEN, str(big), chunk_size=7, chunk_above=10)
|
||||
assert result["id"] == 1
|
||||
dest = tmp_path / "back.bin"
|
||||
assert pull(base, TOKEN, 1, str(dest))["size"] == 100
|
||||
assert dest.read_bytes() == b"y" * 100
|
||||
|
||||
|
||||
def test_push_small_file_stays_single_shot(tmp_path, monkeypatch):
|
||||
from test_library_server import start_server as _start
|
||||
import library.client as client_mod
|
||||
_, base = _start(tmp_path)
|
||||
small = tmp_path / "small.bin"
|
||||
small.write_bytes(b"z" * 10)
|
||||
calls = []
|
||||
real = client_mod._push_session
|
||||
monkeypatch.setattr(client_mod, "_push_session",
|
||||
lambda *a, **k: calls.append(1) or real(*a, **k))
|
||||
push(base, TOKEN, str(small), chunk_above=1024)
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_push_missing_file_still_raises(tmp_path):
|
||||
with pytest.raises(LibraryError):
|
||||
push("http://127.0.0.1:1", TOKEN, str(tmp_path / "nope.bin"))
|
||||
Reference in new issue
Block a user