Database-level multi-library support #48 (#58)

* feat: start of library backends

* feat: update backend routes and create initializer

* feat: add legacy library creation

* fix: resolve frontend type errors

* fix: runtime errors

* fix: lint

server/internal/library/filesystem.ts

@@ -0,0 +1,107 @@
+import { ArkErrors, type } from "arktype";
+import {
+  GameNotFoundError,
+  VersionNotFoundError,
+  type LibraryProvider,
+} from "./provider";
+import { LibraryBackend } from "~/prisma/client";
+import fs from "fs";
+import path from "path";
+import droplet from "@drop-oss/droplet";
+import type { Readable } from "stream";
+
+export const FilesystemProviderConfig = type({
+  baseDir: "string",
+});
+
+export class FilesystemProvider
+  implements LibraryProvider<typeof FilesystemProviderConfig.infer>
+{
+  private config: typeof FilesystemProviderConfig.infer;
+  private myId: string;
+
+  constructor(rawConfig: unknown, id: string) {
+    const config = FilesystemProviderConfig(rawConfig);
+    if (config instanceof ArkErrors) {
+      throw new Error(
+        `Failed to create filesystem provider: ${config.summary}`,
+      );
+    }
+
+    this.myId = id;
+    this.config = config;
+    fs.mkdirSync(this.config.baseDir, { recursive: true });
+  }
+
+  id(): string {
+    return this.myId;
+  }
+
+  type(): LibraryBackend {
+    return LibraryBackend.Filesystem;
+  }
+
+  async listGames(): Promise<string[]> {
+    const dirs = fs.readdirSync(this.config.baseDir);
+    const folderDirs = dirs.filter((e) => {
+      const fullDir = path.join(this.config.baseDir, e);
+      return fs.lstatSync(fullDir).isDirectory();
+    });
+    return folderDirs;
+  }
+
+  async listVersions(game: string): Promise<string[]> {
+    const gameDir = path.join(this.config.baseDir, game);
+    if (!fs.existsSync(gameDir)) throw new GameNotFoundError();
+    const versionDirs = fs.readdirSync(gameDir);
+    const validVersionDirs = versionDirs.filter((e) => {
+      const fullDir = path.join(this.config.baseDir, game, e);
+      return droplet.hasBackendForPath(fullDir);
+    });
+    return validVersionDirs;
+  }
+
+  async versionReaddir(game: string, version: string): Promise<string[]> {
+    const versionDir = path.join(this.config.baseDir, game, version);
+    if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
+    return droplet.listFiles(versionDir);
+  }
+
+  async generateDropletManifest(
+    game: string,
+    version: string,
+    progress: (err: Error | null, v: number) => void,
+    log: (err: Error | null, v: string) => void,
+  ): Promise<string> {
+    const versionDir = path.join(this.config.baseDir, game, version);
+    if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
+    const manifest = await new Promise<string>((r, j) =>
+      droplet.generateManifest(versionDir, progress, log, (err, result) => {
+        if (err) return j(err);
+        r(result);
+      }),
+    );
+    return manifest;
+  }
+
+  // TODO: move this over to the droplet.readfile function it works
+  async readFile(
+    game: string,
+    version: string,
+    filename: string,
+    options?: { start?: number; end?: number },
+  ): Promise<Readable | undefined> {
+    const filepath = path.join(this.config.baseDir, game, version, filename);
+    if (!fs.existsSync(filepath)) return undefined;
+    const stream = fs.createReadStream(filepath, options);
+
+    return stream;
+  }
+
+  async peekFile(game: string, version: string, filename: string) {
+    const filepath = path.join(this.config.baseDir, game, version, filename);
+    if (!fs.existsSync(filepath)) return undefined;
+    const stat = fs.statSync(filepath);
+    return { size: stat.size };
+  }
+}

server/internal/library/index.ts

@@ -5,60 +5,63 @@
  * It also provides the endpoints with information about unmatched games
  */
 
-import fs from "fs";
 import path from "path";
 import prisma from "../db/database";
-import type { GameVersion } from "~/prisma/client";
 import { fuzzy } from "fast-fuzzy";
-import { recursivelyReaddir } from "../utils/recursivedirs";
 import taskHandler from "../tasks";
 import { parsePlatform } from "../utils/parseplatform";
-import droplet from "@drop-oss/droplet";
 import notificationSystem from "../notifications";
-import { systemConfig } from "../config/sys-conf";
+import type { LibraryProvider } from "./provider";
 
 class LibraryManager {
-  private basePath: string;
+  private libraries: Map<string, LibraryProvider<unknown>> = new Map();
 
-  constructor() {
-    this.basePath = systemConfig.getLibraryFolder();
-    fs.mkdirSync(this.basePath, { recursive: true });
-  }
-
-  fetchLibraryPath() {
-    return this.basePath;
+  addLibrary(library: LibraryProvider<unknown>) {
+    this.libraries.set(library.id(), library);
   }
 
   async fetchAllUnimportedGames() {
-    const dirs = fs.readdirSync(this.basePath).filter((e) => {
-      const fullDir = path.join(this.basePath, e);
-      return fs.lstatSync(fullDir).isDirectory();
-    });
+    const unimportedGames: { [key: string]: string[] } = {};
 
-    const validGames = await prisma.game.findMany({
-      where: {
-        libraryBasePath: { in: dirs },
-      },
-      select: {
-        libraryBasePath: true,
-      },
-    });
-    const validGameDirs = validGames.map((e) => e.libraryBasePath);
+    for (const [id, library] of this.libraries.entries()) {
+      const games = await library.listGames();
+      const validGames = await prisma.game.findMany({
+        where: {
+          libraryId: id,
+          libraryPath: { in: games },
+        },
+        select: {
+          libraryPath: true,
+        },
+      });
+      const providerUnimportedGames = games.filter(
+        (e) => validGames.findIndex((v) => v.libraryPath == e) == -1,
+      );
+      unimportedGames[id] = providerUnimportedGames;
+    }
 
-    const unregisteredGames = dirs.filter((e) => !validGameDirs.includes(e));
-
-    return unregisteredGames;
+    return unimportedGames;
   }
 
-  async fetchUnimportedGameVersions(
-    libraryBasePath: string,
-    versions: Array<GameVersion>,
-  ) {
-    const gameDir = path.join(this.basePath, libraryBasePath);
-    const versionsDirs = fs.readdirSync(gameDir);
-    const importedVersionDirs = versions.map((e) => e.versionName);
-    const unimportedVersions = versionsDirs.filter(
-      (e) => !importedVersionDirs.includes(e),
+  async fetchUnimportedGameVersions(libraryId: string, libraryPath: string) {
+    const provider = this.libraries.get(libraryId);
+    if (!provider) return undefined;
+    const game = await prisma.game.findUnique({
+      where: {
+        libraryKey: {
+          libraryId,
+          libraryPath,
+        },
+      },
+      select: {
+        versions: true,
+      },
+    });
+    if (!game) return undefined;
+
+    const versions = await provider.listVersions(libraryPath);
+    const unimportedVersions = versions.filter(
+      (e) => game.versions.findIndex((v) => v.versionName == e) == -1,
     );
 
     return unimportedVersions;
@@ -73,7 +76,8 @@ class LibraryManager {
         mShortDescription: true,
         metadataSource: true,
         mIconObjectId: true,
-        libraryBasePath: true,
+        libraryId: true,
+        libraryPath: true,
       },
       orderBy: {
         mName: "asc",
@@ -85,60 +89,24 @@ class LibraryManager {
         game: e,
         status: {
           noVersions: e.versions.length == 0,
-          unimportedVersions: await this.fetchUnimportedGameVersions(
-            e.libraryBasePath,
-            e.versions,
-          ),
+          unimportedVersions: (await this.fetchUnimportedGameVersions(
+            e.libraryId ?? "",
+            e.libraryPath,
+          ))!,
         },
       })),
     );
   }
 
-  async fetchUnimportedVersions(gameId: string) {
-    const game = await prisma.game.findUnique({
-      where: { id: gameId },
-      select: {
-        versions: {
-          select: {
-            versionName: true,
-          },
-        },
-        libraryBasePath: true,
-      },
-    });
-
-    if (!game) return undefined;
-    const targetDir = path.join(this.basePath, game.libraryBasePath);
-    if (!fs.existsSync(targetDir))
-      throw new Error(
-        "Game in database, but no physical directory? Something is very very wrong...",
-      );
-    const versions = fs.readdirSync(targetDir);
-    const validVersions = versions.filter((versionDir) => {
-      const versionPath = path.join(targetDir, versionDir);
-      const stat = fs.statSync(versionPath);
-      return stat.isDirectory();
-    });
-    const currentVersions = game.versions.map((e) => e.versionName);
-
-    const unimportedVersions = validVersions.filter(
-      (e) => !currentVersions.includes(e),
-    );
-    return unimportedVersions;
-  }
-
   async fetchUnimportedVersionInformation(gameId: string, versionName: string) {
     const game = await prisma.game.findUnique({
       where: { id: gameId },
-      select: { libraryBasePath: true, mName: true },
+      select: { libraryPath: true, libraryId: true, mName: true },
     });
-    if (!game) return undefined;
-    const targetDir = path.join(
-      this.basePath,
-      game.libraryBasePath,
-      versionName,
-    );
-    if (!fs.existsSync(targetDir)) return undefined;
+    if (!game || !game.libraryId) return undefined;
+
+    const library = this.libraries.get(game.libraryId);
+    if (!library) return undefined;
 
     const fileExts: { [key: string]: string[] } = {
       Linux: [
@@ -165,7 +133,7 @@ class LibraryManager {
       match: number;
     }> = [];
 
-    const files = recursivelyReaddir(targetDir, 2);
+    const files = await library.versionReaddir(game.libraryPath, versionName);
     for (const file of files) {
       const filename = path.basename(file);
       const dotLocation = file.lastIndexOf(".");
@@ -174,10 +142,9 @@ class LibraryManager {
         for (const checkExt of checkExts) {
           if (checkExt != ext) continue;
           const fuzzyValue = fuzzy(filename, game.mName);
-          const relative = path.relative(targetDir, file);
           options.push({
-            filename: relative,
-            platform: platform,
+            filename,
+            platform,
             match: fuzzyValue,
           });
         }
@@ -190,17 +157,22 @@ class LibraryManager {
   }
 
   // Checks are done in least to most expensive order
-  async checkUnimportedGamePath(targetPath: string) {
-    const targetDir = path.join(this.basePath, targetPath);
-    if (!fs.existsSync(targetDir)) return false;
-
+  async checkUnimportedGamePath(libraryId: string, libraryPath: string) {
     const hasGame =
-      (await prisma.game.count({ where: { libraryBasePath: targetPath } })) > 0;
+      (await prisma.game.count({ where: { libraryId, libraryPath } })) > 0;
     if (hasGame) return false;
 
     return true;
   }
 
+  /*
+  Game creation happens in metadata, because it's primarily a metadata object
+
+  async createGame(libraryId: string, libraryPath: string, game: Omit<Game, "libraryId" | "libraryPath">) {
+
+  }
+  */
+
   async importVersion(
     gameId: string,
     versionName: string,
@@ -224,12 +196,12 @@ class LibraryManager {
 
     const game = await prisma.game.findUnique({
       where: { id: gameId },
-      select: { mName: true, libraryBasePath: true },
+      select: { mName: true, libraryId: true, libraryPath: true },
     });
-    if (!game) return undefined;
+    if (!game || !game.libraryId) return undefined;
 
-    const baseDir = path.join(this.basePath, game.libraryBasePath, versionName);
-    if (!fs.existsSync(baseDir)) return undefined;
+    const library = this.libraries.get(game.libraryId);
+    if (!library) return undefined;
 
     taskHandler.create({
       id: taskId,
@@ -238,23 +210,18 @@ class LibraryManager {
       async run({ progress, log }) {
         // First, create the manifest via droplet.
         // This takes up 90% of our progress, so we wrap it in a *0.9
-        const manifest = await new Promise<string>((resolve, reject) => {
-          droplet.generateManifest(
-            baseDir,
-            (err, value) => {
-              if (err) return reject(err);
-              progress(value * 0.9);
-            },
-            (err, line) => {
-              if (err) return reject(err);
-              log(line);
-            },
-            (err, manifest) => {
-              if (err) return reject(err);
-              resolve(manifest);
-            },
-          );
-        });
+        const manifest = await library.generateDropletManifest(
+          game.libraryPath,
+          versionName,
+          (err, value) => {
+            if (err) throw err;
+            progress(value * 0.9);
+          },
+          (err, value) => {
+            if (err) throw err;
+            log(value);
+          },
+        );
 
         log("Created manifest successfully!");
 
@@ -315,6 +282,29 @@ class LibraryManager {
 
     return taskId;
   }
+
+  async peekFile(
+    libraryId: string,
+    game: string,
+    version: string,
+    filename: string,
+  ) {
+    const library = this.libraries.get(libraryId);
+    if (!library) return undefined;
+    return library.peekFile(game, version, filename);
+  }
+
+  async readFile(
+    libraryId: string,
+    game: string,
+    version: string,
+    filename: string,
+    options?: { start?: number; end?: number },
+  ) {
+    const library = this.libraries.get(libraryId);
+    if (!library) return undefined;
+    return library.readFile(game, version, filename, options);
+  }
 }
 
 export const libraryManager = new LibraryManager();

server/internal/library/provider.ts

@@ -0,0 +1,64 @@
+import type { Readable } from "stream";
+import type { LibraryBackend } from "~/prisma/client";
+
+export abstract class LibraryProvider<CFG> {
+  constructor(_config: CFG, _id: string) {
+    throw new Error("Library doesn't have a proper constructor");
+  }
+
+  /**
+   * @returns ID of the current library provider (fs, smb, s3, etc)
+   */
+  abstract type(): LibraryBackend;
+
+  /**
+   * @returns the specific ID of this current provider
+   */
+  abstract id(): string;
+
+  /**
+   * @returns list of (usually) top-level game folder names
+   */
+  abstract listGames(): Promise<string[]>;
+
+  /**
+   * @param game folder name of the game to list versions for
+   * @returns list of version folder names
+   */
+  abstract listVersions(game: string): Promise<string[]>;
+
+  /**
+   * @param game folder name of the game
+   * @param version folder name of the version
+   * @returns recursive list of all files in version, relative to the version folder (e.g. ./setup.exe)
+   */
+  abstract versionReaddir(game: string, version: string): Promise<string[]>;
+
+  /**
+   * @param game folder name of the game
+   * @param version folder name of the version
+   * @returns string of JSON of the droplet manifest
+   */
+  abstract generateDropletManifest(
+    game: string,
+    version: string,
+    progress: (err: Error | null, v: number) => void,
+    log: (err: Error | null, v: string) => void,
+  ): Promise<string>;
+
+  abstract peekFile(
+    game: string,
+    version: string,
+    filename: string,
+  ): Promise<{ size: number } | undefined>;
+
+  abstract readFile(
+    game: string,
+    version: string,
+    filename: string,
+    options?: { start?: number; end?: number },
+  ): Promise<Readable | undefined>;
+}
+
+export class GameNotFoundError extends Error {}
+export class VersionNotFoundError extends Error {}

server/internal/metadata/index.ts

@@ -97,18 +97,15 @@ export class MetadataHandler {
     return successfulResults;
   }
 
-  async createGameWithoutMetadata(libraryBasePath: string) {
+  async createGameWithoutMetadata(libraryId: string, libraryPath: string) {
     return await this.createGame(
       {
         id: "",
-        name: libraryBasePath,
-        icon: "",
-        description: "",
-        year: 0,
+        name: libraryPath,
         sourceId: "manual",
-        sourceName: "Manual",
       },
-      libraryBasePath,
+      libraryId,
+      libraryPath,
     );
   }
 
@@ -165,8 +162,9 @@ export class MetadataHandler {
   }
 
   async createGame(
-    result: InternalGameMetadataResult,
-    libraryBasePath: string,
+    result: { sourceId: string; id: string; name: string },
+    libraryId: string,
+    libraryPath: string,
   ) {
     const provider = this.providers.get(result.sourceId);
     if (!provider)
@@ -231,7 +229,8 @@ export class MetadataHandler {
           connectOrCreate: this.parseTags(metadata.tags),
         },
 
-        libraryBasePath,
+        libraryId,
+        libraryPath,
       },
     });