feat: tighter delta version support

This commit is contained in:
DecDuck
2026-02-01 12:08:12 +11:00
parent 83d95459ea
commit dcb45da68b
16 changed files with 186 additions and 312 deletions

View File

@@ -173,7 +173,7 @@
:title="t('home.admin.biggestGamesToDownload')"
:subtitle="t('home.admin.latestVersionOnly')"
>
<RankingList :items="biggestGamesLatest.map(gameToRankItem)" />
<!-- <RankingList :items="biggestGamesLatest.map(gameToRankItem)" />-->
</TileWithLink>
</div>
<div class="col-span-6 lg:col-span-2">
@@ -181,7 +181,7 @@
:title="t('home.admin.biggestGamesOnServer')"
:subtitle="t('home.admin.allVersionsCombined')"
>
<RankingList :items="biggestGamesCombined.map(gameToRankItem)" />
<!-- <RankingList :items="biggestGamesCombined.map(gameToRankItem)" />-->
</TileWithLink>
</div>
</div>
@@ -196,7 +196,6 @@ import DropLogo from "~/components/DropLogo.vue";
import { ServerStackIcon, UserGroupIcon } from "@heroicons/vue/24/outline";
import { getPercentage } from "~/utils/utils";
import { getBarColor } from "~/utils/colors";
import type { GameSize } from "~/server/internal/gamesize";
import type { RankItem } from "~/components/RankingList.vue";
definePageMeta({
@@ -216,16 +215,8 @@ const {
gameCount,
sources,
userStats,
biggestGamesLatest,
biggestGamesCombined,
} = await $dropFetch("/api/v1/admin/home");
const gameToRankItem = (game: GameSize, rank: number): RankItem => ({
rank: rank + 1,
name: game.gameName,
value: formatBytes(game.size),
});
const pieChartData = [
{
label: t("home.admin.inactiveUsers"),

View File

@@ -93,10 +93,21 @@
{{ $t("store.size") }}
</td>
<td
v-if="size"
v-if="size.versions.length > 0"
class="whitespace-nowrap inline-flex gap-x-4 px-3 py-4 text-sm text-zinc-400"
>
{{ formatBytes(size) }}
<ul>
<ol
v-for="version in size.versions"
:key="version.versionId"
>
{{
formatBytes(version.installSize)
}}
-
{{ formatBytes(version.downloadSize) }}
</ol>
</ul>
</td>
<td
v-else

View File

@@ -1,7 +1,7 @@
import { ArkErrors, type } from "arktype";
import prisma from "~/server/internal/db/database";
import type { H3Event } from "h3";
import { castManifest } from "~/server/internal/library/manifest";
import { castManifest } from "~/server/internal/library/manifest/utils";
const AUTHORIZATION_HEADER_PREFIX = "Bearer ";

View File

@@ -1,17 +1,16 @@
import type { GameVersion, Prisma } from "~/prisma/client/client";
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import gameSizeManager from "~/server/internal/gamesize";
import type { UnimportedVersionInformation } from "~/server/internal/library";
import libraryManager from "~/server/internal/library";
async function getGameVersionSize<
T extends Omit<GameVersion, "dropletManifest">,
>(gameId: string, version: T) {
const size = await libraryManager.getGameVersionSize(
gameId,
version.versionId,
);
return { ...version, size };
const clientSize = await gameSizeManager.getVersionSize(version.versionId);
const diskSize = await gameSizeManager.getVersionDiskSize(version.versionId);
return { ...version, diskSize, clientSize };
}
export type AdminFetchGameType = Prisma.GameGetPayload<{

View File

@@ -10,18 +10,10 @@ export default defineEventHandler(async (h3) => {
const sources = await libraryManager.fetchLibraries();
const userStats = await userStatsManager.getUserStats();
const biggestGamesCombined =
await libraryManager.getBiggestGamesCombinedVersions(5);
const biggestGamesLatest =
await libraryManager.getBiggestGamesLatestVersions(5);
return {
gameCount: await prisma.game.count(),
version: systemConfig.getDropVersion(),
userStats,
sources,
biggestGamesLatest,
biggestGamesCombined,
};
});

View File

@@ -50,7 +50,12 @@ export default defineEventHandler(async (h3) => {
where: {
gameId: body.id,
delta: false,
launches: { some: { platform: platformObject.platform } },
OR: [
{ launches: { some: { platform: platformObject.platform } } },
{
setups: { some: { platform: platformObject.platform } },
},
],
},
});
if (validOverlayVersions == 0)

View File

@@ -23,7 +23,7 @@ export default defineEventHandler<{
if (!authManager.getAuthProviders().Simple)
throw createError({
statusCode: 403,
statusMessage: t("errors.auth.method.signinDisabled"),
message: t("errors.auth.method.signinDisabled"),
});
const body = signinValidator(await readBody(h3));
@@ -33,7 +33,7 @@ export default defineEventHandler<{
throw createError({
statusCode: 400,
statusMessage: body.summary,
message: body.summary,
});
}
@@ -57,13 +57,13 @@ export default defineEventHandler<{
if (!authMek)
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidUserOrPass"),
message: t("errors.auth.invalidUserOrPass"),
});
if (!authMek.user.enabled)
throw createError({
statusCode: 403,
statusMessage: t("errors.auth.disabled"),
message: t("errors.auth.disabled"),
});
// LEGACY bcrypt
@@ -74,13 +74,13 @@ export default defineEventHandler<{
if (!hash)
throw createError({
statusCode: 500,
statusMessage: t("errors.auth.invalidPassState"),
message: t("errors.auth.invalidPassState"),
});
if (!(await checkHashBcrypt(body.password, hash)))
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidUserOrPass"),
message: t("errors.auth.invalidUserOrPass"),
});
// TODO: send user to forgot password screen or something to force them to change their password to new system
@@ -101,13 +101,13 @@ export default defineEventHandler<{
if (!hash || typeof hash !== "string")
throw createError({
statusCode: 500,
statusMessage: t("errors.auth.invalidPassState"),
message: t("errors.auth.invalidPassState"),
});
if (!(await checkHashArgon2(body.password, hash)))
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidUserOrPass"),
message: t("errors.auth.invalidUserOrPass"),
});
const result = await sessionHandler.signin(h3, authMek.userId, {

View File

@@ -27,7 +27,7 @@ export default defineEventHandler<{
if (!authManager.getAuthProviders().Simple)
throw createError({
statusCode: 403,
statusMessage: t("errors.auth.method.signinDisabled"),
message: t("errors.auth.method.signinDisabled"),
});
const user = await readValidatedBody(h3, CreateUserValidator);
@@ -38,7 +38,7 @@ export default defineEventHandler<{
if (!invitation)
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidInvite"),
message: t("errors.auth.invalidInvite"),
});
// reuse items from invite
@@ -51,7 +51,7 @@ export default defineEventHandler<{
if (existing > 0)
throw createError({
statusCode: 400,
statusMessage: t("errors.auth.usernameTaken"),
message: t("errors.auth.usernameTaken"),
});
const userId = randomUUID();

View File

@@ -1,6 +1,5 @@
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
export default defineClientEventHandler(async (h3) => {
const id = getRouterParam(h3, "id");
@@ -57,8 +56,5 @@ export default defineClientEventHandler(async (h3) => {
})),
};
return {
...gameVersionMapped,
size: libraryManager.getGameVersionSize(id, version),
};
return gameVersionMapped;
});

View File

@@ -1,6 +1,7 @@
import type { Platform } from "~/prisma/client/enums";
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import prisma from "~/server/internal/db/database";
import type { GameVersionSize } from "~/server/internal/gamesize";
import gameSizeManager from "~/server/internal/gamesize";
type VersionDownloadOption = {
@@ -8,14 +9,14 @@ type VersionDownloadOption = {
displayName?: string | undefined;
versionPath?: string | undefined;
platform: Platform;
size: number;
size: GameVersionSize;
requiredContent: Array<{
gameId: string;
versionId: string;
name: string;
iconObjectId: string;
shortDescription: string;
size: number;
size: GameVersionSize;
}>;
};
@@ -86,19 +87,14 @@ export default defineClientEventHandler(async (h3) => {
iconObjectId: launch.executor.gameVersion.game.mIconObjectId,
shortDescription:
launch.executor.gameVersion.game.mShortDescription,
size:
(await gameSizeManager.getGameVersionSize(
launch.executor.gameVersion.game.id,
launch.executor.gameVersion.versionId,
)) ?? 0,
size: (await gameSizeManager.getVersionSize(
launch.executor.gameVersion.versionId,
))!,
});
}
}
const size = await gameSizeManager.getGameVersionSize(
v.gameId,
v.versionId,
);
const size = await gameSizeManager.getVersionSize(v.versionId);
return platformOptions
.entries()

View File

@@ -11,5 +11,6 @@ export default defineClientEventHandler(async (h3) => {
});
const result = await createDownloadManifestDetails(version);
console.log(result);
return result;
});

View File

@@ -1,6 +1,6 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
import gameSizeManager from "~/server/internal/gamesize";
export default defineEventHandler(async (h3) => {
const userId = await aclManager.getUserIdACL(h3, ["store:read"]);
@@ -57,7 +57,7 @@ export default defineEventHandler(async (h3) => {
},
});
const size = await libraryManager.getGameVersionSize(game.id);
const size = (await gameSizeManager.getGameBreakdown(gameId))!;
return { game, rating, size };
});

View File

@@ -1,228 +1,116 @@
import cacheHandler from "../cache";
import prisma from "../db/database";
import { sum } from "../../../utils/array";
import type { Game, GameVersion } from "~/prisma/client/client";
import { castManifest } from "../library/manifest";
import { createDownloadManifestDetails } from "../library/manifest";
import { castManifest } from "../library/manifest/utils";
export type GameSize = {
gameName: string;
size: number;
gameId: string;
export type GameVersionSize = {
versionId: string;
installSize: number;
downloadSize: number;
};
export type VersionSize = GameSize & {
latest: boolean;
};
type VersionsSizes = {
[versionName: string]: VersionSize;
};
type GameVersionsSize = {
[gameId: string]: VersionsSizes;
export type GameSizeBreakdown = {
diskSize: number;
versions: Array<GameVersionSize & { diskSize: number; name: string }>;
};
class GameSizeManager {
private gameVersionsSizesCache =
cacheHandler.createCache<GameVersionsSize>("gameVersionsSizes");
// All versions sizes combined
private gameSizesCache = cacheHandler.createCache<GameSize>("gameSizes");
cacheHandler.createCache<GameVersionSize>("versionSizes");
private gameBreakdownCache =
cacheHandler.createCache<GameSizeBreakdown>("gameBreakdown");
private async clearGameVersionsSizesCache() {
(await this.gameVersionsSizesCache.getKeys()).map((key) =>
this.gameVersionsSizesCache.remove(key),
);
}
private async clearGameSizesCache() {
(await this.gameSizesCache.getKeys()).map((key) =>
this.gameSizesCache.remove(key),
);
}
// All versions of a game combined
async getCombinedGameSize(gameId: string) {
const versions = await prisma.gameVersion.findMany({
where: { gameId },
});
const sizes = await Promise.all(
versions.map((version) => castManifest(version.dropletManifest).size),
);
return sum(sizes);
}
async getGameVersionSize(
gameId: string,
versionId?: string,
): Promise<number | null> {
if (!versionId) {
const version = await prisma.gameVersion.findFirst({
where: { gameId },
orderBy: {
versionIndex: "desc",
},
});
if (!version) {
return null;
}
versionId = version.versionId;
/***
* Gets the size of the game to the user:
* - installSize: size on disk after install
* - downloadSize: how many bytes are downloaded (but not necessarily stored)
*/
async getVersionSize(versionId: string): Promise<GameVersionSize | null> {
if (await this.gameVersionsSizesCache.has(versionId))
return await this.gameVersionsSizesCache.get(versionId);
try {
const { downloadSize, installSize } =
await createDownloadManifestDetails(versionId);
const result = {
downloadSize,
installSize,
versionId,
} satisfies GameVersionSize;
await this.gameVersionsSizesCache.set(versionId, result);
return result;
} catch {
return null;
}
const { dropletManifest } = (await prisma.gameVersion.findUnique({
where: { versionId },
}))!;
return castManifest(dropletManifest).size;
}
private async isLatestVersion(
gameVersions: GameVersion[],
version: GameVersion,
): Promise<boolean> {
return gameVersions.length > 0
? gameVersions[0].versionId === version.versionId
: false;
}
async getBiggestGamesLatestVersion(top: number): Promise<VersionSize[]> {
const gameIds = await this.gameVersionsSizesCache.getKeys();
const latestGames = await Promise.all(
gameIds.map(async (gameId) => {
const versionsSizes = await this.gameVersionsSizesCache.get(gameId);
if (!versionsSizes) {
return null;
}
const latestVersionName = Object.keys(versionsSizes).find(
(versionName) => versionsSizes[versionName].latest,
);
if (!latestVersionName) {
return null;
}
return versionsSizes[latestVersionName] || null;
}),
);
return latestGames
.filter((game) => game !== null)
.sort((gameA, gameB) => gameB.size - gameA.size)
.slice(0, top);
}
async isGameVersionsSizesCacheEmpty() {
return (await this.gameVersionsSizesCache.getKeys()).length === 0;
}
async isGameSizesCacheEmpty() {
return (await this.gameSizesCache.getKeys()).length === 0;
}
async cacheAllCombinedGames() {
await this.clearGameSizesCache();
const games = await prisma.game.findMany({ include: { versions: true } });
await Promise.all(games.map((game) => this.cacheCombinedGame(game)));
}
async cacheCombinedGame(game: Game) {
const size = await this.getCombinedGameSize(game.id);
if (!size) {
this.gameSizesCache.remove(game.id);
return;
}
const gameSize = {
size,
gameName: game.mName,
gameId: game.id,
};
await this.gameSizesCache.set(game.id, gameSize);
}
async cacheAllGameVersions() {
await this.clearGameVersionsSizesCache();
const games = await prisma.game.findMany({
include: {
versions: {
orderBy: {
versionIndex: "desc",
},
take: 1,
},
/***
* Get the size of the game on disk
*/
async getVersionDiskSize(versionId: string): Promise<number | null> {
const version = await prisma.gameVersion.findUnique({
where: {
versionId,
},
select: {
dropletManifest: true,
},
});
await Promise.all(games.map((game) => this.cacheGameVersion(game)));
if (!version) return null;
return castManifest(version.dropletManifest).size;
}
async cacheGameVersion(
game: Game & { versions: GameVersion[] },
versionId?: string,
) {
const cacheVersion = async (version: GameVersion) => {
const size = await this.getGameVersionSize(game.id, version.versionId);
if (!version.versionId || !size) {
return;
}
const versionsSizes = {
[version.versionId]: {
size,
gameName: game.mName,
gameId: game.id,
latest: await this.isLatestVersion(game.versions, version),
},
};
const allVersionsSizes =
(await this.gameVersionsSizesCache.get(game.id)) || {};
await this.gameVersionsSizesCache.set(game.id, {
...allVersionsSizes,
...versionsSizes,
});
};
if (versionId) {
const version = await prisma.gameVersion.findFirst({
where: { gameId: game.id, versionId },
});
if (!version) {
return;
}
cacheVersion(version);
return;
}
if ("versions" in game) {
await Promise.all(game.versions.map(cacheVersion));
}
}
async getBiggestGamesAllVersions(top: number): Promise<GameSize[]> {
const gameIds = await this.gameSizesCache.getKeys();
const allGames = await Promise.all(
gameIds.map(async (gameId) => await this.gameSizesCache.get(gameId)),
/**
* Calculate the total disk usage of a game
* @param gameId Game ID to calculate
* @returns Total **disk** size of the game
*/
async getGameDiskSize(gameId: string): Promise<number> {
const versions = await prisma.gameVersion.findMany({
where: { gameId },
select: {
versionId: true,
},
});
const sizes = await Promise.all(
versions.map((version) => this.getVersionDiskSize(version.versionId)),
);
return allGames
.filter((game) => game !== null)
.sort((gameA, gameB) => gameB.size - gameA.size)
.slice(0, top);
return sum(sizes.filter((v) => v !== null));
}
async deleteGameVersion(gameId: string, version: string) {
const game = await prisma.game.findFirst({ where: { id: gameId } });
if (game) {
await this.cacheCombinedGame(game);
}
const versionsSizes = await this.gameVersionsSizesCache.get(gameId);
if (!versionsSizes) {
return;
}
// Remove the version from the VersionsSizes object
const { [version]: _, ...updatedVersionsSizes } = versionsSizes;
await this.gameVersionsSizesCache.set(gameId, updatedVersionsSizes);
}
async getGameBreakdown(gameId: string): Promise<GameSizeBreakdown | null> {
const versions = await prisma.gameVersion.findMany({
where: { gameId },
orderBy: { versionIndex: "desc" },
select: { versionId: true, displayName: true, versionPath: true },
});
if (!versions) return null;
async deleteGame(gameId: string) {
this.gameSizesCache.remove(gameId);
this.gameVersionsSizesCache.remove(gameId);
const breakdownKey = `${gameId} ${versions.map((v) => v.versionId).join(" ")}`;
if (await this.gameBreakdownCache.has(breakdownKey))
return (await this.gameBreakdownCache.get(breakdownKey))!;
let diskSize = 0;
const versionInformation = [];
for (const version of versions) {
const size = (await this.getVersionSize(version.versionId))!;
const vDiskSize = (await this.getVersionDiskSize(version.versionId))!;
diskSize += vDiskSize;
versionInformation.push({
...size,
diskSize: vDiskSize,
name: (version.displayName ?? version.versionPath)!,
});
}
const result = {
diskSize,
versions: versionInformation,
};
await this.gameBreakdownCache.set(breakdownKey, result);
return result;
}
}
export const manager = new GameSizeManager();
export default manager;
export const gameSizeManager = new GameSizeManager();
export default gameSizeManager;

View File

@@ -19,7 +19,7 @@ import gameSizeManager from "~/server/internal/gamesize";
import { TORRENTIAL_SERVICE } from "../services/services/torrential";
import type { ImportVersion } from "~/server/api/v1/admin/import/version/index.post";
import { GameType, type Platform } from "~/prisma/client/enums";
import { castManifest } from "./manifest";
import { castManifest } from "./manifest/utils";
export function createGameImportTaskId(libraryId: string, libraryPath: string) {
return createHash("md5")
@@ -500,14 +500,18 @@ class LibraryManager {
acls: ["system:import:version:read"],
});
await libraryManager.cacheCombinedGameSize(gameId);
await libraryManager.cacheGameVersionSize(gameId, newVersion.versionId);
await TORRENTIAL_SERVICE.utils().invalidate(
gameId,
newVersion.versionId,
);
// Ensure cache is filled (also pre-caches the manifest)
try {
await gameSizeManager.getVersionSize(newVersion.versionId);
} catch (e) {
logger.warn(`Failed to pre-cache game size and manifest: ${e}`);
}
if (version.type === "depot") {
// SAFETY: we can only reach this if the type is depot and identifier is valid
// eslint-disable-next-line drop/no-prisma-delete
@@ -552,8 +556,6 @@ class LibraryManager {
versionId: version,
},
});
await gameSizeManager.deleteGameVersion(gameId, version);
}
async deleteGame(gameId: string) {
@@ -562,7 +564,6 @@ class LibraryManager {
id: gameId,
},
});
await gameSizeManager.deleteGame(gameId);
// Delete all game versions that depended on this game
await prisma.gameVersion.deleteMany({
where: {
@@ -578,46 +579,6 @@ class LibraryManager {
},
});
}
async getGameVersionSize(
gameId: string,
versionName?: string,
): Promise<number | null> {
return gameSizeManager.getGameVersionSize(gameId, versionName);
}
async getBiggestGamesCombinedVersions(top: number) {
if (await gameSizeManager.isGameSizesCacheEmpty()) {
await gameSizeManager.cacheAllCombinedGames();
}
return gameSizeManager.getBiggestGamesAllVersions(top);
}
async getBiggestGamesLatestVersions(top: number) {
if (await gameSizeManager.isGameVersionsSizesCacheEmpty()) {
await gameSizeManager.cacheAllGameVersions();
}
return gameSizeManager.getBiggestGamesLatestVersion(top);
}
async cacheCombinedGameSize(gameId: string) {
const game = await prisma.game.findFirst({ where: { id: gameId } });
if (!game) {
return;
}
await gameSizeManager.cacheCombinedGame(game);
}
async cacheGameVersionSize(gameId: string, versionId: string) {
const game = await prisma.game.findFirst({
where: { id: gameId },
include: { versions: true },
});
if (!game) {
return;
}
await gameSizeManager.cacheGameVersion(game, versionId);
}
}
export const libraryManager = new LibraryManager();

View File

@@ -1,14 +1,27 @@
import cacheHandler from "../../cache";
import prisma from "../../db/database";
import { castManifest, type DropletManifest } from "../manifest";
import { castManifest, type DropletManifest } from "./utils";
export type DownloadManifestDetails = {
/***
* Version ID to manifest
*/
manifests: { [key: string]: DropletManifest };
/***
* File name to version ID
*/
fileList: { [key: string]: string };
/// Size on disk after download
installSize: number;
/// Size of download
downloadSize: number;
};
function convertMap<T>(map: Map<string, T>): { [key: string]: T } {
return Object.fromEntries(map.entries().toArray());
}
const manifestCache =
cacheHandler.createCache<DownloadManifestDetails>("manifestCache");
/**
*
@@ -17,7 +30,9 @@ function convertMap<T>(map: Map<string, T>): { [key: string]: T } {
*/
export async function createDownloadManifestDetails(
versionId: string,
refresh = false,
): Promise<DownloadManifestDetails> {
if(await manifestCache.has(versionId) && !refresh) return (await manifestCache.get(versionId))!;
const mainVersion = await prisma.gameVersion.findUnique({
where: { versionId },
select: {
@@ -75,6 +90,9 @@ export async function createDownloadManifestDetails(
}
}
let installSize = 0;
let downloadSize = 0;
// Now that we have our file list, filter the manifests
const manifests = new Map<string, DropletManifest>();
for (const version of versionOrder) {
@@ -86,9 +104,17 @@ export async function createDownloadManifestDetails(
const fileNames = Object.fromEntries(files);
const manifest = castManifest(version.dropletManifest);
const filteredChunks = Object.fromEntries(
Object.entries(manifest.chunks).filter(([, chunkData]) =>
chunkData.files.some((fileEntry) => !!fileNames[fileEntry.filename]),
),
Object.entries(manifest.chunks).filter(([, chunkData]) => {
let flag = false;
chunkData.files.forEach((fileEntry) => {
if (fileNames[fileEntry.filename]) {
flag = true;
installSize += fileEntry.length;
}
downloadSize += fileEntry.length;
});
return flag;
}),
);
manifests.set(version.versionId, {
...manifest,
@@ -96,5 +122,13 @@ export async function createDownloadManifestDetails(
});
}
return { fileList: convertMap(fileList), manifests: convertMap(manifests) };
const result = {
fileList: convertMap(fileList),
manifests: convertMap(manifests),
installSize,
downloadSize,
};
await manifestCache.set(versionId, result);
return result;
}