From c4d8b24295e3c5b128b45c29cd344ca1f9d38fd7 Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Wed, 9 Apr 2025 14:48:13 -0400 Subject: [PATCH] feat: hash objects for etag value --- server/api/v1/object/[id]/index.get.ts | 9 +++--- server/api/v1/object/[id]/index.head.ts | 5 +-- server/internal/objects/fsBackend.ts | 39 ++++++++++++++++++++++-- server/internal/objects/objectHandler.ts | 21 ++++++++----- 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/server/api/v1/object/[id]/index.get.ts b/server/api/v1/object/[id]/index.get.ts index 7bb63ab..1db25ea 100644 --- a/server/api/v1/object/[id]/index.get.ts +++ b/server/api/v1/object/[id]/index.get.ts @@ -12,15 +12,16 @@ export default defineEventHandler(async (h3) => { throw createError({ statusCode: 404, statusMessage: "Object not found" }); // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/ETag - const etagValue = h3.headers.get("If-None-Match"); - if (etagValue !== null) { + const etagRequestValue = h3.headers.get("If-None-Match"); + const etagActualValue = await objectHandler.fetchHash(id); + if (etagRequestValue !== null && etagActualValue === etagRequestValue) { // would compare if etag is valid, but objects should never change setResponseStatus(h3, 304); return null; } - // just return object id has etag since object should never change - setHeader(h3, "ETag", id); + // TODO: fix undefined etagValue + setHeader(h3, "ETag", etagActualValue ?? ""); setHeader(h3, "Content-Type", object.mime); setHeader( h3, diff --git a/server/api/v1/object/[id]/index.head.ts b/server/api/v1/object/[id]/index.head.ts index a981568..b3f836e 100644 --- a/server/api/v1/object/[id]/index.head.ts +++ b/server/api/v1/object/[id]/index.head.ts @@ -13,8 +13,9 @@ export default defineEventHandler(async (h3) => { throw createError({ statusCode: 404, statusMessage: "Object not found" }); // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/ETag - const etagValue = h3.headers.get("If-None-Match"); - if (etagValue !== null) { + const etagRequestValue = h3.headers.get("If-None-Match"); + const etagActualValue = await objectHandler.fetchHash(id); + if (etagRequestValue !== null && etagActualValue === etagRequestValue) { // would compare if etag is valid, but objects should never change setResponseStatus(h3, 304); return null; diff --git a/server/internal/objects/fsBackend.ts b/server/internal/objects/fsBackend.ts index d9be122..391e14b 100644 --- a/server/internal/objects/fsBackend.ts +++ b/server/internal/objects/fsBackend.ts @@ -7,15 +7,22 @@ import { } from "./objectHandler"; import sanitize from "sanitize-filename"; - +import { LRUCache } from "lru-cache"; import fs from "fs"; import path from "path"; import { Readable, Stream } from "stream"; +import { createHash } from "crypto"; export class FsObjectBackend extends ObjectBackend { private baseObjectPath: string; private baseMetadataPath: string; + // TODO: should probably make this save into db or something if we agree to never + // overwrite an object + private cache = new LRUCache({ + max: 1000, // number of items + }); + constructor() { super(); const basePath = process.env.FS_BACKEND_PATH ?? "./.data/objects"; @@ -35,6 +42,9 @@ export class FsObjectBackend extends ObjectBackend { const objectPath = path.join(this.baseObjectPath, sanitize(id)); if (!fs.existsSync(objectPath)) return false; + // remove item from cache + this.cache.delete(id); + if (source instanceof Readable) { const outputStream = fs.createWriteStream(objectPath); source.pipe(outputStream, { end: true }); @@ -52,7 +62,8 @@ export class FsObjectBackend extends ObjectBackend { async startWriteStream(id: ObjectReference) { const objectPath = path.join(this.baseObjectPath, sanitize(id)); if (!fs.existsSync(objectPath)) return undefined; - + // remove item from cache + this.cache.delete(id); return fs.createWriteStream(objectPath); } async create( @@ -100,6 +111,8 @@ export class FsObjectBackend extends ObjectBackend { const objectPath = path.join(this.baseObjectPath, sanitize(id)); if (!fs.existsSync(objectPath)) return true; fs.rmSync(objectPath); + // remove item from cache + this.cache.delete(id); return true; } async fetchMetadata( @@ -125,4 +138,26 @@ export class FsObjectBackend extends ObjectBackend { fs.writeFileSync(metadataPath, JSON.stringify(metadata)); return true; } + async fetchHash(id: ObjectReference): Promise { + const cacheResult = this.cache.get(id); + if (cacheResult !== undefined) return cacheResult; + + const obj = await this.fetch(id); + if (obj === undefined) return; + + // local variable to point to object + const cache = this.cache; + + // hash object + const hash = createHash("md5"); + hash.setEncoding("hex"); + obj.on("end", function () { + hash.end(); + cache.set(id, hash.read()); + }); + // read obj into hash + obj.pipe(hash); + + return this.cache.get(id); + } } diff --git a/server/internal/objects/objectHandler.ts b/server/internal/objects/objectHandler.ts index 84039e4..e51cc7b 100644 --- a/server/internal/objects/objectHandler.ts +++ b/server/internal/objects/objectHandler.ts @@ -49,20 +49,21 @@ export abstract class ObjectBackend { abstract create( id: string, source: Source, - metadata: ObjectMetadata + metadata: ObjectMetadata, ): Promise; abstract createWithWriteStream( id: string, - metadata: ObjectMetadata + metadata: ObjectMetadata, ): Promise; abstract delete(id: ObjectReference): Promise; abstract fetchMetadata( - id: ObjectReference + id: ObjectReference, ): Promise; abstract writeMetadata( id: ObjectReference, - metadata: ObjectMetadata + metadata: ObjectMetadata, ): Promise; + abstract fetchHash(id: ObjectReference): Promise; private async fetchMimeType(source: Source) { if (source instanceof ReadableStream) { @@ -86,7 +87,7 @@ export abstract class ObjectBackend { id: string, sourceFetcher: () => Promise, metadata: { [key: string]: string }, - permissions: Array + permissions: Array, ) { const { source, mime } = await this.fetchMimeType(await sourceFetcher()); if (!mime) @@ -102,7 +103,7 @@ export abstract class ObjectBackend { async createWithStream( id: string, metadata: { [key: string]: string }, - permissions: Array + permissions: Array, ) { return this.createWithWriteStream(id, { permissions, @@ -111,6 +112,12 @@ export abstract class ObjectBackend { }); } + /** + * Fetches object, but also checks if user has perms to access it + * @param id + * @param userId + * @returns + */ async fetchWithPermissions(id: ObjectReference, userId?: string) { const metadata = await this.fetchMetadata(id); if (!metadata) return; @@ -147,7 +154,7 @@ export abstract class ObjectBackend { async writeWithPermissions( id: ObjectReference, sourceFetcher: () => Promise, - userId?: string + userId?: string, ) { const metadata = await this.fetchMetadata(id); if (!metadata) return false;