diff --git a/backend/endpoints/v1/workspaces/index.js b/backend/endpoints/v1/workspaces/index.js index cb6b4e3..4a98ac1 100644 --- a/backend/endpoints/v1/workspaces/index.js +++ b/backend/endpoints/v1/workspaces/index.js @@ -22,6 +22,9 @@ const { createWorkspaceSyncJob, } = require("../../../utils/jobs/createWorkspaceSyncJob"); const { cloneWorkspaceJob } = require("../../../utils/jobs/cloneWorkspaceJob"); +const { + workspaceDocumentSearch, +} = require("../../../utils/search/workspaceDocuments"); process.env.NODE_ENV === "development" ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` }) @@ -443,6 +446,43 @@ function workspaceEndpoints(app) { } } ); + + app.get( + "/v1/workspace/:workspaceId/search-documents", + [validSessionForUser], + async function (request, response) { + try { + const { workspaceId } = request.params; + const { method, q: query } = request.query; + const user = await userFromSession(request); + if (!user) { + response.sendStatus(403).end(); + return; + } + + const workspace = await OrganizationWorkspace.get( + `id = ${workspaceId}` + ); + if (!workspace) { + response.status(200).json({ + documents: [], + error: "No workspace found.", + }); + return; + } + + const { documents, error } = await workspaceDocumentSearch( + workspace, + method, + query + ); + response.status(200).json({ documents, error }); + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); } module.exports = { workspaceEndpoints }; diff --git a/backend/utils/search/workspaceDocuments/exactText.js b/backend/utils/search/workspaceDocuments/exactText.js new file mode 100644 index 0000000..797e543 --- /dev/null +++ b/backend/utils/search/workspaceDocuments/exactText.js @@ -0,0 +1,51 @@ +const { WorkspaceDocument } = require("../../../models/workspaceDocument"); +const { readJSON } = require("../../storage"); + +// Dirty, but works fast for most cases. Wont be perfect but also not something we should rely +// heavily on for exact text searching. +function fuzzyMatch(pattern, str) { + pattern = ".*" + pattern.split("").join(".*") + ".*"; + const re = new RegExp(pattern); + return re.test(str); +} + +async function findTextInDoc(wsDoc, query) { + try { + const data = await readJSON(WorkspaceDocument.vectorFilepath(wsDoc)); + + for (const chunk of data) { + if (!chunk.hasOwnProperty("metadata")) continue; + for (const value of Object.values(chunk?.metadata)) { + const valid = fuzzyMatch(query, String(value)); + if (valid) return wsDoc; + } + } + + return false; + } catch (e) { + console.error(e); + return false; + } +} + +async function exactTextSearch(workspace, query) { + const workspaceDocs = await WorkspaceDocument.where( + `workspace_id = ${workspace.id}` + ); + const promises = []; + + for (const wsDoc of workspaceDocs) { + promises.push( + new Promise(async (resolve) => { + resolve(await findTextInDoc(wsDoc, query)); + }) + ); + } + + const matches = (await Promise.all(promises)).filter((res) => res !== false); + return { documents: matches, error: null }; +} + +module.exports = { + exactTextSearch, +}; diff --git a/backend/utils/search/workspaceDocuments/index.js b/backend/utils/search/workspaceDocuments/index.js new file mode 100644 index 0000000..5e5555d --- /dev/null +++ b/backend/utils/search/workspaceDocuments/index.js @@ -0,0 +1,34 @@ +// export type ISearchTypes = 'semantic' | 'exactText' | 'metadata' | 'vectorId'; +const { exactTextSearch } = require("./exactText"); + +const SEARCH_METHODS = { + semantic: () => { + return { documents: [], error: "unsupported" }; + }, + exactText: exactTextSearch, + metadata: () => { + return { documents: [], error: "unsupported" }; + }, + vectorId: () => { + return { documents: [], error: "unsupported" }; + }, +}; + +function validSearchMethod(method) { + return Object.keys(SEARCH_METHODS).includes(method); +} + +async function workspaceDocumentSearch(workspace, method, query) { + try { + if (!validSearchMethod(method)) + throw new Error(`Invalid search method ${method}`); + return await SEARCH_METHODS[method](workspace, query); + } catch (e) { + console.error("Workspace document search", e.message); + return { documents: [], error: e.message }; + } +} + +module.exports = { + workspaceDocumentSearch, +}; diff --git a/frontend/src/models/workspace.ts b/frontend/src/models/workspace.ts index c4a1666..df4ae4b 100644 --- a/frontend/src/models/workspace.ts +++ b/frontend/src/models/workspace.ts @@ -1,3 +1,4 @@ +import { ISearchTypes } from '../pages/WorkspaceDashboard/DocumentsList/SearchView'; import { API_BASE } from '../utils/constants'; import { baseHeaders } from '../utils/request'; @@ -87,8 +88,7 @@ const Workspace = { pageSize?: number ) => { return fetch( - `${API_BASE}/v1/org/${orgSlug}/workspace/${workspaceSlug}/documents?page=${page}&pageSize=${ - pageSize || Workspace.documentPageSize + `${API_BASE}/v1/org/${orgSlug}/workspace/${workspaceSlug}/documents?page=${page}&pageSize=${pageSize || Workspace.documentPageSize }`, { method: 'GET', @@ -175,6 +175,29 @@ const Workspace = { return { success: false, error: e.message }; }); }, -}; + searchDocuments: async ( + workspaceId: number, + method: ISearchTypes, + query: string, + ): Promise<{ documents: object[] }> => { + const searchEndpoint = new URL(`${API_BASE}/v1/workspace/${workspaceId}/search-documents`) + searchEndpoint.searchParams.append('method', method); + searchEndpoint.searchParams.append('q', encodeURIComponent(query)); + return await fetch( + searchEndpoint, + { + method: 'GET', + headers: baseHeaders(), + } + ) + .then((res) => res.json()) + .then((res) => res?.documents || []) + .catch((e) => { + console.error(e.message); + return []; + }); + }, + +} export default Workspace; diff --git a/frontend/src/pages/WorkspaceDashboard/DocumentsList/SearchView/index.tsx b/frontend/src/pages/WorkspaceDashboard/DocumentsList/SearchView/index.tsx index 7ff235d..17c3048 100644 --- a/frontend/src/pages/WorkspaceDashboard/DocumentsList/SearchView/index.tsx +++ b/frontend/src/pages/WorkspaceDashboard/DocumentsList/SearchView/index.tsx @@ -1,11 +1,12 @@ -import { useState } from 'react'; -import { ChevronDown, FileText, Search } from 'react-feather'; +import { SyntheticEvent, useState } from 'react'; +import { ChevronDown, FileText, Search, Loader } from 'react-feather'; import { CopyDocToModal } from '..'; import truncate from 'truncate'; import moment from 'moment'; import paths from '../../../../utils/paths'; +import Workspace from '../../../../models/workspace'; -type ISearchTypes = 'semantic' | 'exactText' | 'metadata' | 'vectorId'; +export type ISearchTypes = 'semantic' | 'exactText' | 'metadata' | 'vectorId'; const SEARCH_MODES = { exactText: { @@ -44,19 +45,25 @@ export default function SearchView({ const [searching, setSearching] = useState(false); const [showSearchMethods, setShowSearchMethods] = useState(false); const [searchBy, setSearchBy] = useState('exactText'); - const [searchTerm, setSearchTem] = useState(''); const [documents, setDocuments] = useState([]); const clearSearch = () => { setSearchBy('semantic'); - setSearchTem(''); setDocuments([]); + setSearching(false); stopSearching(); }; + const handleSearch = async (e: SyntheticEvent) => { + e.preventDefault(); + const formData = new FormData(e.target as any) + // setSearching(true) + const matches = await Workspace.searchDocuments(workspace.id, searchBy, formData.get('query') as string); + setDocuments(matches); + } return (
-
+
@@ -160,7 +169,7 @@ export default function SearchView({
- {document.workspace.name || ''} + {workspace.name || ''}
@@ -191,7 +200,7 @@ export default function SearchView({ setSearchMode(true)} className="group flex items-center gap-2 rounded-lg bg-transparent px-4 py-2 hover:bg-blue-300/20" > - -

- Search documents in workspace + +

+ Search documents