Mintplex-Labs/vector-admin
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/vector-admin.git synced 2026-08-27 10:31:23 -04:00
Code Issues 79 Packages Projects Releases Wiki Activity

Exact text search

Browse Source
This commit is contained in:
timothycarambat
2023-09-13 19:02:42 -07:00
parent c71b8b959b
commit fbbcbdb36d
6 changed files with 172 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/endpoints/v1/workspaces/index.js
+40
View File
@@ -22,6 +22,9 @@ const {
createWorkspaceSyncJob,
} = require("../../../utils/jobs/createWorkspaceSyncJob");
const { cloneWorkspaceJob } = require("../../../utils/jobs/cloneWorkspaceJob");
const {
workspaceDocumentSearch,
} = require("../../../utils/search/workspaceDocuments");
process.env.NODE_ENV === "development"
? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` })
@@ -443,6 +446,43 @@ function workspaceEndpoints(app) {
}
}
);
app.get(
"/v1/workspace/:workspaceId/search-documents",
[validSessionForUser],
async function (request, response) {
try {
const { workspaceId } = request.params;
const { method, q: query } = request.query;
const user = await userFromSession(request);
if (!user) {
response.sendStatus(403).end();
return;
}
const workspace = await OrganizationWorkspace.get(
`id = ${workspaceId}`
);
if (!workspace) {
response.status(200).json({
documents: [],
error: "No workspace found.",
});
return;
}
const { documents, error } = await workspaceDocumentSearch(
workspace,
method,
query
);
response.status(200).json({ documents, error });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
}
module.exports = { workspaceEndpoints };
backend/utils/search/workspaceDocuments/exactText.js
+51
View File
@@ -0,0 +1,51 @@
const { WorkspaceDocument } = require("../../../models/workspaceDocument");
const { readJSON } = require("../../storage");
// Dirty, but works fast for most cases. Wont be perfect but also not something we should rely
// heavily on for exact text searching.
function fuzzyMatch(pattern, str) {
pattern = ".*" + pattern.split("").join(".*") + ".*";
const re = new RegExp(pattern);
return re.test(str);
}
async function findTextInDoc(wsDoc, query) {
try {
const data = await readJSON(WorkspaceDocument.vectorFilepath(wsDoc));
for (const chunk of data) {
if (!chunk.hasOwnProperty("metadata")) continue;
for (const value of Object.values(chunk?.metadata)) {
const valid = fuzzyMatch(query, String(value));
if (valid) return wsDoc;
}
}
return false;
} catch (e) {
console.error(e);
return false;
}
}
async function exactTextSearch(workspace, query) {
const workspaceDocs = await WorkspaceDocument.where(
`workspace_id = ${workspace.id}`
);
const promises = [];
for (const wsDoc of workspaceDocs) {
promises.push(
new Promise(async (resolve) => {
resolve(await findTextInDoc(wsDoc, query));
})
);
}
const matches = (await Promise.all(promises)).filter((res) => res !== false);
return { documents: matches, error: null };
}
module.exports = {
exactTextSearch,
};
backend/utils/search/workspaceDocuments/index.js
+34
View File
@@ -0,0 +1,34 @@
// export type ISearchTypes = 'semantic' | 'exactText' | 'metadata' | 'vectorId';
const { exactTextSearch } = require("./exactText");
const SEARCH_METHODS = {
semantic: () => {
return { documents: [], error: "unsupported" };
},
exactText: exactTextSearch,
metadata: () => {
return { documents: [], error: "unsupported" };
},
vectorId: () => {
return { documents: [], error: "unsupported" };
},
};
function validSearchMethod(method) {
return Object.keys(SEARCH_METHODS).includes(method);
}
async function workspaceDocumentSearch(workspace, method, query) {
try {
if (!validSearchMethod(method))
throw new Error(`Invalid search method ${method}`);
return await SEARCH_METHODS[method](workspace, query);
} catch (e) {
console.error("Workspace document search", e.message);
return { documents: [], error: e.message };
}
}
module.exports = {
workspaceDocumentSearch,
};
frontend/src/models/workspace.ts
+26 -3
View File
@@ -1,3 +1,4 @@
import { ISearchTypes } from '../pages/WorkspaceDashboard/DocumentsList/SearchView';
import { API_BASE } from '../utils/constants';
import { baseHeaders } from '../utils/request';
@@ -87,8 +88,7 @@ const Workspace = {
pageSize?: number
) => {
return fetch(
`${API_BASE}/v1/org/${orgSlug}/workspace/${workspaceSlug}/documents?page=${page}&pageSize=${
pageSize || Workspace.documentPageSize
`${API_BASE}/v1/org/${orgSlug}/workspace/${workspaceSlug}/documents?page=${page}&pageSize=${pageSize || Workspace.documentPageSize
}`,
{
method: 'GET',
@@ -175,6 +175,29 @@ const Workspace = {
return { success: false, error: e.message };
});
},
};
searchDocuments: async (
workspaceId: number,
method: ISearchTypes,
query: string,
): Promise<{ documents: object[] }> => {
const searchEndpoint = new URL(`${API_BASE}/v1/workspace/${workspaceId}/search-documents`)
searchEndpoint.searchParams.append('method', method);
searchEndpoint.searchParams.append('q', encodeURIComponent(query));
return await fetch(
searchEndpoint,
{
method: 'GET',
headers: baseHeaders(),
}
)
.then((res) => res.json())
.then((res) => res?.documents || [])
.catch((e) => {
console.error(e.message);
return [];
});
},
}
export default Workspace;
frontend/src/pages/WorkspaceDashboard/DocumentsList/SearchView/index.tsx
+18 -9
View File
@@ -1,11 +1,12 @@
import { useState } from 'react';
import { ChevronDown, FileText, Search } from 'react-feather';
import { SyntheticEvent, useState } from 'react';
import { ChevronDown, FileText, Search, Loader } from 'react-feather';
import { CopyDocToModal } from '..';
import truncate from 'truncate';
import moment from 'moment';
import paths from '../../../../utils/paths';
import Workspace from '../../../../models/workspace';
type ISearchTypes = 'semantic' | 'exactText' | 'metadata' | 'vectorId';
export type ISearchTypes = 'semantic' | 'exactText' | 'metadata' | 'vectorId';
const SEARCH_MODES = {
exactText: {
@@ -44,19 +45,25 @@ export default function SearchView({
const [searching, setSearching] = useState(false);
const [showSearchMethods, setShowSearchMethods] = useState(false);
const [searchBy, setSearchBy] = useState<ISearchTypes>('exactText');
const [searchTerm, setSearchTem] = useState('');
const [documents, setDocuments] = useState([]);
const clearSearch = () => {
setSearchBy('semantic');
setSearchTem('');
setDocuments([]);
setSearching(false);
stopSearching();
};
const handleSearch = async (e: SyntheticEvent<HTMLElement, SubmitEvent>) => {
e.preventDefault();
const formData = new FormData(e.target as any)
// setSearching(true)
const matches = await Workspace.searchDocuments(workspace.id, searchBy, formData.get('query') as string);
setDocuments(matches);
}
return (
<div className="col-span-12 flex-1 rounded-sm border border-stroke bg-white py-6 shadow-default dark:border-strokedark dark:bg-boxdark xl:col-span-4">
<div className="mx-4 mb-6 flex items-center">
<form className="w-full">
<form onSubmit={handleSearch} className="w-full">
<div className="relative flex">
<button
onClick={() => setShowSearchMethods(!showSearchMethods)}
@@ -97,15 +104,17 @@ export default function SearchView({
<div className="relative w-full">
<input
type="search"
name='query'
className="z-20 block w-full rounded-r-lg border border-l-2 border-gray-300 border-l-gray-50 bg-gray-50 p-2.5 text-sm text-gray-900 focus:border-blue-500 focus:ring-blue-500 dark:border-gray-600 dark:border-l-gray-700 dark:bg-gray-700 dark:text-white dark:placeholder-gray-400 dark:focus:border-blue-500"
placeholder={SEARCH_MODES[searchBy].placeholder}
required
/>
<button
type="submit"
disabled={searching}
className="absolute right-0 top-0 h-full rounded-r-lg border border-blue-700 bg-blue-700 p-2.5 text-sm font-medium text-white hover:bg-blue-800 focus:outline-none focus:ring-4 focus:ring-blue-300 dark:bg-blue-600 dark:hover:bg-blue-700 dark:focus:ring-blue-800"
>
<Search size={18} />
{searching ? <Loader size={18} className='animate-spin' /> : <Search size={18} />}
<span className="sr-only">Search</span>
</button>
</div>
@@ -160,7 +169,7 @@ export default function SearchView({
</div>
<div className="w-6/12 2xsm:w-5/12 md:w-3/12">
<span className="font-medium">
{document.workspace.name || ''}
{workspace.name || ''}
</span>
</div>
<div className="hidden w-3/12 overflow-x-scroll md:block xl:w-3/12">
@@ -191,7 +200,7 @@ export default function SearchView({
<a
href={paths.document(
organization.slug,
document.workspace.slug,
workspace.slug,
document.id
)}
className="rounded-lg px-2 py-1 text-blue-400 transition-all duration-300 hover:bg-blue-50 hover:text-blue-600"
frontend/src/pages/WorkspaceDashboard/DocumentsList/index.tsx
+3 -3
View File
@@ -116,9 +116,9 @@ export default function DocumentsList({
onClick={() => setSearchMode(true)}
className="group flex items-center gap-2 rounded-lg bg-transparent px-4 py-2 hover:bg-blue-300/20"
>
<Search size={20} className="text-blue-600" />
<p className="pointer-events-none translate-y-[20px] text-blue-600 opacity-0 transition-all duration-[300ms] group-hover:translate-y-[0px] group-hover:opacity-100">
Search documents in workspace
<Search size={20} className="text-gray-600 group-hover:text-blue-600" />
<p className="pointer-events-none text-gray-600 group-hover:text-blue-600 transition-all duration-[300ms]">
Search documents
</p>
</button>
</div>