mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2026-07-21 09:35:22 -04:00
format
This commit is contained in:
Marcello Fitton
@@ -1506,8 +1506,8 @@ function systemEndpoints(app) {
|
||||
"/system/validate-sql-connection",
|
||||
[validatedRequest, flexUserRoleValid([ROLES.admin])],
|
||||
async (request, response) => {
|
||||
const { engine, connectionString } = reqBody(request);
|
||||
try {
|
||||
const { engine, connectionString } = reqBody(request);
|
||||
if (!engine || !connectionString) {
|
||||
return response.status(400).json({
|
||||
success: false,
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
const path = require('node:path');
|
||||
const fs = require('node:fs');
|
||||
const { parentPort } = require('node:worker_threads');
|
||||
const path = require("node:path");
|
||||
const fs = require("node:fs");
|
||||
const { parentPort } = require("node:worker_threads");
|
||||
const documentsPath =
|
||||
process.env.NODE_ENV === "development"
|
||||
? path.resolve(__dirname, `../../storage/documents`)
|
||||
: path.resolve(process.env.STORAGE_DIR, `documents`);
|
||||
|
||||
function log(stringContent = '') {
|
||||
if (parentPort) parentPort.postMessage(`\x1b[33m[${process.pid}]\x1b[0m: ${stringContent}`); // running as worker
|
||||
else process.send(`\x1b[33m[${process.ppid}:${process.pid}]\x1b[0m: ${stringContent}`); // running as child_process
|
||||
function log(stringContent = "") {
|
||||
if (parentPort)
|
||||
parentPort.postMessage(`\x1b[33m[${process.pid}]\x1b[0m: ${stringContent}`); // running as worker
|
||||
else
|
||||
process.send(
|
||||
`\x1b[33m[${process.ppid}:${process.pid}]\x1b[0m: ${stringContent}`
|
||||
); // running as child_process
|
||||
}
|
||||
|
||||
function conclude() {
|
||||
if (parentPort) parentPort.postMessage('done');
|
||||
if (parentPort) parentPort.postMessage("done");
|
||||
else process.exit(0);
|
||||
}
|
||||
|
||||
@@ -27,4 +31,4 @@ module.exports = {
|
||||
log,
|
||||
conclude,
|
||||
updateSourceDocument,
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,152 +1,208 @@
|
||||
const { Document } = require('../models/documents.js');
|
||||
const { DocumentSyncQueue } = require('../models/documentSyncQueue.js');
|
||||
const { CollectorApi } = require('../utils/collectorApi');
|
||||
const { Document } = require("../models/documents.js");
|
||||
const { DocumentSyncQueue } = require("../models/documentSyncQueue.js");
|
||||
const { CollectorApi } = require("../utils/collectorApi");
|
||||
const { fileData } = require("../utils/files");
|
||||
const { log, conclude, updateSourceDocument } = require('./helpers/index.js');
|
||||
const { getVectorDbClass } = require('../utils/helpers/index.js');
|
||||
const { DocumentSyncRun } = require('../models/documentSyncRun.js');
|
||||
const { log, conclude, updateSourceDocument } = require("./helpers/index.js");
|
||||
const { getVectorDbClass } = require("../utils/helpers/index.js");
|
||||
const { DocumentSyncRun } = require("../models/documentSyncRun.js");
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const queuesToProcess = await DocumentSyncQueue.staleDocumentQueues();
|
||||
if (queuesToProcess.length === 0) {
|
||||
log('No outstanding documents to sync. Exiting.');
|
||||
log("No outstanding documents to sync. Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
const collector = new CollectorApi();
|
||||
if (!(await collector.online())) {
|
||||
log('Could not reach collector API. Exiting.');
|
||||
log("Could not reach collector API. Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
log(`${queuesToProcess.length} watched documents have been found to be stale and will be updated now.`)
|
||||
log(
|
||||
`${queuesToProcess.length} watched documents have been found to be stale and will be updated now.`
|
||||
);
|
||||
for (const queue of queuesToProcess) {
|
||||
let newContent = null;
|
||||
const document = queue.workspaceDoc;
|
||||
const workspace = document.workspace;
|
||||
const { metadata, type, source } = Document.parseDocumentTypeAndSource(document);
|
||||
const { metadata, type, source } =
|
||||
Document.parseDocumentTypeAndSource(document);
|
||||
|
||||
if (!metadata || !DocumentSyncQueue.validFileTypes.includes(type)) {
|
||||
// Document is either broken, invalid, or not supported so drop it from future queues.
|
||||
log(`Document ${document.filename} has no metadata, is broken, or invalid and has been removed from all future runs.`)
|
||||
log(
|
||||
`Document ${document.filename} has no metadata, is broken, or invalid and has been removed from all future runs.`
|
||||
);
|
||||
await DocumentSyncQueue.unwatch(document);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (['link', 'youtube'].includes(type)) {
|
||||
if (["link", "youtube"].includes(type)) {
|
||||
const response = await collector.forwardExtensionRequest({
|
||||
endpoint: "/ext/resync-source-document",
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
type,
|
||||
options: { link: source }
|
||||
})
|
||||
options: { link: source },
|
||||
}),
|
||||
});
|
||||
newContent = response?.content;
|
||||
}
|
||||
|
||||
if (['confluence', 'github', 'gitlab', 'drupalwiki'].includes(type)) {
|
||||
if (["confluence", "github", "gitlab", "drupalwiki"].includes(type)) {
|
||||
const response = await collector.forwardExtensionRequest({
|
||||
endpoint: "/ext/resync-source-document",
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
type,
|
||||
options: { chunkSource: metadata.chunkSource }
|
||||
})
|
||||
options: { chunkSource: metadata.chunkSource },
|
||||
}),
|
||||
});
|
||||
newContent = response?.content;
|
||||
}
|
||||
|
||||
if (!newContent) {
|
||||
// Check if the last "x" runs were all failures (not exits!). If so - remove the job entirely since it is broken.
|
||||
const failedRunCount = (await DocumentSyncRun.where({ queueId: queue.id }, DocumentSyncQueue.maxRepeatFailures, { createdAt: 'desc' })).filter((run) => run.status === DocumentSyncRun.statuses.failed).length;
|
||||
const failedRunCount = (
|
||||
await DocumentSyncRun.where(
|
||||
{ queueId: queue.id },
|
||||
DocumentSyncQueue.maxRepeatFailures,
|
||||
{ createdAt: "desc" }
|
||||
)
|
||||
).filter(
|
||||
(run) => run.status === DocumentSyncRun.statuses.failed
|
||||
).length;
|
||||
if (failedRunCount >= DocumentSyncQueue.maxRepeatFailures) {
|
||||
log(`Document ${document.filename} has failed to refresh ${failedRunCount} times continuously and will now be removed from the watched document set.`)
|
||||
log(
|
||||
`Document ${document.filename} has failed to refresh ${failedRunCount} times continuously and will now be removed from the watched document set.`
|
||||
);
|
||||
await DocumentSyncQueue.unwatch(document);
|
||||
continue;
|
||||
}
|
||||
|
||||
log(`Failed to get a new content response from collector for source ${source}. Skipping, but will retry next worker interval. Attempt ${failedRunCount === 0 ? 1 : failedRunCount}/${DocumentSyncQueue.maxRepeatFailures}`);
|
||||
await DocumentSyncQueue.saveRun(queue.id, DocumentSyncRun.statuses.failed, { filename: document.filename, workspacesModified: [], reason: 'No content found.' })
|
||||
log(
|
||||
`Failed to get a new content response from collector for source ${source}. Skipping, but will retry next worker interval. Attempt ${failedRunCount === 0 ? 1 : failedRunCount}/${DocumentSyncQueue.maxRepeatFailures}`
|
||||
);
|
||||
await DocumentSyncQueue.saveRun(
|
||||
queue.id,
|
||||
DocumentSyncRun.statuses.failed,
|
||||
{
|
||||
filename: document.filename,
|
||||
workspacesModified: [],
|
||||
reason: "No content found.",
|
||||
}
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const currentDocumentData = await fileData(document.docpath)
|
||||
const currentDocumentData = await fileData(document.docpath);
|
||||
if (currentDocumentData.pageContent === newContent) {
|
||||
const nextSync = DocumentSyncQueue.calcNextSync(queue)
|
||||
log(`Source ${source} is unchanged and will be skipped. Next sync will be ${nextSync.toLocaleString()}.`);
|
||||
await DocumentSyncQueue._update(
|
||||
const nextSync = DocumentSyncQueue.calcNextSync(queue);
|
||||
log(
|
||||
`Source ${source} is unchanged and will be skipped. Next sync will be ${nextSync.toLocaleString()}.`
|
||||
);
|
||||
await DocumentSyncQueue._update(queue.id, {
|
||||
lastSyncedAt: new Date().toISOString(),
|
||||
nextSyncAt: nextSync.toISOString(),
|
||||
});
|
||||
await DocumentSyncQueue.saveRun(
|
||||
queue.id,
|
||||
DocumentSyncRun.statuses.exited,
|
||||
{
|
||||
lastSyncedAt: new Date().toISOString(),
|
||||
nextSyncAt: nextSync.toISOString(),
|
||||
filename: document.filename,
|
||||
workspacesModified: [],
|
||||
reason: "Content unchanged.",
|
||||
}
|
||||
);
|
||||
await DocumentSyncQueue.saveRun(queue.id, DocumentSyncRun.statuses.exited, { filename: document.filename, workspacesModified: [], reason: 'Content unchanged.' })
|
||||
continue;
|
||||
}
|
||||
|
||||
// update the defined document and workspace vectorDB with the latest information
|
||||
// it will skip cache and create a new vectorCache file.
|
||||
const vectorDatabase = getVectorDbClass();
|
||||
await vectorDatabase.deleteDocumentFromNamespace(workspace.slug, document.docId);
|
||||
await vectorDatabase.deleteDocumentFromNamespace(
|
||||
workspace.slug,
|
||||
document.docId
|
||||
);
|
||||
await vectorDatabase.addDocumentToNamespace(
|
||||
workspace.slug,
|
||||
{ ...currentDocumentData, pageContent: newContent, docId: document.docId },
|
||||
document.docpath,
|
||||
true
|
||||
);
|
||||
updateSourceDocument(
|
||||
document.docpath,
|
||||
{
|
||||
...currentDocumentData,
|
||||
pageContent: newContent,
|
||||
docId: document.docId,
|
||||
published: (new Date).toLocaleString(),
|
||||
// Todo: Update word count and token_estimate?
|
||||
}
|
||||
)
|
||||
log(`Workspace "${workspace.name}" vectors of ${source} updated. Document and vector cache updated.`)
|
||||
|
||||
},
|
||||
document.docpath,
|
||||
true
|
||||
);
|
||||
updateSourceDocument(document.docpath, {
|
||||
...currentDocumentData,
|
||||
pageContent: newContent,
|
||||
docId: document.docId,
|
||||
published: new Date().toLocaleString(),
|
||||
// Todo: Update word count and token_estimate?
|
||||
});
|
||||
log(
|
||||
`Workspace "${workspace.name}" vectors of ${source} updated. Document and vector cache updated.`
|
||||
);
|
||||
|
||||
// Now we can bloom the results to all matching documents in all other workspaces
|
||||
const workspacesModified = [workspace.slug];
|
||||
const moreReferences = await Document.where({
|
||||
id: { not: document.id },
|
||||
filename: document.filename
|
||||
}, null, null, { workspace: true });
|
||||
const moreReferences = await Document.where(
|
||||
{
|
||||
id: { not: document.id },
|
||||
filename: document.filename,
|
||||
},
|
||||
null,
|
||||
null,
|
||||
{ workspace: true }
|
||||
);
|
||||
|
||||
if (moreReferences.length !== 0) {
|
||||
log(`${source} is referenced in ${moreReferences.length} other workspaces. Updating those workspaces as well...`)
|
||||
log(
|
||||
`${source} is referenced in ${moreReferences.length} other workspaces. Updating those workspaces as well...`
|
||||
);
|
||||
for (const additionalDocumentRef of moreReferences) {
|
||||
const additionalWorkspace = additionalDocumentRef.workspace;
|
||||
workspacesModified.push(additionalWorkspace.slug);
|
||||
|
||||
await vectorDatabase.deleteDocumentFromNamespace(additionalWorkspace.slug, additionalDocumentRef.docId);
|
||||
await vectorDatabase.deleteDocumentFromNamespace(
|
||||
additionalWorkspace.slug,
|
||||
additionalDocumentRef.docId
|
||||
);
|
||||
await vectorDatabase.addDocumentToNamespace(
|
||||
additionalWorkspace.slug,
|
||||
{ ...currentDocumentData, pageContent: newContent, docId: additionalDocumentRef.docId },
|
||||
additionalDocumentRef.docpath,
|
||||
{
|
||||
...currentDocumentData,
|
||||
pageContent: newContent,
|
||||
docId: additionalDocumentRef.docId,
|
||||
},
|
||||
additionalDocumentRef.docpath
|
||||
);
|
||||
log(
|
||||
`Workspace "${additionalWorkspace.name}" vectors for ${source} was also updated with the new content from cache.`
|
||||
);
|
||||
log(`Workspace "${additionalWorkspace.name}" vectors for ${source} was also updated with the new content from cache.`)
|
||||
}
|
||||
}
|
||||
|
||||
const nextRefresh = DocumentSyncQueue.calcNextSync(queue);
|
||||
log(`${source} has been refreshed in all workspaces it is currently referenced in. Next refresh will be ${nextRefresh.toLocaleString()}.`)
|
||||
await DocumentSyncQueue._update(
|
||||
queue.id,
|
||||
{
|
||||
lastSyncedAt: new Date().toISOString(),
|
||||
nextSyncAt: nextRefresh.toISOString(),
|
||||
}
|
||||
log(
|
||||
`${source} has been refreshed in all workspaces it is currently referenced in. Next refresh will be ${nextRefresh.toLocaleString()}.`
|
||||
);
|
||||
await DocumentSyncQueue._update(queue.id, {
|
||||
lastSyncedAt: new Date().toISOString(),
|
||||
nextSyncAt: nextRefresh.toISOString(),
|
||||
});
|
||||
await DocumentSyncQueue.saveRun(
|
||||
queue.id,
|
||||
DocumentSyncRun.statuses.success,
|
||||
{ filename: document.filename, workspacesModified }
|
||||
);
|
||||
await DocumentSyncQueue.saveRun(queue.id, DocumentSyncRun.statuses.success, { filename: document.filename, workspacesModified })
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
log(`errored with ${e.message}`)
|
||||
console.error(e);
|
||||
log(`errored with ${e.message}`);
|
||||
} finally {
|
||||
conclude();
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
function waitForElm(selector) {
|
||||
return new Promise(resolve => {
|
||||
return new Promise((resolve) => {
|
||||
if (document.querySelector(selector)) {
|
||||
return resolve(document.querySelector(selector));
|
||||
}
|
||||
|
||||
const observer = new MutationObserver(mutations => {
|
||||
const observer = new MutationObserver((mutations) => {
|
||||
if (document.querySelector(selector)) {
|
||||
resolve(document.querySelector(selector));
|
||||
observer.disconnect();
|
||||
@@ -13,16 +13,16 @@ function waitForElm(selector) {
|
||||
|
||||
observer.observe(document.body, {
|
||||
childList: true,
|
||||
subtree: true
|
||||
subtree: true,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Force change the Swagger logo in the header
|
||||
waitForElm('.topbar-wrapper').then((elm) => {
|
||||
if (window.SWAGGER_DOCS_ENV === 'development') {
|
||||
elm.innerHTML = `<img href='${window.location.origin}' src='http://localhost:3000/public/anything-llm-light.png' width='200'/>`
|
||||
waitForElm(".topbar-wrapper").then((elm) => {
|
||||
if (window.SWAGGER_DOCS_ENV === "development") {
|
||||
elm.innerHTML = `<img href='${window.location.origin}' src='http://localhost:3000/public/anything-llm-light.png' width='200'/>`;
|
||||
} else {
|
||||
elm.innerHTML = `<img href='${window.location.origin}' src='${window.location.origin}/anything-llm-light.png' width='200'/>`
|
||||
elm.innerHTML = `<img href='${window.location.origin}' src='${window.location.origin}/anything-llm-light.png' width='200'/>`;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
+34
-36
@@ -1,11 +1,11 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const swaggerUi = require('swagger-ui-express');
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const swaggerUi = require("swagger-ui-express");
|
||||
|
||||
function faviconUrl() {
|
||||
return process.env.NODE_ENV === "production" ?
|
||||
'/public/favicon.png' :
|
||||
'http://localhost:3000/public/favicon.png'
|
||||
return process.env.NODE_ENV === "production"
|
||||
? "/public/favicon.png"
|
||||
: "http://localhost:3000/public/favicon.png";
|
||||
}
|
||||
|
||||
function useSwagger(app) {
|
||||
@@ -15,44 +15,42 @@ function useSwagger(app) {
|
||||
);
|
||||
return;
|
||||
}
|
||||
app.use('/api/docs', swaggerUi.serve);
|
||||
app.use("/api/docs", swaggerUi.serve);
|
||||
const options = {
|
||||
customCss: [
|
||||
fs.readFileSync(path.resolve(__dirname, 'index.css')),
|
||||
fs.readFileSync(path.resolve(__dirname, 'dark-swagger.css'))
|
||||
].join('\n\n\n'),
|
||||
customSiteTitle: 'AnythingLLM Developer API Documentation',
|
||||
fs.readFileSync(path.resolve(__dirname, "index.css")),
|
||||
fs.readFileSync(path.resolve(__dirname, "dark-swagger.css")),
|
||||
].join("\n\n\n"),
|
||||
customSiteTitle: "AnythingLLM Developer API Documentation",
|
||||
customfavIcon: faviconUrl(),
|
||||
}
|
||||
};
|
||||
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
const swaggerDocument = require('./openapi.json');
|
||||
app.get('/api/docs', swaggerUi.setup(
|
||||
swaggerDocument,
|
||||
{
|
||||
...options,
|
||||
customJsStr: 'window.SWAGGER_DOCS_ENV = "production";\n\n' + fs.readFileSync(path.resolve(__dirname, 'index.js'), 'utf8'),
|
||||
},
|
||||
));
|
||||
} else {
|
||||
// we regenerate the html page only in development mode to ensure it is up-to-date when the code is hot-reloaded.
|
||||
const swaggerDocument = require("./openapi.json");
|
||||
app.get(
|
||||
"/api/docs",
|
||||
async (_, response) => {
|
||||
// #swagger.ignore = true
|
||||
const swaggerDocument = require('./openapi.json');
|
||||
return response.send(
|
||||
swaggerUi.generateHTML(
|
||||
swaggerDocument,
|
||||
{
|
||||
...options,
|
||||
customJsStr: 'window.SWAGGER_DOCS_ENV = "development";\n\n' + fs.readFileSync(path.resolve(__dirname, 'index.js'), 'utf8'),
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
swaggerUi.setup(swaggerDocument, {
|
||||
...options,
|
||||
customJsStr:
|
||||
'window.SWAGGER_DOCS_ENV = "production";\n\n' +
|
||||
fs.readFileSync(path.resolve(__dirname, "index.js"), "utf8"),
|
||||
})
|
||||
);
|
||||
} else {
|
||||
// we regenerate the html page only in development mode to ensure it is up-to-date when the code is hot-reloaded.
|
||||
app.get("/api/docs", async (_, response) => {
|
||||
// #swagger.ignore = true
|
||||
const swaggerDocument = require("./openapi.json");
|
||||
return response.send(
|
||||
swaggerUi.generateHTML(swaggerDocument, {
|
||||
...options,
|
||||
customJsStr:
|
||||
'window.SWAGGER_DOCS_ENV = "development";\n\n' +
|
||||
fs.readFileSync(path.resolve(__dirname, "index.js"), "utf8"),
|
||||
})
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { faviconUrl, useSwagger }
|
||||
module.exports = { faviconUrl, useSwagger };
|
||||
|
||||
Reference in New Issue
Block a user