VectorAdmin v0.0.1-beta

2026-07-18 10:07:07 -04:00 · 2023-07-25 13:56:36 -07:00
commit 03b3066c7b
242 changed files with 20174 additions and 0 deletions
@@ -0,0 +1,6 @@
+hotdir/*
+hotdir/failed/*
+hotdir/processed/*
+!hotdir/failed
+!hotdir/processed
+!hotdir/__HOTDIR__.md
@@ -0,0 +1,4 @@
+
+### Running the document processing API locally
+From the `collector` directory with the `v-env` active run `flask run --host '0.0.0.0' --port 8888`.
+Now uploads from the frontend will be processed as if you ran the `watch.py` script manually.
@@ -0,0 +1,21 @@
+from flask import Flask, json, request
+from scripts.extract_text import extract_text
+from scripts.filetypes import ACCEPTED_MIMES
+api = Flask(__name__)
+
+WATCH_DIRECTORY = "hotdir"
+@api.route('/process', methods=['POST'])
+def prepare_for_embed():
+  content = request.json
+  target_filename = content.get('filename')
+  print(f"Processing {target_filename}")
+  success, reason, metadata = extract_text(WATCH_DIRECTORY, target_filename)
+  return json.dumps({'filename': target_filename, 'success': success, 'reason': reason, 'metadata': metadata})
+
+@api.route('/accepts', methods=['GET'])
+def get_accepted_filetypes():
+  return json.dumps(ACCEPTED_MIMES)
+
+@api.route('/', methods=['GET'])
+def root():
+  return "<p>Use POST /process with filename key in JSON body in order to process a file. File by that name must exist in hotdir already.</p>"
@@ -0,0 +1 @@
+placeholder
@@ -0,0 +1,112 @@
+about-time==4.2.1
+aiohttp==3.8.4
+aiosignal==1.3.1
+alive-progress==3.1.2
+anyio==3.7.0
+appdirs==1.4.4
+argilla==1.8.0
+async-timeout==4.0.2
+attrs==23.1.0
+backoff==2.2.1
+beautifulsoup4==4.12.2
+blinker==1.6.2
+bs4==0.0.1
+certifi==2023.5.7
+cffi==1.15.1
+chardet==5.1.0
+charset-normalizer==3.1.0
+click==8.1.3
+commonmark==0.9.1
+cryptography==41.0.1
+cssselect==1.2.0
+dataclasses-json==0.5.7
+Deprecated==1.2.14
+docx2txt==0.8
+et-xmlfile==1.1.0
+exceptiongroup==1.1.1
+fake-useragent==1.1.3
+Flask==2.3.2
+frozenlist==1.3.3
+grapheme==0.6.0
+greenlet==2.0.2
+gunicorn==20.1.0
+h11==0.14.0
+httpcore==0.16.3
+httpx==0.23.3
+idna==3.4
+importlib-metadata==6.6.0
+importlib-resources==5.12.0
+inquirerpy==0.3.4
+install==1.3.5
+itsdangerous==2.1.2
+Jinja2==3.1.2
+joblib==1.2.0
+langchain==0.0.189
+lxml==4.9.2
+Markdown==3.4.3
+MarkupSafe==2.1.3
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+monotonic==1.6
+msg-parser==1.2.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+nltk==3.8.1
+numexpr==2.8.4
+numpy==1.23.5
+olefile==0.46
+openapi-schema-pydantic==1.2.4
+openpyxl==3.1.2
+packaging==23.1
+pandas==1.5.3
+parse==1.19.0
+pdfminer.six==20221105
+pfzy==0.3.4
+Pillow==9.5.0
+prompt-toolkit==3.0.38
+pycparser==2.21
+pydantic==1.10.8
+pyee==8.2.2
+Pygments==2.15.1
+pypandoc==1.4
+pypdf==3.9.0
+pyppeteer==1.0.2
+pyquery==2.0.0
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==0.21.1
+python-magic==0.4.27
+python-pptx==0.6.21
+python-slugify==8.0.1
+pytz==2023.3
+PyYAML==6.0
+regex==2023.5.5
+requests==2.31.0
+requests-html==0.10.0
+rfc3986==1.5.0
+rich==13.0.1
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+SQLAlchemy==2.0.15
+tabulate==0.9.0
+tenacity==8.2.2
+text-unidecode==1.3
+tiktoken==0.4.0
+tqdm==4.65.0
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.6.3
+unstructured==0.7.1
+urllib3==1.26.16
+uuid==1.30
+w3lib==2.1.1
+wcwidth==0.2.6
+websockets==10.4
+Werkzeug==2.3.6
+wrapt==1.14.1
+xlrd==2.0.1
+XlsxWriter==3.1.2
+yarl==1.9.2
+youtube-transcript-api==0.6.0
+zipp==3.15.0
@@ -0,0 +1,35 @@
+import os
+from .filetypes import FILETYPES
+from .utils import move_source
+
+RESERVED = ['__HOTDIR__.md']
+
+# This script will do a one-off processing of a specific document that exists in hotdir.
+# For this function we remove the original source document since there is no need to keep it and it will
+# only occupy additional disk space.
+def extract_text(directory, target_doc):
+  if os.path.isdir(f"{directory}/{target_doc}") or target_doc in RESERVED: return (False, "Not a file", [])
+  
+  if os.path.exists(f"{directory}/{target_doc}") is False: 
+    print(f"{directory}/{target_doc} does not exist.")
+    return (False, f"{directory}/{target_doc} does not exist.", [])
+
+  filename, fileext = os.path.splitext(target_doc)
+  if filename in ['.DS_Store'] or fileext == '': return False
+  if fileext == '.lock':
+    print(f"{filename} is locked - skipping until unlocked")
+    return (False, f"{filename} is locked - skipping until unlocked", [])
+
+  if fileext not in FILETYPES.keys():
+    print(f"{fileext} not a supported file type for conversion. It will not be processed.")
+    move_source(new_destination_filename=target_doc, failed=True, remove=True)
+    return (False, f"{fileext} not a supported file type for conversion. It will not be processed.", [])
+
+  metadata = FILETYPES[fileext](
+    directory=directory,
+    filename=filename,
+    ext=fileext,
+    remove_on_complete=True # remove source document to save disk space.
+  )
+
+  return (True, None, metadata)
@@ -0,0 +1,22 @@
+from .parsers.as_text import as_text
+from .parsers.as_markdown import as_markdown
+from .parsers.as_pdf import as_pdf
+from .parsers.as_docx import as_docx, as_odt
+from .parsers.as_mbox import as_mbox
+
+FILETYPES = {
+    '.txt': as_text,
+    '.md': as_markdown,
+    '.pdf': as_pdf,
+    '.docx': as_docx,
+    '.odt': as_odt,
+    '.mbox': as_mbox, 
+}
+
+ACCEPTED_MIMES = {
+    'text/plain': ['.txt', '.md'],
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
+    'application/vnd.oasis.opendocument.text': ['.odt'],
+    'application/pdf': ['.pdf'],
+    'application/mbox': ['.mbox'], 
+}
@@ -0,0 +1,58 @@
+import os
+from langchain.document_loaders import Docx2txtLoader, UnstructuredODTLoader
+from ..utils import guid, file_creation_time, move_source, tokenize
+
+# Process all text-related documents.
+def as_docx(**kwargs):
+  parent_dir = kwargs.get('directory', 'hotdir')
+  filename = kwargs.get('filename')
+  ext = kwargs.get('ext', '.txt')
+  remove = kwargs.get('remove_on_complete', False)
+  fullpath = f"{parent_dir}/{filename}{ext}"
+
+  loader = Docx2txtLoader(fullpath)
+  data = loader.load()[0]
+  content = data.page_content
+
+  print(f"-- Working {fullpath} --")
+  data = {
+    'id': guid(), 
+    'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
+    'title': f"{filename}{ext}",
+    'description': "a custom file uploaded by the user.",
+    'published': file_creation_time(fullpath),
+    'wordCount': len(content),
+    'pageContent': content,
+    'token_count_estimate': len(tokenize(content))
+  }
+  
+  move_source(parent_dir, f"{filename}{ext}", remove=remove)
+  print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
+  return [data]
+
+def as_odt(**kwargs):
+  parent_dir = kwargs.get('directory', 'hotdir')
+  filename = kwargs.get('filename')
+  ext = kwargs.get('ext', '.txt')
+  remove = kwargs.get('remove_on_complete', False)
+  fullpath = f"{parent_dir}/{filename}{ext}"
+
+  loader = UnstructuredODTLoader(fullpath)
+  data = loader.load()[0]
+  content = data.page_content
+
+  print(f"-- Working {fullpath} --")
+  data = {
+    'id': guid(), 
+    'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
+    'title': f"{filename}{ext}",
+    'description': "a custom file uploaded by the user.",
+    'published': file_creation_time(fullpath),
+    'wordCount': len(content),
+    'pageContent': content,
+    'token_count_estimate': len(tokenize(content))
+  }
+  
+  move_source(parent_dir, f"{filename}{ext}", remove=remove)
+  print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
+  return [data]
@@ -0,0 +1,31 @@
+import os
+from langchain.document_loaders import UnstructuredMarkdownLoader
+from ..utils import guid, file_creation_time, move_source, tokenize
+
+# Process all text-related documents.
+def as_markdown(**kwargs):
+  parent_dir = kwargs.get('directory', 'hotdir')
+  filename = kwargs.get('filename')
+  ext = kwargs.get('ext', '.txt')
+  remove = kwargs.get('remove_on_complete', False)
+  fullpath = f"{parent_dir}/{filename}{ext}"
+
+  loader = UnstructuredMarkdownLoader(fullpath)
+  data = loader.load()[0]
+  content = data.page_content
+
+  print(f"-- Working {fullpath} --")
+  data = {
+    'id': guid(), 
+    'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
+    'title': f"{filename}{ext}",
+    'description': "a custom file uploaded by the user.",
+    'published': file_creation_time(fullpath),
+    'wordCount': len(content),
+    'pageContent': content,
+    'token_count_estimate': len(tokenize(content))
+  }
+  
+  move_source(parent_dir, f"{filename}{ext}", remove=remove)
+  print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
+  return [data]
@@ -0,0 +1,58 @@
+import os
+import datetime  
+import email.utils
+from mailbox import mbox
+from slugify import slugify
+from ..utils import guid, file_creation_time, move_source, tokenize
+from bs4 import BeautifulSoup
+
+# Process all mbox-related documents.
+def as_mbox(**kwargs):
+    parent_dir = kwargs.get('directory', 'hotdir')
+    filename = kwargs.get('filename')
+    ext = kwargs.get('ext', '.mbox')
+    remove = kwargs.get('remove_on_complete', False)
+    fullpath = f"{parent_dir}/{filename}{ext}"
+
+    print(f"-- Working {fullpath} --")
+    box = mbox(fullpath)
+
+    metadata = []
+    for message in box:
+        content = ""
+        if message.is_multipart():
+            for part in message.get_payload():
+                if part.get_content_type() == 'text/plain':
+                    content = part.get_payload()
+                elif part.get_content_type() == 'text/html':
+                    soup = BeautifulSoup(part.get_payload(), 'html.parser')
+                    content = soup.get_text()
+        else:
+            content = message.get_payload()
+
+        date_tuple = email.utils.parsedate_tz(message['Date'])
+        if date_tuple:
+            local_date = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple))
+            date_sent = local_date.strftime("%a, %d %b %Y %H:%M:%S")
+        else:
+            date_sent = None
+            
+        data = {
+            'id': guid(), 
+            'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{slugify(filename)}-{guid()}{ext}"),
+            'title': f"{filename}{ext}",
+            'description': "a custom file uploaded by the user.",
+            'published': file_creation_time(fullpath),
+            'sender': message['From'],
+            'recipient': message['To'],
+            'subject': message['Subject'],
+            'date_sent': date_sent,
+            'wordCount': len(content),
+            'pageContent': content,
+            'token_count_estimate': len(tokenize(content))
+        }
+        metadata.append(data)
+
+    move_source(parent_dir, f"{filename}{ext}", remove=remove)
+    print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
+    return metadata
@@ -0,0 +1,38 @@
+import os
+from langchain.document_loaders import PyPDFLoader
+from slugify import slugify
+from ..utils import guid, file_creation_time, write_to_server_documents, move_source, tokenize
+
+# Process all text-related documents.
+def as_pdf(**kwargs):
+  parent_dir = kwargs.get('directory', 'hotdir')
+  filename = kwargs.get('filename')
+  ext = kwargs.get('ext', '.txt')
+  remove = kwargs.get('remove_on_complete', False)
+  fullpath = f"{parent_dir}/{filename}{ext}"
+
+  loader = PyPDFLoader(fullpath)
+  pages = loader.load_and_split()
+
+  print(f"-- Working {fullpath} --")
+  metadata = []
+  for page in pages:
+    pg_num = page.metadata.get('page')
+    print(f"-- Working page {pg_num} --")
+
+    content = page.page_content
+    data = {
+      'id': guid(), 
+      'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
+      'title': f"{filename}_pg{pg_num}{ext}",
+      'description': "a custom file uploaded by the user.",
+      'published': file_creation_time(fullpath),
+      'wordCount': len(content),
+      'pageContent': content,
+      'token_count_estimate': len(tokenize(content))
+    }
+    metadata.append(data)
+
+  move_source(parent_dir, f"{filename}{ext}", remove=remove)
+  print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
+  return metadata
@@ -0,0 +1,28 @@
+import os
+from slugify import slugify
+from ..utils import tokenize, guid, file_creation_time, move_source
+
+# Process all text-related documents.
+def as_text(**kwargs):
+  parent_dir = kwargs.get('directory', 'hotdir')
+  filename = kwargs.get('filename')
+  ext = kwargs.get('ext', '.txt')
+  remove = kwargs.get('remove_on_complete', False)
+  fullpath = f"{parent_dir}/{filename}{ext}"
+  content = open(fullpath).read()
+
+  print(f"-- Working {fullpath} --")
+  data = {
+    'id': guid(), 
+    'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
+    'title': f"{filename}{ext}",
+    'description': "a custom file uploaded by the user.",
+    'published': file_creation_time(fullpath),
+    'wordCount': len(content),
+    'pageContent': content,
+    'token_count_estimate': len(tokenize(content))
+  }
+  
+  move_source(parent_dir, f"{filename}{ext}", remove=remove)
+  print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
+  return [data]
@@ -0,0 +1,44 @@
+import os, json, tiktoken
+from datetime import datetime
+from uuid import uuid4
+
+def guid():
+  return str(uuid4())
+
+def file_creation_time(path_to_file):
+  try:
+    if os.name == 'nt':
+      return datetime.fromtimestamp(os.path.getctime(path_to_file)).strftime('%Y-%m-%d %H:%M:%S')
+    else:
+      stat = os.stat(path_to_file)
+      return datetime.fromtimestamp(stat.st_birthtime).strftime('%Y-%m-%d %H:%M:%S')
+  except AttributeError:
+    return datetime.today().strftime('%Y-%m-%d %H:%M:%S')
+
+def move_source(working_dir='hotdir', new_destination_filename='', failed=False, remove=False):
+  if remove and os.path.exists(f"{working_dir}/{new_destination_filename}"):
+    print(f"{new_destination_filename} deleted from filesystem")
+    os.remove(f"{working_dir}/{new_destination_filename}")
+    return
+
+  destination = f"{working_dir}/processed" if not failed else f"{working_dir}/failed"
+  if os.path.exists(destination) == False:
+    os.mkdir(destination)
+  
+  os.replace(f"{working_dir}/{new_destination_filename}", f"{destination}/{new_destination_filename}")
+  return
+
+def write_to_server_documents(data, filename):
+  destination = f"../server/storage/documents/custom-documents"
+  if os.path.exists(destination) == False: os.makedirs(destination)
+  with open(f"{destination}/{filename}.json", 'w', encoding='utf-8') as file:
+    json.dump(data, file, ensure_ascii=True, indent=4)
+
+def tokenize(fullText):
+  encoder = tiktoken.encoding_for_model("text-embedding-ada-002")
+  return encoder.encode(fullText)
+
+def ada_v2_cost(tokenCount):
+  rate_per = 0.0004 / 1_000 # $0.0004 / 1K tokens
+  total = tokenCount * rate_per
+  return '${:,.2f}'.format(total) if total >= 0.01 else '< $0.01'
@@ -0,0 +1,4 @@
+from api import api
+
+if __name__ == '__main__':
+  api.run(debug=False)