mirror of
https://github.com/Mintplex-Labs/vector-admin.git
synced 2026-07-18 18:14:31 -04:00
timothycarambat
44 lines
1.7 KiB
Python
44 lines
1.7 KiB
Python
import os, json, tiktoken
|
|
from datetime import datetime
|
|
from uuid import uuid4
|
|
|
|
def guid():
|
|
return str(uuid4())
|
|
|
|
def file_creation_time(path_to_file):
|
|
try:
|
|
if os.name == 'nt':
|
|
return datetime.fromtimestamp(os.path.getctime(path_to_file)).strftime('%Y-%m-%d %H:%M:%S')
|
|
else:
|
|
stat = os.stat(path_to_file)
|
|
return datetime.fromtimestamp(stat.st_birthtime).strftime('%Y-%m-%d %H:%M:%S')
|
|
except AttributeError:
|
|
return datetime.today().strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
def move_source(working_dir='hotdir', new_destination_filename='', failed=False, remove=False):
|
|
if remove and os.path.exists(f"{working_dir}/{new_destination_filename}"):
|
|
print(f"{new_destination_filename} deleted from filesystem")
|
|
os.remove(f"{working_dir}/{new_destination_filename}")
|
|
return
|
|
|
|
destination = f"{working_dir}/processed" if not failed else f"{working_dir}/failed"
|
|
if os.path.exists(destination) == False:
|
|
os.mkdir(destination)
|
|
|
|
os.replace(f"{working_dir}/{new_destination_filename}", f"{destination}/{new_destination_filename}")
|
|
return
|
|
|
|
def write_to_server_documents(data, filename):
|
|
destination = f"../server/storage/documents/custom-documents"
|
|
if os.path.exists(destination) == False: os.makedirs(destination)
|
|
with open(f"{destination}/{filename}.json", 'w', encoding='utf-8') as file:
|
|
json.dump(data, file, ensure_ascii=True, indent=4)
|
|
|
|
def tokenize(fullText):
|
|
encoder = tiktoken.encoding_for_model("text-embedding-ada-002")
|
|
return encoder.encode(fullText)
|
|
|
|
def ada_v2_cost(tokenCount):
|
|
rate_per = 0.0004 / 1_000 # $0.0004 / 1K tokens
|
|
total = tokenCount * rate_per
|
|
return '${:,.2f}'.format(total) if total >= 0.01 else '< $0.01' |