Merge pull request #114 from MervinPraison/develop

Changing the default DB location

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.11-slim
 WORKDIR /app
 COPY . .
-RUN pip install flask praisonai==0.0.55 gunicorn markdown
+RUN pip install flask praisonai==0.0.57 gunicorn markdown
 EXPOSE 8080
 CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]

docs/api/praisonai/deploy.html

@@ -110,7 +110,7 @@ Loads environment variables from .env file or system and sets them.</p>
             file.write(&#34;FROM python:3.11-slim\n&#34;)
             file.write(&#34;WORKDIR /app\n&#34;)
             file.write(&#34;COPY . .\n&#34;)
-            file.write(&#34;RUN pip install flask praisonai==0.0.55 gunicorn markdown\n&#34;)
+            file.write(&#34;RUN pip install flask praisonai==0.0.57 gunicorn markdown\n&#34;)
             file.write(&#34;EXPOSE 8080\n&#34;)
             file.write(&#39;CMD [&#34;gunicorn&#34;, &#34;-b&#34;, &#34;0.0.0.0:8080&#34;, &#34;api:app&#34;]\n&#39;)
             

docs/api/praisonai/ui/context.html

@@ -68,10 +68,12 @@ hljs.highlightAll();
         self.max_file_size = max_file_size
         self.max_tokens = int(os.getenv("PRAISONAI_MAX_TOKENS", max_tokens))
         self.ignore_patterns = self.get_ignore_patterns()
+        self.include_paths = self.get_include_paths()
+        self.included_files = []
 
     def get_ignore_patterns(self):
         """
-        Loads ignore patterns from various sources, prioritizing them in 
+        Loads ignore patterns from various sources, prioritizing them in
         the following order:
             1. .praisonignore
             2. settings.yaml (under code.ignore_files)
@@ -129,6 +131,19 @@ hljs.highlightAll();
         logger.debug(f"Final ignore patterns: {modified_ignore_patterns}")
         return modified_ignore_patterns
 
+    def get_include_paths(self):
+        include_paths = []
+
+        # 1. Load from .praisoninclude
+        include_file = os.path.join(self.directory, '.praisoninclude')
+        if os.path.exists(include_file):
+            with open(include_file, 'r') as f:
+                include_paths.extend(
+                    line.strip() for line in f
+                    if line.strip() and not line.startswith('#')
+                )
+        return include_paths
+
     def should_ignore(self, file_path):
         """
         Check if a file or directory should be ignored based on patterns.
@@ -150,31 +165,65 @@ hljs.highlightAll();
                any(file_path.endswith(ext) for ext in self.relevant_extensions)
 
     def gather_context(self):
-        """Gather context from relevant files, respecting ignore patterns."""
+        """Gather context from relevant files, respecting ignore patterns and include paths."""
         context = []
         total_files = 0
         processed_files = 0
 
-        for root, dirs, files in os.walk(self.directory):
-            total_files += len(files) 
-            dirs[:] = [d for d in dirs if not self.should_ignore(os.path.join(root, d))]
-            for file in files:
-                file_path = os.path.join(root, file)
-                if not self.should_ignore(file_path) and self.is_relevant_file(file_path):
+        if not self.include_paths:
+            # No include paths specified, process the entire directory
+            for root, dirs, files in os.walk(self.directory):
+                total_files += len(files)
+                dirs[:] = [d for d in dirs if not self.should_ignore(os.path.join(root, d))]
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if not self.should_ignore(file_path) and self.is_relevant_file(file_path):
+                        try:
+                            with open(file_path, 'r', encoding='utf-8') as f:
+                                content = f.read()
+                                context.append(f"File: {file_path}\n\n{content}\n\n{'='*50}\n")
+                                self.included_files.append(Path(file_path).relative_to(self.directory))
+                        except Exception as e:
+                            logger.error(f"Error reading {file_path}: {e}")
+                    processed_files += 1
+                    print(f"\rProcessed {processed_files}/{total_files} files", end="", flush=True)
+        else:
+            # Process specified include paths
+            for include_path in self.include_paths:
+                full_path = os.path.join(self.directory, include_path)
+                if os.path.isdir(full_path):
+                    for root, dirs, files in os.walk(full_path):
+                        total_files += len(files)
+                        dirs[:] = [d for d in dirs if not self.should_ignore(os.path.join(root, d))]
+                        for file in files:
+                            file_path = os.path.join(root, file)
+                            if not self.should_ignore(file_path) and self.is_relevant_file(file_path):
+                                try:
+                                    with open(file_path, 'r', encoding='utf-8') as f:
+                                        content = f.read()
+                                        context.append(f"File: {file_path}\n\n{content}\n\n{'='*50}\n")
+                                        self.included_files.append(Path(file_path).relative_to(self.directory))
+                                except Exception as e:
+                                    logger.error(f"Error reading {file_path}: {e}")
+                            processed_files += 1
+                            print(f"\rProcessed {processed_files}/{total_files} files", end="", flush=True)
+                elif os.path.isfile(full_path) and self.is_relevant_file(full_path):
                     try:
-                        with open(file_path, 'r', encoding='utf-8') as f:
+                        with open(full_path, 'r', encoding='utf-8') as f:
                             content = f.read()
-                            context.append(f"File: {file_path}\n\n{content}\n\n{'='*50}\n")
+                            context.append(f"File: {full_path}\n\n{content}\n\n{'='*50}\n")
+                            self.included_files.append(Path(full_path).relative_to(self.directory))
                     except Exception as e:
-                        logger.error(f"Error reading {file_path}: {e}")
-                processed_files += 1
-                print(f"\rProcessed {processed_files}/{total_files} files", end="", flush=True)
+                        logger.error(f"Error reading {full_path}: {e}")
+                    processed_files += 1
+                    print(f"\rProcessed {processed_files}/{total_files} files", end="", flush=True)
+
         print()  # New line after progress indicator
         return '\n'.join(context)
 
     def count_tokens(self, text):
         """Count tokens using a simple whitespace-based tokenizer."""
-        return len(text.split()) 
+        return len(text.split())
 
     def truncate_context(self, context):
         """Truncate context to stay within the token limit."""
@@ -199,12 +248,9 @@ hljs.highlightAll();
             contents = sorted(path.iterdir())
             pointers = [('└── ' if i == len(contents) - 1 else '├── ') for i in range(len(contents))]
             for pointer, item in zip(pointers, contents):
-                # Use should_ignore for consistency
-                if self.should_ignore(item):
-                    continue
-
                 rel_path = item.relative_to(start_dir)
-                tree.append(f"{prefix}{pointer}{rel_path}")
+                if rel_path in self.included_files:
+                    tree.append(f"{prefix}{pointer}{rel_path}")
 
                 if item.is_dir():
                     add_to_tree(item, prefix + ('    ' if pointer == '└── ' else '│   '))
@@ -236,7 +282,7 @@ hljs.highlightAll();
 <span>def <span class="ident">gather_context</span></span>(<span>self)</span>
 </code></dt>
 <dd>
-<div class="desc"><p>Gather context from relevant files, respecting ignore patterns.</p></div>
+<div class="desc"><p>Gather context from relevant files, respecting ignore patterns and include paths.</p></div>
 </dd>
 <dt id="praisonai.ui.context.ContextGatherer.get_context_tree"><code class="name flex">
 <span>def <span class="ident">get_context_tree</span></span>(<span>self)</span>
@@ -256,6 +302,12 @@ the following order:
 4. .gitignore
 5. Default patterns</p></div>
 </dd>
+<dt id="praisonai.ui.context.ContextGatherer.get_include_paths"><code class="name flex">
+<span>def <span class="ident">get_include_paths</span></span>(<span>self)</span>
+</code></dt>
+<dd>
+<div class="desc"></div>
+</dd>
 <dt id="praisonai.ui.context.ContextGatherer.is_relevant_file"><code class="name flex">
 <span>def <span class="ident">is_relevant_file</span></span>(<span>self, file_path)</span>
 </code></dt>
@@ -316,6 +368,7 @@ Handles both file names and directory names for more comprehensive filtering.</p
 <li><code><a title="praisonai.ui.context.ContextGatherer.gather_context" href="#praisonai.ui.context.ContextGatherer.gather_context">gather_context</a></code></li>
 <li><code><a title="praisonai.ui.context.ContextGatherer.get_context_tree" href="#praisonai.ui.context.ContextGatherer.get_context_tree">get_context_tree</a></code></li>
 <li><code><a title="praisonai.ui.context.ContextGatherer.get_ignore_patterns" href="#praisonai.ui.context.ContextGatherer.get_ignore_patterns">get_ignore_patterns</a></code></li>
+<li><code><a title="praisonai.ui.context.ContextGatherer.get_include_paths" href="#praisonai.ui.context.ContextGatherer.get_include_paths">get_include_paths</a></code></li>
 <li><code><a title="praisonai.ui.context.ContextGatherer.is_relevant_file" href="#praisonai.ui.context.ContextGatherer.is_relevant_file">is_relevant_file</a></code></li>
 <li><code><a title="praisonai.ui.context.ContextGatherer.run" href="#praisonai.ui.context.ContextGatherer.run">run</a></code></li>
 <li><code><a title="praisonai.ui.context.ContextGatherer.save_context" href="#praisonai.ui.context.ContextGatherer.save_context">save_context</a></code></li>

docs/ui/chat.md

@@ -25,4 +25,4 @@ export OPENAI_API_KEY=xxxxxxxx
 praisonai chat
 ```
 
-4. Set Model name to be gpt-3.5-turbo in the settings 
+4. Set Model name to be gpt-4o-mini in the settings 

docs/ui/code.md

@@ -10,6 +10,21 @@ PraisonAI Code helps you to interact with your whole codebase using the power of
 | **Chat** | Chat with 100+ LLMs, single AI Agent | [https://docs.praison.ai/ui/chat](https://docs.praison.ai/ui/chat) |
 | **Code** | Chat with entire Codebase, single AI Agent | [https://docs.praison.ai/ui/code](https://docs.praison.ai/ui/code) |
 
+## Table of Contents
+
+- [Install PraisonAI Code](#install-praisonai-code)
+- [Other Models](#other-models)
+- [To Use Gemini 1.5](#to-use-gemini-15)
+- [Ignore Files](#ignore-files)
+  - [Using .praisonignore](#using-praisonignore)
+  - [Using settings.yaml](#using-settingsyaml)
+  - [Using .env File](#using-env-file)
+  - [Using Environment Variables in the Terminal](#using-environment-variables-in-the-terminal)
+- [Include Files](#include-files)
+- [Set Max Tokens](#set-max-tokens)
+
+## Install PraisonAI Code
+
 1. 
 ```bash
 pip install "praisonai[code]"
@@ -27,12 +42,12 @@ praisonai code
 
 4. Username and Password will be asked for the first time. `admin` is the default username and password.
 
-5. Set Model name to be gpt-3.5-turbo in the settings 
+5. Set Model name to be gpt-4o-mini in the settings 
 
 
 ## Other Models
 
-* Use 100+ LLMs
+* Use 100+ LLMs - [Litellm](https://litellm.vercel.app/docs/providers)
 * Includes Gemini 1.5 for 2 Million Context Length
 
 ## To Use Gemini 1.5
@@ -43,9 +58,29 @@ praisonai code
 
 ## Ignore Files
 
-### Using settings.yaml
+### Using .praisonignore
 
-* Create a settings.yaml file in the root folder of the project
+* Create a `.praisonignore` file in the root folder of the project
+* Add files to ignore
+
+```bash
+.*
+*.pyc
+pycache
+.git
+.gitignore
+.vscode
+.idea
+.DS_Store
+.lock
+.pyc
+.env
+```
+
+### Using settings.yaml 
+(.praisonignore is preferred)
+
+* Create a `settings.yaml` file in the root folder of the project
 * Add below Variables and required Ignore Files
 
 ```yaml
@@ -66,25 +101,39 @@ code:
 
 ### Using .env File
 
-* Create a .env file in the root folder of the project
+* Create a `.env` file in the root folder of the project
 * Add below Variables and required Ignore Files
 
 ```bash
-PRAISONAI_IGNORE_FILES=".*,*.pyc,__pycache__,.git,.gitignore,.vscode,.idea,.DS_Store,*.lock,.env,docs,tests,test,tmp,temp,*.txt,*.md,*.json,*.csv,*.tsv,public,*.sql,*.sqlite,*.db,*.db3,*.sqlite3,*.log,*.zip,*.gz,*.tar,*.rar,*.7z,*.pdf,*.jpg,*.jpeg,*.png,*.gif,*.svg,cookbooks,assets,dist,build,node_modules,venv,crewAI,.cache,*.__pycache__,*chroma.sqlite3,test/,dist/,praisonAI.egg-info,test.yaml,db,praisonai_prompt.txt,watch.sh,docs.sh,other,output,*.chainlit,.files,site,flagged,*public,threads.db,trained_agents_data.pkl,.pytest_cache"
+PRAISONAI_IGNORE_FILES=".*,*.pyc,__pycache__,.git,.gitignore,.vscode"
 ```
 
 ### Using Environment Variables in the Terminal
 
 ```bash
-export PRAISONAI_IGNORE_FILES=".*,*.pyc,__pycache__,.git,.gitignore,.vscode,.idea,.DS_Store,*.lock,.env,docs,tests,test,tmp,temp,*.txt,*.md,*.json,*.csv,*.tsv,public,*.sql,*.sqlite,*.db,*.db3,*.sqlite3,*.log,*.zip,*.gz,*.tar,*.rar,*.7z,*.pdf,*.jpg,*.jpeg,*.png,*.gif,*.svg,cookbooks,assets,dist,build,node_modules,venv,crewAI,.cache,*.__pycache__,*chroma.sqlite3,test/,dist/,praisonAI.egg-info,test.yaml,db,praisonai_prompt.txt,watch.sh,docs.sh,other,output,*.chainlit,.files,site,flagged,*public,threads.db,trained_agents_data.pkl,.pytest_cache"
+export PRAISONAI_IGNORE_FILES=".*,*.pyc,__pycache__,.git,.gitignore,.vscode"
+```
+
+## Include Files
+
+- Add files you wish to Include files in the context
+- This will automatically ignore all the ignore files option
+- This will include only the files/folders mentioned in `.praisoninclude` to the context
+
+* Create a `.praisoninclude` file in the root folder of the project
+* Add files to Include
+
+```bash
+projectfiles
+docs
 ```
 
 ## Set Max Tokens
 
-Note: By Default Max Tokens set is 128,000
+Note: By Default Max Tokens set is 900,000
 
 ```bash
-export PRAISONAI_MAX_TOKENS=200000
+export PRAISONAI_MAX_TOKENS=1000000
 ```
 
 or 
@@ -92,5 +141,5 @@ or
 * Create a .env file in the root folder of the project
 * Add below Variables and required Max Tokens
 * ```
-  PRAISONAI_MAX_TOKENS=200000
+  PRAISONAI_MAX_TOKENS=1000000
   ```

praisonai/deploy.py

@@ -56,7 +56,7 @@ class CloudDeployer:
             file.write("FROM python:3.11-slim\n")
             file.write("WORKDIR /app\n")
             file.write("COPY . .\n")
-            file.write("RUN pip install flask praisonai==0.0.55 gunicorn markdown\n")
+            file.write("RUN pip install flask praisonai==0.0.57 gunicorn markdown\n")
             file.write("EXPOSE 8080\n")
             file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
             

praisonai/test.py

@@ -7,7 +7,7 @@ load_dotenv()
 import autogen
 config_list = [
     {
-        'model': os.environ.get("OPENAI_MODEL_NAME", "gpt-3.5-turbo"),
+        'model': os.environ.get("OPENAI_MODEL_NAME", "gpt-4o-mini"),
         'base_url': os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1"),
         'api_key': os.environ.get("OPENAI_API_KEY")
     }

praisonai/ui/chat.py

@@ -40,9 +40,10 @@ now = utc_now()
 
 create_step_counter = 0
 
-DB_PATH = "threads.db"
+DB_PATH = os.path.expanduser("~/.praison/database.sqlite")
 
 def initialize_db():
+    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
     cursor.execute('''
@@ -179,7 +180,7 @@ async def start():
         cl.user_session.set("model_name", model_name)
     else:
         # If no setting found, use default or environment variable
-        model_name = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
+        model_name = os.getenv("MODEL_NAME", "gpt-4o-mini")
         cl.user_session.set("model_name", model_name)
     logger.debug(f"Model name: {model_name}")
     settings = cl.ChatSettings(
@@ -187,7 +188,7 @@ async def start():
             TextInput(
                 id="model_name",
                 label="Enter the Model Name",
-                placeholder="e.g., gpt-3.5-turbo",
+                placeholder="e.g., gpt-4o-mini",
                 initial=model_name
             )
         ]
@@ -221,7 +222,7 @@ async def setup_agent(settings):
 
 @cl.on_message
 async def main(message: cl.Message):
-    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-3.5-turbo"
+    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
     message_history = cl.user_session.get("message_history", [])
     message_history.append({"role": "user", "content": message.content})
 
@@ -268,14 +269,14 @@ async def send_count():
 @cl.on_chat_resume
 async def on_chat_resume(thread: cl_data.ThreadDict):
     logger.info(f"Resuming chat: {thread['id']}")
-    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-3.5-turbo"
+    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
     logger.debug(f"Model name: {model_name}")
     settings = cl.ChatSettings(
         [
             TextInput(
                 id="model_name",
                 label="Enter the Model Name",
-                placeholder="e.g., gpt-3.5-turbo",
+                placeholder="e.g., gpt-4o-mini",
                 initial=model_name
             )
         ]

praisonai/ui/code.py

@@ -41,9 +41,10 @@ now = utc_now()
 
 create_step_counter = 0
 
-DB_PATH = "threads.db"
+DB_PATH = os.path.expanduser("~/.praison/database.sqlite")
 
 def initialize_db():
+    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
     cursor.execute('''
@@ -180,7 +181,7 @@ async def start():
         cl.user_session.set("model_name", model_name)
     else:
         # If no setting found, use default or environment variable
-        model_name = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
+        model_name = os.getenv("MODEL_NAME", "gpt-4o-mini")
         cl.user_session.set("model_name", model_name)
     logger.debug(f"Model name: {model_name}")
     settings = cl.ChatSettings(
@@ -188,7 +189,7 @@ async def start():
             TextInput(
                 id="model_name",
                 label="Enter the Model Name",
-                placeholder="e.g., gpt-3.5-turbo",
+                placeholder="e.g., gpt-4o-mini",
                 initial=model_name
             )
         ]
@@ -228,7 +229,7 @@ async def setup_agent(settings):
 
 @cl.on_message
 async def main(message: cl.Message):
-    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-3.5-turbo"
+    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
     message_history = cl.user_session.get("message_history", [])
     message_history.append({"role": "user", "content": message.content})
     gatherer = ContextGatherer()
@@ -282,14 +283,14 @@ async def send_count():
 @cl.on_chat_resume
 async def on_chat_resume(thread: cl_data.ThreadDict):
     logger.info(f"Resuming chat: {thread['id']}")
-    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-3.5-turbo"
+    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
     logger.debug(f"Model name: {model_name}")
     settings = cl.ChatSettings(
         [
             TextInput(
                 id="model_name",
                 label="Enter the Model Name",
-                placeholder="e.g., gpt-3.5-turbo",
+                placeholder="e.g., gpt-4o-mini",
                 initial=model_name
             )
         ]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "PraisonAI"
-version = "0.0.55"
+version = "0.0.57"
 description = "PraisonAI application combines AutoGen and CrewAI or similar frameworks into a low-code solution for building and managing multi-agent LLM systems, focusing on simplicity, customization, and efficient human-agent collaboration."
 authors = ["Mervin Praison"]
 license = ""

tests/source/autogen_function_tools.py

@@ -21,7 +21,7 @@ assistant = ConversableAgent(
     system_message="You are a helpful AI assistant. "
     "You can help with simple calculations. "
     "Return 'TERMINATE' when the task is done.",
-    llm_config={"config_list": [{"model": "gpt-3.5-turbo", "api_key": os.environ["OPENAI_API_KEY"]}]},
+    llm_config={"config_list": [{"model": "gpt-4o-mini", "api_key": os.environ["OPENAI_API_KEY"]}]},
 )
 
 user_proxy = ConversableAgent(

tests/source/autogen_langchain_tools.py

@@ -63,7 +63,7 @@ llm_config = {
         generate_llm_config(custom_tool),
         generate_llm_config(read_file_tool),
     ],
-    "config_list": [{"model": "gpt-3.5-turbo", "api_key": os.environ["OPENAI_API_KEY"]}],
+    "config_list": [{"model": "gpt-4o-mini", "api_key": os.environ["OPENAI_API_KEY"]}],
     "timeout": 120,
 }