mirror of
https://github.com/Significant-Gravitas/AutoGPT-Code-Ability.git
synced 2026-11-09 23:51:28 -10:00
* Route schema changes * Delete ruff.config.json * save current changes * style fixes by ruff * Add class parsing capability * Fix bugs * Address comments * Fix benchmarks * Fix benchmarks * Fix benchmarks * Add comments * fixing types * Add error handling and status messages for application creation * Refactor fetch_deliverable function and update application creation status * Refactor benchmark output formatting * Remove unnecessary print statement in fetch_deliverable function * Propagate DB schema changes: Function <-> CompiledRouteSpec (#65) Propagate DB schema changes: Function <-> CompiledRouteSpec (#65) --------- Co-authored-by: majdyz <[email protected]> Co-authored-by: Swifty <[email protected]> Co-authored-by: majdyz <[email protected]> * emoved broken hardcoded requirements * Add benchmarking functionality * Update branch restrictions in GitHub workflows * Add pytest-cov to dependencies * Update ruff formatter command to include --check option * Update workflow names * Refactor code formatting and fix indentation issues * Update Python dependency caching and installation process * Add Prisma client generation step to CI workflow * Refactor test_process_object_type to use async/await * Add pytest and ValidationError imports, update arg_types assertion, and add test for ValidationError * Remove unused imports and commented code * Add logger to OpenAIChatClient class * Add async context manager for database connection and disconnection * Add blank line for readability * Refactor function_def.arg_types assignment in ast_test.py * mark integration tests * Update test workflow name * Add OpenAI API key configuration * Better Types for Requirements Request/Response (#76) Better Types for Requirements Request/Response --------- Co-authored-by: majdyz <[email protected]> * [AGPT-474] End-to-end Benchmark (#77) * Remove unnecessary fetch_deliverable calls and update invoice_generator_requirements * Fix typo in hardcoded.py * Add missing app IDs and interview IDs for calendar booking, inventory management, and invoicing systems * Add options to benchmark and example commands * Refactor benchmarking code and add new functions * Add task description method to ExampleTask enum * Added end-to-end running * Add hardcoded flag to filter examples in benchmark.py * Refactor run_benchmark_example function to handle exceptions * Fix error handling in specification routes * Fix Function generation bug on nested object type (#79) Co-authored-by: majdyz <[email protected]> * Speeding up requirements system --------- Co-authored-by: Zamil Majdy <[email protected]> Co-authored-by: majdyz <[email protected]> * Refactor code: Remove unused imports and clean up formatting * [AGPT-478] Add Support for Composite Type with multiple Pydantic/nont-primitive classes (#78) * [AGPT-478] Add support for multiple composite Pydantic objects * Ruff Ruff! * Fix failing test * formatting * fix populate-db command * fix run all benchmarks --------- Co-authored-by: majdyz <[email protected]> Co-authored-by: Swifty <[email protected]> * Refactor package handling in packager.py (#81) * [Fix] User provided app name being replaced (#82) * Only make databases when required (#83) * feat: make the name match * feat: database becomes optional * Update agent.py * Integrate related types into the requirements system (#84) * feat: make the name match * feat: database becomes optional * feat: initial model changes * feat: add prompts * feat: more prompts * fix: nits * fix: wrong type 😢 * feat: improved prompts * Cleanup for field type (#85) Co-authored-by: majdyz <[email protected]> * [AGPT-489] Improve type composite-type matching & extracting algorithm (#86) Co-authored-by: majdyz <[email protected]> * [AGPT-489] Add Autogeneration of Typing import (#87) Co-authored-by: majdyz <[email protected]> * [AGPT-489] Fix broken class compiling logic (#88) Co-authored-by: majdyz <[email protected]> * [AGPT-489] Fix function fetching to always include RelatedTypes (#89) [AGPT-489] Fix function fetching to always include RelatedTypes --------- Co-authored-by: majdyz <[email protected]> * fix lock file * feat: endpoint updates (#95) * feat: endpoint updates * fix: ambiguous param * [AGPT-486] Fix codegen generate code with missing pydantic object (#92) Co-authored-by: majdyz <[email protected]> * Fix retry prompt (#96) Co-authored-by: majdyz <[email protected]> * Allow benchmark on $PORT (#98) * Ntindle/agpt 506 add a bit of debug tooling for requirements gen (#100) * [AGPT-485] Fix missing pydantic object during requirement-step; leading to LLM to guess wrong signature (#97) [AGPT-485] Fix missing pydantic object during requirement-step; leading to LLM to guess wrong signature --------- Co-authored-by: majdyz <[email protected]> * [AGPT-157] Add Database Integration Generation (#90) * Added prisma client to server code * Add DatabaseSchema to INCLUDE_FUNC * Update functions to be async and add database schema documentation * Add DatabaseSchema and DatabaseTables to ApiRouteSpec * Updated dev prompts for db * Update profile management descriptions * Update route function definition to be async * Remove logging of database schema * Add Prisma schema file generation * Refactor benchmark and agent modules * Refactor code and comment out debug statements * formatting * Add available_functions to develop_route * Update prisma import and method calls * loading all related tables * Refactor create_prisma_scheama_file function to handle cases with no database schema * Fix async function call in packager.py * Fix database table query in packager.py * Fix database access and unnecessary imports * review comments * [AGPT-508] Enum support for Codex - Generation Logic Layer (#102) Co-authored-by: majdyz <[email protected]> * Add Enum Generation to Requriements (#101) * Added prisma client to server code * Add DatabaseSchema to INCLUDE_FUNC * Update functions to be async and add database schema documentation * Add DatabaseSchema and DatabaseTables to ApiRouteSpec * Updated dev prompts for db * Update profile management descriptions * Update route function definition to be async * Remove logging of database schema * Add Prisma schema file generation * Refactor benchmark and agent modules * Refactor code and comment out debug statements * formatting * Add available_functions to develop_route * Update prisma import and method calls * loading all related tables * Refactor create_prisma_scheama_file function to handle cases with no database schema * Fix async function call in packager.py * Fix database table query in packager.py * Fix database access and unnecessary imports * feat: add enums to the database gen * fix: formatting * Updated to new schema * update hardcoded requirements * fix: objectType generation without type bool * Add database enum import statement * removed unused var as failing lint check --------- Co-authored-by: SwiftyOS <[email protected]> * More Linting Issues Fixes (#99) * fix: more linting issues * Remove error log for async functions * Fix return_type default value in FunctionDef class * Added another validation check --------- Co-authored-by: Swifty <[email protected]> * [AGPT-463] Add code generation validation using Ruff (#103) --------- Co-authored-by: majdyz <[email protected]> * feat: extract external command on in memory code to a function with tests * feat: prisma model parser * feat: database test updates now requires prisma to run * feat: test for prisma accessible * feat: update to use format and validate Also saves the formatted and validated models in place * fix: lint * fix: remove unused variable * feat: allow multiple generators * .gitignore and .env.example (#107) * feat: .env.example * feat: gitignore * fix: replace all bad chars with NOTHING also fix the format string * Add Database Docker Compose File (#108) * feat: .env.example * feat: gitignore * fix: replace all bad chars with NOTHING also fix the format string * feat: add docker compose with only db --------- Co-authored-by: Swifty <[email protected]> * feat: remove vector requirement (#106) * Fix lifecycle and prisma models and enums missing (#105) * fix: add prisma model imports fro root level * fix: reorder start code and add lifecycle * Validate Database Generation (#104) * feat: extract external command on in memory code to a function with tests * feat: prisma model parser * feat: database test updates now requires prisma to run * feat: test for prisma accessible * feat: update to use format and validate Also saves the formatted and validated models in place * fix: lint * fix: remove unused variable * feat: allow multiple generators * formatting * Change logger.info to logger.debug in exec_external_on_contents function * Update error logging in exec_external_on_contents function * Update response model and route annotation in compile.py * Force * Ruff * Fix Api route spec * formatting * Dont generate envexample passwords with symbols in them (#117) * fix: don't use special chars in db password * fix: linting * [AGPT-540] Add missing initialization of available object types (#113) * [AGPT-540] Add missing initialization of available object types * [AGPT-540] Add missing initialization of available object types * [AGPT-540] Add missing initialization of available object types * Ruff * allow non-annotated type checking * Include all available function for ruff * Fix tests * Make DB work * Make stubbing work * Make papa-lint happy * Ruffs * Add prisma model validation layer * Improve prisma querying by avoiding conflict * Move on during failure formatting --------- Co-authored-by: majdyz <[email protected]> * main into dev (#118) * Update setup guide. * [AGPT-544] Vague 500 errors in interview stage (#115) * Add error handling for user not found in database during interview step. * Add error handling for app not found in database during interview step. * Add error handling for interview not found in database during interview step. * Add error handling for nonexistent questions in interview step. * Add error handling for non-"ask" tool in user answers in interview step. * Clarify comment describing interview next_step. --------- Co-authored-by: Toran Bruce Richards <[email protected]> * [AGPT-511] Resume running from any point (#114) * Refactor populate_db function and add get_resume_points command * Fix app ID reference in benchmark.py * Add ResumePoint model * Added runner funcitons to running codex from the cli * Add application description field to ApplicationBase model and update create_app method in CodexClient * Refactor code to run tasks concurrently * Add include parameter to find_many() method in get_resume_points() * Add disconnect calls to asyncio event loop * Update resume_point name to display only the first 30 characters * Remove commented out code and unused imports * Remove unused code * formatting * Fix validation error handling in develop.py * formatting * fixes * Add validation error for unimplemented main function * Validate and try-fix generated endpoint db and model usage (#116) * feat: log status * feat: trivial implementation of copy works for request -> response and response-> request for unfilled object but not list[unfilled_object] * fix: i cant rememember * feat: more complex * feat: mostly working * fix: fixes from tests * feat: tests * fix: linting problems and test name * feat: add more supported types * fix: remove unused data * fix: stop using print * fix: arrays of arrays * fix: lint * fix: reduce mess * feat: more complex type that I hope we never actually use * fix: lint --------- Co-authored-by: majdyz <[email protected]> Co-authored-by: majdyz <[email protected]> Co-authored-by: Zamil Majdy <[email protected]> Co-authored-by: Nicholas Tindle <[email protected]> Co-authored-by: Nicholas Tindle <[email protected]> Co-authored-by: Toran Bruce Richards <[email protected]>
195 lines
7.1 KiB
Python
195 lines
7.1 KiB
Python
import re
|
|
from typing import Dict, List, Optional, Union
|
|
|
|
from pydantic import BaseModel, validator
|
|
|
|
|
|
class FieldInfo(BaseModel):
|
|
type: str
|
|
attributes: List[str] = []
|
|
relation: Optional[str] = None
|
|
|
|
|
|
class ModelInfo(BaseModel):
|
|
name: str
|
|
fields: Dict[str, FieldInfo]
|
|
definition: str
|
|
|
|
|
|
class EnumInfo(BaseModel):
|
|
name: str
|
|
values: List[str]
|
|
definition: str
|
|
|
|
|
|
class DatasourceInfo(BaseModel):
|
|
provider: str
|
|
name: str
|
|
url: str
|
|
extensions: Optional[List[str]] = None
|
|
|
|
@validator("provider")
|
|
def validate_provider(cls, v):
|
|
if v not in ["postgresql", "mysql", "sqlite"]:
|
|
raise ValueError(f"Invalid datasource provider: {v}")
|
|
return v
|
|
|
|
|
|
class GeneratorInfo(BaseModel):
|
|
provider: str
|
|
name: str
|
|
config: Dict[str, Union[str, List[str]]]
|
|
definition: str
|
|
|
|
@validator("provider")
|
|
def validate_provider(cls, v):
|
|
if v not in ["prisma-client-py", "prisma-docs", "prisma-dbml-generator"]:
|
|
raise ValueError(f"Invalid generator provider: {v}")
|
|
return v
|
|
|
|
|
|
class SchemaInfo(BaseModel):
|
|
datasource: Optional[DatasourceInfo] = None
|
|
generators: list[GeneratorInfo] = []
|
|
enums: Dict[str, EnumInfo] = {}
|
|
models: Dict[str, ModelInfo] = {}
|
|
|
|
|
|
def parse_prisma_schema(schema_text: str) -> SchemaInfo:
|
|
# # Regular expressions to match different parts of the schema
|
|
datasource_pattern = re.compile(r"datasource\s+(\w+)\s*{\s*([\s\S]*?)\s*}")
|
|
generator_pattern = re.compile(r"generator\s+(\w+)\s*{\s*([\s\S]*?)\s*}")
|
|
enum_pattern = re.compile(r"enum\s+(\w+)\s*{\s*([\s\S]*?)\s*}")
|
|
model_pattern = re.compile(r"model\s+(\w+)\s*{\s*([\s\S]*?)\s*}")
|
|
field_pattern = re.compile(r"(\w+)\s+([\w\[\]?]+)\s*(.*)")
|
|
attribute_pattern = re.compile(r"@\w+(\((?:[^()]*|\([^()]*\))*\))?")
|
|
relation_pattern = re.compile(r"@relation\((.*?)\)")
|
|
|
|
# Parse datasource
|
|
datasource = None
|
|
datasource_match = datasource_pattern.search(schema_text)
|
|
if datasource_match:
|
|
datasource_name = datasource_match.group(1)
|
|
datasource_fields = datasource_match.group(2).strip().split("\n")
|
|
datasource_info = {}
|
|
for field in datasource_fields:
|
|
field = field.strip()
|
|
if field:
|
|
key, value = field.split("=", 1)
|
|
key = key.strip()
|
|
value = value.strip().strip('"').strip("'")
|
|
if key == "extensions":
|
|
value = [
|
|
ext.strip().strip('"').strip("'")
|
|
for ext in value[1:-1].split(",")
|
|
]
|
|
datasource_info[key] = value
|
|
datasource = DatasourceInfo(name=datasource_name, **datasource_info)
|
|
|
|
# Parse generators
|
|
generators = []
|
|
for generator_match in generator_pattern.finditer(schema_text):
|
|
generator_name = generator_match.group(1)
|
|
generator_definition = generator_match.group(0)
|
|
generator_fields = generator_match.group(2).strip().split("\n")
|
|
generator_config = {}
|
|
generator_provider = None
|
|
for field in generator_fields:
|
|
field = field.strip()
|
|
if field:
|
|
key, value = field.split("=", 1)
|
|
key = key.strip()
|
|
value = value.strip()
|
|
if key == "provider":
|
|
generator_provider = value.strip('"')
|
|
else:
|
|
if value.startswith("[") and value.endswith("]"):
|
|
value = [
|
|
item.strip().strip('"') for item in value[1:-1].split(",")
|
|
]
|
|
else:
|
|
value = value.strip('"')
|
|
generator_config[key] = value
|
|
if generator_provider:
|
|
generator = GeneratorInfo(
|
|
name=generator_name,
|
|
provider=generator_provider,
|
|
config=generator_config,
|
|
definition=generator_definition,
|
|
)
|
|
generators.append(generator)
|
|
|
|
# Parse enums
|
|
enums = {}
|
|
for enum_match in enum_pattern.finditer(schema_text):
|
|
enum_name = enum_match.group(1)
|
|
enum_values = [value.strip() for value in enum_match.group(2).split()]
|
|
enum_definition = enum_match.group(0)
|
|
enums[enum_name] = EnumInfo(
|
|
name=enum_name, values=enum_values, definition=enum_definition
|
|
)
|
|
|
|
# Parse models
|
|
models = {}
|
|
for model_match in model_pattern.finditer(schema_text):
|
|
model_name = model_match.group(1)
|
|
model_definition = model_match.group(0)
|
|
model_fields = {}
|
|
fields = model_match.group(2).strip().split("\n")
|
|
for field in fields:
|
|
field = field.strip()
|
|
if field:
|
|
field_match = field_pattern.match(field)
|
|
if field_match:
|
|
field_name = field_match.group(1)
|
|
field_type = field_match.group(2)
|
|
field_attributes = []
|
|
field_relation = None
|
|
for attribute_match in attribute_pattern.finditer(
|
|
field_match.group(3)
|
|
):
|
|
attribute = attribute_match.group()
|
|
if attribute.startswith("@relation"):
|
|
relation_match = relation_pattern.search(attribute)
|
|
if relation_match:
|
|
field_relation = f"@relation({relation_match.group(1)})"
|
|
else:
|
|
field_attributes.append(attribute)
|
|
model_fields[field_name] = FieldInfo(
|
|
type=field_type,
|
|
attributes=field_attributes,
|
|
relation=field_relation,
|
|
)
|
|
models[model_name] = ModelInfo(
|
|
name=model_name, fields=model_fields, definition=model_definition
|
|
)
|
|
|
|
return SchemaInfo(
|
|
datasource=datasource, generators=generators, enums=enums, models=models
|
|
)
|
|
|
|
|
|
def print_parsed_schema(parsed_schema: SchemaInfo):
|
|
if parsed_schema.datasource:
|
|
print("Datasource:")
|
|
print(f" Provider: {parsed_schema.datasource.provider}")
|
|
print(f" URL: {parsed_schema.datasource.url}")
|
|
print(f" Extensions: {parsed_schema.datasource.extensions}")
|
|
if len(parsed_schema.generators) > 0:
|
|
for generator in parsed_schema.generators:
|
|
print(f"Generator {generator.name}:")
|
|
print(f" Provider: {generator.provider}")
|
|
print(f" Config: {generator.config}")
|
|
for enum_name, enum_info in parsed_schema.enums.items():
|
|
print(f"Enum: {enum_name}")
|
|
print(f" Definition: {enum_info.definition}")
|
|
print(f" Values: {enum_info.values}")
|
|
for model_name, model_info in parsed_schema.models.items():
|
|
print(f"Model: {model_name}")
|
|
print(f" Definition: {model_info.definition}")
|
|
for field_name, field_data in model_info.fields.items():
|
|
print(f" Field: {field_name}")
|
|
print(f" Type: {field_data.type}")
|
|
print(f" Attributes: {field_data.attributes}")
|
|
print(f" Relation: {field_data.relation}")
|