mirror of
https://github.com/MervinPraison/PraisonAI-Tools.git
synced 2026-11-13 05:01:37 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a79978d3b7 | ||
|
|
fb45d41170 | ||
|
|
191a4a6641 | ||
|
|
3060daa92c | ||
|
|
bcaa5a5219 | ||
|
|
b96bfbc7e2 | ||
|
|
f146c6c7ad | ||
|
|
82fc2031f4 | ||
|
|
ca809f7f4d | ||
|
|
a5d92e8cb4 | ||
|
|
ca4267be06 | ||
|
|
4b03908c11 |
No files matched your search
@@ -29,4 +29,53 @@ try {
|
||||
|
||||
assert('PACKAGE_PATHS includes pyproject', rg.PACKAGE_PATHS.includes('pyproject.toml'));
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
(async () => {
|
||||
const noonUtc = new Date('2026-07-08T12:00:00Z');
|
||||
const mockGithub = {
|
||||
rest: {
|
||||
actions: {
|
||||
listWorkflowRuns: async () => ({
|
||||
data: {
|
||||
workflow_runs: [
|
||||
{
|
||||
conclusion: 'success',
|
||||
created_at: '2026-07-08T09:00:00Z',
|
||||
status: 'completed',
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
assert('PATCH_RELEASE_INTERVAL_DAYS is 3', rg.PATCH_RELEASE_INTERVAL_DAYS === 3);
|
||||
assert(
|
||||
'hasSuccessfulReleaseWithinDays blocks recent release',
|
||||
await rg.hasSuccessfulReleaseWithinDays(mockGithub, 'o', 'r', noonUtc)
|
||||
);
|
||||
|
||||
const oldReleaseGithub = {
|
||||
rest: {
|
||||
actions: {
|
||||
listWorkflowRuns: async () => ({
|
||||
data: {
|
||||
workflow_runs: [
|
||||
{
|
||||
conclusion: 'success',
|
||||
created_at: '2026-07-04T09:00:00Z',
|
||||
status: 'completed',
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
assert(
|
||||
'hasSuccessfulReleaseWithinDays allows after 3 days',
|
||||
!(await rg.hasSuccessfulReleaseWithinDays(oldReleaseGithub, 'o', 'r', noonUtc))
|
||||
);
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
})();
|
||||
@@ -6,6 +6,8 @@ const https = require('https');
|
||||
const config = require('./gate-config.js');
|
||||
|
||||
const PACKAGE_PATHS = config.packagePaths;
|
||||
/** Minimum days between successful patch auto-releases. */
|
||||
const PATCH_RELEASE_INTERVAL_DAYS = 3;
|
||||
const ACTIVE_RELEASE_STATUSES = new Set([
|
||||
'queued', 'in_progress', 'waiting', 'pending', 'requested',
|
||||
]);
|
||||
@@ -61,8 +63,18 @@ function utcDayStart(now = new Date()) {
|
||||
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
|
||||
}
|
||||
|
||||
async function hasSuccessfulReleaseToday(github, owner, repo, now = new Date()) {
|
||||
const dayStart = utcDayStart(now);
|
||||
function releaseIntervalStart(now = new Date(), intervalDays = PATCH_RELEASE_INTERVAL_DAYS) {
|
||||
return new Date(now.getTime() - intervalDays * 24 * 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
async function hasSuccessfulReleaseWithinDays(
|
||||
github,
|
||||
owner,
|
||||
repo,
|
||||
now = new Date(),
|
||||
intervalDays = PATCH_RELEASE_INTERVAL_DAYS,
|
||||
) {
|
||||
const windowStart = releaseIntervalStart(now, intervalDays);
|
||||
const runs = await github.rest.actions.listWorkflowRuns({
|
||||
owner,
|
||||
repo,
|
||||
@@ -71,10 +83,15 @@ async function hasSuccessfulReleaseToday(github, owner, repo, now = new Date())
|
||||
per_page: 30,
|
||||
});
|
||||
return runs.data.workflow_runs.some(
|
||||
(r) => r.conclusion === 'success' && new Date(r.created_at) >= dayStart
|
||||
(r) => r.conclusion === 'success' && new Date(r.created_at) >= windowStart
|
||||
);
|
||||
}
|
||||
|
||||
/** @deprecated Use hasSuccessfulReleaseWithinDays — kept for selftests. */
|
||||
async function hasSuccessfulReleaseToday(github, owner, repo, now = new Date()) {
|
||||
return hasSuccessfulReleaseWithinDays(github, owner, repo, now, 1);
|
||||
}
|
||||
|
||||
async function lastGreenCiSha(github, owner, repo) {
|
||||
const runs = await github.rest.actions.listWorkflowRuns({
|
||||
owner,
|
||||
@@ -116,9 +133,11 @@ async function evaluateReleasePreflight(github, owner, repo, options, core) {
|
||||
}
|
||||
|
||||
const referenceTime = options.now instanceof Date ? options.now : new Date();
|
||||
if (await hasSuccessfulReleaseToday(github, owner, repo, referenceTime)) {
|
||||
const day = referenceTime.toISOString().slice(0, 10);
|
||||
reasons.push(`already released today (UTC ${day}); max one patch release per day`);
|
||||
if (await hasSuccessfulReleaseWithinDays(github, owner, repo, referenceTime)) {
|
||||
reasons.push(
|
||||
`already released within last ${PATCH_RELEASE_INTERVAL_DAYS} days; `
|
||||
+ `max one patch release every ${PATCH_RELEASE_INTERVAL_DAYS} days`
|
||||
);
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -196,6 +215,9 @@ module.exports = {
|
||||
bumpPatch,
|
||||
readVersionsFromTree,
|
||||
pypiVersionExists,
|
||||
PATCH_RELEASE_INTERVAL_DAYS,
|
||||
releaseIntervalStart,
|
||||
hasSuccessfulReleaseWithinDays,
|
||||
hasSuccessfulReleaseToday,
|
||||
utcDayStart,
|
||||
evaluateReleasePreflight,
|
||||
|
||||
@@ -2,11 +2,12 @@ name: Nightly Release Gate
|
||||
|
||||
# Preflight before PyPI Release (patch bump).
|
||||
# Primary: CI success on main (workflow_run).
|
||||
# Backstop: nightly cron at 00:00 UTC.
|
||||
# Backstop: cron every 3 days at 00:00 UTC.
|
||||
# Dedupe: max one successful patch release every 3 days (see release-gate.js).
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
- cron: '0 0 */3 * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
|
||||
@@ -133,7 +133,9 @@ jobs:
|
||||
- name: Publish to PyPI
|
||||
if: inputs.dry_run != true
|
||||
env:
|
||||
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN || secrets.PYPI_API_TOKEN }}
|
||||
# PyPI API token auth: username __token__, password = pypi-… token (see pypi.org/help/#invalid-auth)
|
||||
UV_PUBLISH_USERNAME: __token__
|
||||
UV_PUBLISH_PASSWORD: ${{ secrets.PYPI_TOKEN || secrets.PYPI_API_TOKEN }}
|
||||
run: |
|
||||
uv publish
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ def __getattr__(name):
|
||||
# Firecrawl
|
||||
"FirecrawlTool": "firecrawl_tool",
|
||||
"firecrawl_scrape": "firecrawl_tool",
|
||||
"firecrawl_search": "firecrawl_tool",
|
||||
# Serper
|
||||
"SerperTool": "serper_tool",
|
||||
"serper_search": "serper_tool",
|
||||
@@ -685,6 +686,7 @@ __all__ = [
|
||||
# Firecrawl Tool
|
||||
"FirecrawlTool",
|
||||
"firecrawl_scrape",
|
||||
"firecrawl_search",
|
||||
# Serper Tool
|
||||
"SerperTool",
|
||||
"serper_search",
|
||||
|
||||
@@ -25,7 +25,7 @@ class FirecrawlTool(BaseTool):
|
||||
"""Tool for web scraping using Firecrawl."""
|
||||
|
||||
name = "firecrawl"
|
||||
description = "Scrape and crawl websites using Firecrawl."
|
||||
description = "Scrape, crawl, and search the web using Firecrawl."
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
self.api_key = api_key or os.getenv("FIRECRAWL_API_KEY")
|
||||
@@ -38,10 +38,12 @@ class FirecrawlTool(BaseTool):
|
||||
if not self.api_key:
|
||||
raise ValueError("FIRECRAWL_API_KEY is required")
|
||||
try:
|
||||
from firecrawl import FirecrawlApp
|
||||
from firecrawl import Firecrawl
|
||||
except ImportError:
|
||||
raise ImportError("firecrawl-py not installed. Install with: pip install firecrawl-py")
|
||||
self._client = FirecrawlApp(api_key=self.api_key)
|
||||
raise ImportError(
|
||||
"firecrawl-py>=4 not installed. Install with: pip install 'firecrawl-py>=4'"
|
||||
)
|
||||
self._client = Firecrawl(api_key=self.api_key)
|
||||
return self._client
|
||||
|
||||
def run(
|
||||
@@ -56,6 +58,8 @@ class FirecrawlTool(BaseTool):
|
||||
return self.scrape(url=url, **kwargs)
|
||||
elif action == "crawl":
|
||||
return self.crawl(url=url, **kwargs)
|
||||
elif action == "search":
|
||||
return self.search(query=kwargs.get("query") or url, limit=kwargs.get("limit", 5))
|
||||
else:
|
||||
return {"error": f"Unknown action: {action}"}
|
||||
|
||||
@@ -68,16 +72,28 @@ class FirecrawlTool(BaseTool):
|
||||
return {"error": "FIRECRAWL_API_KEY not configured"}
|
||||
|
||||
try:
|
||||
params = {}
|
||||
# firecrawl-py v2 (>=4): kwargs instead of a params dict; returns a
|
||||
# typed Document, not a dict.
|
||||
kwargs: Dict[str, Any] = {}
|
||||
if formats:
|
||||
params["formats"] = formats
|
||||
|
||||
result = self.client.scrape_url(url, params=params if params else None)
|
||||
|
||||
kwargs["formats"] = formats
|
||||
|
||||
result = self.client.scrape(url, **kwargs)
|
||||
|
||||
# v2 returns a Document object; metadata is a typed DocumentMetadata.
|
||||
# Coerce to a plain dict to keep this tool's return shape stable.
|
||||
metadata = getattr(result, "metadata", None)
|
||||
if metadata is not None and hasattr(metadata, "model_dump"):
|
||||
metadata = metadata.model_dump(exclude_none=True)
|
||||
elif metadata is None:
|
||||
metadata = {}
|
||||
|
||||
markdown = getattr(result, "markdown", None) or ""
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"markdown": result.get("markdown", "")[:5000],
|
||||
"metadata": result.get("metadata", {}),
|
||||
"markdown": markdown[:5000],
|
||||
"metadata": metadata,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Firecrawl scrape error: {e}")
|
||||
@@ -92,25 +108,77 @@ class FirecrawlTool(BaseTool):
|
||||
return [{"error": "FIRECRAWL_API_KEY not configured"}]
|
||||
|
||||
try:
|
||||
result = self.client.crawl_url(
|
||||
# `limit` may arrive as a string (e.g. "10") or None when invoked by
|
||||
# an LLM/agent; coerce to int so the SDK call and `data[:limit]` slice
|
||||
# below don't raise TypeError.
|
||||
try:
|
||||
limit = int(limit) if limit is not None else 10
|
||||
except (ValueError, TypeError):
|
||||
limit = 10
|
||||
|
||||
# firecrawl-py v2 (>=4): kwargs instead of a params dict; returns a
|
||||
# typed CrawlJob whose `.data` is a list of Document objects.
|
||||
result = self.client.crawl(
|
||||
url,
|
||||
params={"limit": limit},
|
||||
limit=limit,
|
||||
poll_interval=5,
|
||||
)
|
||||
|
||||
|
||||
data = getattr(result, "data", None) or []
|
||||
|
||||
pages = []
|
||||
for page in result.get("data", [])[:limit]:
|
||||
for page in data[:limit]:
|
||||
metadata = getattr(page, "metadata", None)
|
||||
pages.append({
|
||||
"url": page.get("metadata", {}).get("sourceURL"),
|
||||
"title": page.get("metadata", {}).get("title"),
|
||||
"markdown": page.get("markdown", "")[:2000],
|
||||
"url": getattr(metadata, "source_url", None),
|
||||
"title": getattr(metadata, "title", None),
|
||||
"markdown": (getattr(page, "markdown", None) or "")[:2000],
|
||||
})
|
||||
return pages
|
||||
except Exception as e:
|
||||
logger.error(f"Firecrawl crawl error: {e}")
|
||||
return [{"error": str(e)}]
|
||||
|
||||
def search(self, query: str, limit: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search the web and return results."""
|
||||
if not query:
|
||||
return [{"error": "query is required"}]
|
||||
|
||||
if not self.api_key:
|
||||
return [{"error": "FIRECRAWL_API_KEY not configured"}]
|
||||
|
||||
try:
|
||||
# `limit` may arrive as a string or None from an LLM/agent; coerce to
|
||||
# int so the SDK call and `web[:limit]` slice below don't raise.
|
||||
try:
|
||||
limit = int(limit) if limit is not None else 5
|
||||
except (ValueError, TypeError):
|
||||
limit = 5
|
||||
|
||||
# firecrawl-py v2 returns a typed SearchData object whose results are
|
||||
# grouped by source; read the web results and coerce to plain dicts.
|
||||
result = self.client.search(query, limit=limit)
|
||||
|
||||
web = getattr(result, "web", None) or []
|
||||
results = []
|
||||
for item in web[:limit]:
|
||||
results.append({
|
||||
"url": getattr(item, "url", None),
|
||||
"title": getattr(item, "title", None),
|
||||
"description": getattr(item, "description", None),
|
||||
"markdown": (getattr(item, "markdown", None) or "")[:2000],
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.error(f"Firecrawl search error: {e}")
|
||||
return [{"error": str(e)}]
|
||||
|
||||
|
||||
def firecrawl_scrape(url: str) -> Dict[str, Any]:
|
||||
"""Scrape URL with Firecrawl."""
|
||||
return FirecrawlTool().scrape(url=url)
|
||||
|
||||
|
||||
def firecrawl_search(query: str, limit: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search the web with Firecrawl."""
|
||||
return FirecrawlTool().search(query=query, limit=limit)
|
||||
+10
-2
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "praisonai-tools"
|
||||
version = "0.2.32"
|
||||
version = "0.2.34"
|
||||
description = "Extended tools for PraisonAI Agents"
|
||||
authors = [
|
||||
{name = "Mervin Praison"}
|
||||
@@ -32,6 +32,8 @@ praisonai-tools = "praisonai_tools.cli:main"
|
||||
dev = [
|
||||
"pytest>=8.0.0",
|
||||
"pytest-asyncio>=0.23.0",
|
||||
"playwright>=1.40",
|
||||
"imageio-ffmpeg>=0.5",
|
||||
]
|
||||
wordpress = [
|
||||
"praisonaiwp[ai]>=0.1.0",
|
||||
@@ -83,4 +85,10 @@ packages = ["praisonai_tools"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_mode = "auto"
|
||||
|
||||
[tool.ruff.lint]
|
||||
# Pin the rule set to the historical default (Pyflakes + a subset of
|
||||
# pycodestyle) so an unpinned ruff upgrade does not spuriously fail CI by
|
||||
# enabling thousands of new default rules across the pre-existing codebase.
|
||||
select = ["E4", "E7", "E9", "F"]
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Pytest configuration for video/motion-graphics tests.
|
||||
|
||||
Most tests in this tree are fully mocked (protocols, dataclasses, mocked
|
||||
backends) and run without the optional ``video-motion`` dependencies. Only a
|
||||
subset actually constructs :class:`HtmlRenderBackend`, which hard-requires
|
||||
``playwright`` and ``imageio-ffmpeg`` at construction time.
|
||||
|
||||
Rather than skipping the whole tree when those packages are absent (for example
|
||||
a minimal ``pip install -e ".[dev]"`` where the wheels could not be resolved),
|
||||
gate *only* the dependency-bound tests. Mark them with
|
||||
``@pytest.mark.requires_video_deps`` and this hook skips just those when the
|
||||
extras are missing, preserving coverage for the dependency-independent tests.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
|
||||
_VIDEO_DEPS = ("playwright", "imageio_ffmpeg")
|
||||
_MISSING_VIDEO_DEPS = [
|
||||
name for name in _VIDEO_DEPS if importlib.util.find_spec(name) is None
|
||||
]
|
||||
VIDEO_DEPS_AVAILABLE = not _MISSING_VIDEO_DEPS
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"requires_video_deps: mark test as requiring the optional "
|
||||
"video-motion extras (playwright, imageio-ffmpeg).",
|
||||
)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
if VIDEO_DEPS_AVAILABLE:
|
||||
return
|
||||
skip_reason = (
|
||||
"video-motion extra not installed "
|
||||
"(pip install -e '.[dev,video-motion]'); missing: "
|
||||
+ ", ".join(_MISSING_VIDEO_DEPS)
|
||||
)
|
||||
skip_marker = pytest.mark.skip(reason=skip_reason)
|
||||
for item in items:
|
||||
if "requires_video_deps" in item.keywords:
|
||||
item.add_marker(skip_marker)
|
||||
@@ -13,6 +13,7 @@ except ImportError:
|
||||
|
||||
|
||||
@pytest.mark.skipif(not BACKEND_AVAILABLE, reason="Motion graphics backend not available")
|
||||
@pytest.mark.requires_video_deps
|
||||
class TestBackendSafety:
|
||||
"""Test workspace safety features."""
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from praisonai_tools.video.motion_graphics.protocols import RenderOpts
|
||||
class TestHtmlRenderBackend:
|
||||
"""Test HTML render backend."""
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
def test_import_error_handling(self):
|
||||
"""Test that import errors are handled properly."""
|
||||
with patch('praisonai_tools.video.motion_graphics.backend_html.async_playwright', None):
|
||||
|
||||
@@ -96,6 +96,7 @@ class TestRenderTools:
|
||||
class TestResolveBackend:
|
||||
"""Test backend resolution."""
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
def test_resolve_string_backend(self):
|
||||
"""Test resolving string backend specification."""
|
||||
backend = _resolve_backend("html")
|
||||
@@ -119,6 +120,7 @@ class TestResolveBackend:
|
||||
class TestCreateMotionGraphicsAgent:
|
||||
"""Test motion graphics agent factory."""
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_create_agent_defaults(self):
|
||||
@@ -137,6 +139,7 @@ class TestCreateMotionGraphicsAgent:
|
||||
"lint_composition", "render_composition"} <= tool_names
|
||||
assert "motion graphics specialist" in agent.instructions.lower()
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_create_agent_custom_params(self):
|
||||
@@ -159,6 +162,7 @@ class TestCreateMotionGraphicsAgent:
|
||||
with pytest.raises(ImportError, match="praisonaiagents not available"):
|
||||
create_motion_graphics_agent()
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_create_agent_auto_workspace(self):
|
||||
@@ -183,6 +187,7 @@ class TestCreateMotionGraphicsAgent:
|
||||
|
||||
assert agent._motion_graphics_backend is mock_backend
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_create_agent_workspace_creation(self):
|
||||
@@ -195,6 +200,7 @@ class TestCreateMotionGraphicsAgent:
|
||||
assert workspace_path.exists()
|
||||
assert agent._motion_graphics_workspace == workspace_path
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_create_agent_skill_included(self):
|
||||
@@ -207,6 +213,7 @@ class TestCreateMotionGraphicsAgent:
|
||||
assert "timeline" in agent.instructions
|
||||
assert "window.__timelines" in agent.instructions
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_create_agent_output_validation(self):
|
||||
@@ -219,6 +226,7 @@ class TestCreateMotionGraphicsAgent:
|
||||
assert "Never fabricate file paths" in agent.instructions
|
||||
assert "3 failed attempts" in agent.instructions
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.Agent', MockAgent)
|
||||
@patch('praisonai_tools.video.motion_graphics.agent.FileTools', MockFileTools)
|
||||
def test_agent_tools_configuration(self):
|
||||
|
||||
@@ -112,6 +112,7 @@ class TestRenderResult:
|
||||
class TestRenderBackendProtocol:
|
||||
"""Test RenderBackendProtocol protocol."""
|
||||
|
||||
@pytest.mark.requires_video_deps
|
||||
def test_protocol_runtime_checkable(self):
|
||||
"""Test that protocol is runtime checkable."""
|
||||
from praisonai_tools.video.motion_graphics.backend_html import HtmlRenderBackend
|
||||
|
||||
Reference in new issue
Block a user