mirror of
https://github.com/openclaw/clawhub.git
synced 2026-07-22 03:35:29 -04:00
feat: add mdnew skill for clean markdown fetching
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
---
|
||||
name: mdnew
|
||||
description: Fetch clean, agent-optimized Markdown from any URL using the markdown.new service. Use when web_fetch or browser fails to provide clean content, or when you need a token-efficient version of a web page for deep analysis.
|
||||
---
|
||||
|
||||
# mdnew
|
||||
|
||||
Fetch clean Markdown from any URL using the `markdown.new` three-tier conversion pipeline (Header Negotiation -> Workers AI -> Browser Rendering).
|
||||
|
||||
## Usage
|
||||
|
||||
Run the script with a target URL:
|
||||
|
||||
```bash
|
||||
python3 scripts/mdnew.py <url>
|
||||
```
|
||||
|
||||
## Why use mdnew?
|
||||
|
||||
1. **Token Efficiency**: Reduces content size by up to 80% compared to raw HTML.
|
||||
2. **Clean Data**: Strips boilerplate, ads, and nav menus, leaving only core content.
|
||||
3. **JS Execution**: Automatically handles JS-heavy pages via Cloudflare Browser Rendering fallback.
|
||||
4. **Agent-First**: Includes `x-markdown-tokens` tracking to help manage context windows.
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
|
||||
def fetch_markdown(url):
|
||||
# Construct the markdown.new URL
|
||||
md_new_url = f"https://markdown.new/{url}"
|
||||
|
||||
# Request headers to match the tool's optimized behavior
|
||||
headers = {
|
||||
"Accept": "text/markdown, text/html",
|
||||
"User-Agent": "OpenClaw-MDNew-Skill/1.0"
|
||||
}
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(md_new_url, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
content = response.read().decode('utf-8')
|
||||
# Extract custom headers if needed (e.g., token count)
|
||||
tokens = response.getheader('x-markdown-tokens')
|
||||
return content, tokens
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}", None
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: mdnew <url>")
|
||||
sys.exit(1)
|
||||
|
||||
target_url = sys.argv[1]
|
||||
markdown, token_count = fetch_markdown(target_url)
|
||||
|
||||
if token_count:
|
||||
print(f"--- Estimated Tokens: {token_count} ---")
|
||||
print(markdown)
|
||||
Reference in New Issue
Block a user