mirror of
https://github.com/langgenius/dify-plugin-sdks.git
synced 2026-07-22 10:25:23 -04:00
Harry
116 lines
3.5 KiB
YAML
116 lines
3.5 KiB
YAML
description:
|
|
en_US: Recursively search through a urls subdomains, and gather the content.
|
|
zh_Hans: 递归爬取一个网址的子域名，并收集内容。
|
|
identity:
|
|
author: langgenius
|
|
label:
|
|
en_US: Firecrawl
|
|
zh_Hans: Firecrawl
|
|
name: crawl
|
|
parameters:
|
|
- form: llm
|
|
description:
|
|
en_US: The base URL to start crawling from.
|
|
zh_Hans: 要爬取网站的起始URL。
|
|
label:
|
|
en_US: Start URL
|
|
zh_Hans: 起始URL
|
|
name: url
|
|
required: true
|
|
type: string
|
|
- form: form
|
|
description:
|
|
en_US: is crawl subpages
|
|
zh_Hans: 是否爬取子页面
|
|
label:
|
|
zh_Hans: 爬取子页面
|
|
en_US: Crawl Subpages
|
|
default: true
|
|
name: crawl_subpages
|
|
placeholder:
|
|
zh_Hans: 是否爬取子页面
|
|
en_US: is crawl subpages
|
|
type: boolean
|
|
- form: form
|
|
description:
|
|
en_US: 'Pages matching these patterns will be skipped. Example: blog/*, about/*'
|
|
zh_Hans: 匹配这些模式的页面将被跳过。示例：blog/*, about/*
|
|
label:
|
|
en_US: URL patterns to exclude
|
|
zh_Hans: 排除路径
|
|
name: exclude_paths
|
|
placeholder:
|
|
en_US: Use commas to separate multiple tags
|
|
zh_Hans: 多个标签时使用半角逗号分隔
|
|
type: string
|
|
- form: form
|
|
description:
|
|
en_US: 'Only pages matching these patterns will be crawled. Example: blog/*, about/*'
|
|
zh_Hans: 只有与这些模式匹配的页面才会被爬取。示例：blog/*, about/*
|
|
label:
|
|
en_US: URL patterns to include
|
|
zh_Hans: 仅包含路径
|
|
name: include_paths
|
|
placeholder:
|
|
en_US: Use commas to separate multiple tags
|
|
zh_Hans: 多个标签时使用半角逗号分隔
|
|
required: false
|
|
type: string
|
|
- form: form
|
|
description:
|
|
en_US: Maximum depth to crawl relative to the entered URL. A maxDepth of 0 scrapes
|
|
only the entered URL. A maxDepth of 1 scrapes the entered URL and all pages
|
|
one level deep. A maxDepth of 2 scrapes the entered URL and all pages up to
|
|
two levels deep. Higher values follow the same pattern.
|
|
zh_Hans: 相对于输入的URL，爬取的最大深度。maxDepth为0时，仅抓取输入的URL。maxDepth为1时，抓取输入的URL以及所有一级深层页面。maxDepth为2时，抓取输入的URL以及所有两级深层页面。更高值遵循相同模式。
|
|
label:
|
|
en_US: Maximum crawl depth
|
|
zh_Hans: 最大深度
|
|
min: 0
|
|
name: max_depth
|
|
type: number
|
|
default: 2
|
|
- form: form
|
|
description:
|
|
en_US: Specify the maximum number of pages to crawl. The crawler will stop after
|
|
reaching this limit.
|
|
zh_Hans: 指定要爬取的最大页面数。爬虫将在达到此限制后停止。
|
|
label:
|
|
en_US: Maximum pages to crawl
|
|
zh_Hans: 限制数量
|
|
min: 1
|
|
name: limit
|
|
required: false
|
|
type: number
|
|
default: 10
|
|
- form: form
|
|
description:
|
|
en_US: Only return the main content of the page excluding headers, navs, footers,
|
|
etc.
|
|
zh_Hans: 只返回页面的主要内容，不包括头部、导航栏、尾部等。
|
|
label:
|
|
en_US: only Main Content
|
|
zh_Hans: 仅提取主要内容(无标题、导航、页脚等)
|
|
name: only_main_content
|
|
type: boolean
|
|
default: false
|
|
|
|
output_schema:
|
|
type: object
|
|
properties:
|
|
source_url:
|
|
type: string
|
|
description: the source url of the website
|
|
content:
|
|
type: string
|
|
description: the content from the website
|
|
title:
|
|
type: string
|
|
description: the title of the website
|
|
"description":
|
|
type: string
|
|
description: the description of the website
|
|
provider_type: website_crawl
|
|
extra:
|
|
python:
|
|
source: datasources/crawl.py |