langgenius/dify-official-plugins
1
0
Fork 0
mirror of https://github.com/langgenius/dify-official-plugins.git synced 2026-07-22 01:55:27 -04:00
Code Issues 54 Packages Projects Releases Wiki Activity

feat: introduce datasource plugins and chunker plugins for RAG pipeline (#1649)

Browse Source 
* feat: Introduce datasource plugins

Co-authored-by: jingfelix <jingfelix@outlook.com>
Co-authored-by: Harry <xh001x@hotmail.com>
Co-authored-by: Frederick2313072 <2031894837@qq.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: citron <45784494+lcandy2@users.noreply.github.com>

* feat: Introduce chunkers for RAG pipelines

Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Dongyu Li <544104925@qq.com>

* feat: Introduce dify_extractor tool

Co-authored-by: jyong <718720800@qq.com>

* fix: README title (#1648)

Signed-off-by: jingfelix <jingfelix@outlook.com>

* add readme and privacy

---------

Signed-off-by: jingfelix <jingfelix@outlook.com>
Co-authored-by: jingfelix <jingfelix@outlook.com>
Co-authored-by: Harry <xh001x@hotmail.com>
Co-authored-by: Frederick2313072 <2031894837@qq.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: citron <45784494+lcandy2@users.noreply.github.com>
Co-authored-by: Dongyu Li <544104925@qq.com>
This commit is contained in:
QuantumGhost
2025-09-04 15:23:13 +08:00
committed by GitHub
parent 5301fe633a
commit 45f3b7c736
304 changed files with 27228 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
datasources/aws_s3_storage/.difyignore
+179
View File
@@ -0,0 +1,179 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
# Git
.git/
.gitignore
.github/
# Mac
.DS_Store
# Windows
Thumbs.db
datasources/aws_s3_storage/PRIVACY.md
+47
View File
@@ -0,0 +1,47 @@
# Privacy Policy
This Privacy Policy explains how we collect, use, and protect your information when you use the AWS S3 Storage Datasource Plugin for Dify.
## Information Collection
We do not collect, store, or share any personal information from users. All data processed by the plugin is handled locally or through secure connections to Amazon Web Services (AWS) S3, as required for functionality.
## Use of Information
The plugin only accesses your AWS S3 buckets and objects to provide the requested features, such as browsing and downloading files from your S3 storage. No data is stored or transmitted to third parties by the plugin itself.
## Data Security
We are committed to ensuring the security of your data. All communications with AWS S3 services use secure protocols (HTTPS) and AWS's secure authentication mechanisms. We do not retain any user credentials or file data beyond the duration of your session.
## Third-Party Services
This plugin interacts with Amazon Web Services S3 via official AWS SDK (boto3). Please refer to AWS's privacy policy for details on how your data is handled by AWS services.
## Data Access Scope
The plugin requires the following permissions to function:
- S3 bucket listing permissions
- S3 object listing permissions within specified buckets
- S3 object download permissions for specified files
These permissions are used solely for the intended functionality and no additional data is accessed.
## Credential Security
Your AWS credentials (Access Key ID, Secret Access Key, and Region) are used only to authenticate with AWS S3 services. These credentials are:
- Never stored permanently by the plugin
- Only used during active sessions
- Transmitted securely using encrypted connections
- Never shared with any third parties
## Changes to This Policy
We may update this Privacy Policy from time to time. Any changes will be posted in this document with an updated effective date.
## Contact
If you have any questions or concerns about this Privacy Policy, please contact the developer [hello@dify.ai](mailto:hello@dify.ai) or refer to the project repository for more information.
Last updated: December 2024
datasources/aws_s3_storage/README.md
+149
View File
@@ -0,0 +1,149 @@
# AWS S3 Storage Datasource Plugin
**Author**: langgenius
**Version**: 0.3.3
**Type**: datasource
## Introduction
This plugin integrates with Amazon Web Services (AWS) S3 Storage, enabling automated access to your S3 buckets and objects. It supports browsing files and folders within S3 buckets and downloading files directly for use in platforms like Dify.
## Features
- **Browse S3 Buckets**: List all available S3 buckets in your AWS account
- **Browse Bucket Contents**: Navigate through folders and files within S3 buckets
- **Download Files**: Retrieve files from S3 buckets for processing
- **Folder Navigation**: Support for hierarchical folder structure browsing
- **Pagination Support**: Efficiently handle large numbers of files with pagination
## Setup
### Prerequisites
Before using this plugin, you need:
1. An AWS account with active S3 service
2. S3 buckets with the files you want to access
3. AWS IAM credentials with appropriate permissions
### Configuration Steps
1. **Create AWS IAM User** (if not already exists):
- Sign in to the [AWS Management Console](https://console.aws.amazon.com/)
- Navigate to IAM (Identity and Access Management)
- Click "Users" → "Add users"
- Provide a user name (e.g., "dify-s3-plugin")
- Select "Programmatic access" for Access type
2. **Set Permissions**:
- Attach policies that grant S3 access:
- `AmazonS3ReadOnlyAccess` (for read-only access to all buckets)
- Or create a custom policy with specific bucket permissions:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetObject",
"s3:ListAllMyBuckets"
],
"Resource": [
"arn:aws:s3:::your-bucket-name",
"arn:aws:s3:::your-bucket-name/*"
]
}
]
}
```
3. **Generate Access Keys**:
- After creating the user, go to the "Security credentials" tab
- Click "Create access key"
- Choose "Application running outside AWS"
- Save the **Access Key ID** and **Secret Access Key** securely
4. **Configure the Plugin in Dify**:
- **Access Key ID**: Enter the Access Key ID from your AWS IAM user
- **Secret Access Key**: Enter the Secret Access Key from your AWS IAM user
- **Region Name**: Enter your AWS region (e.g., `us-east-1`, `eu-west-1`, `ap-southeast-1`)
- Click "Save" to store the credentials
## Usage
### Browsing Files
The plugin provides an online drive interface for browsing S3 content:
1. **List Buckets**: When no bucket is specified, the plugin lists all available S3 buckets
2. **Browse Bucket Contents**: Select a bucket to view its files and folders
3. **Navigate Folders**: Click on folders to browse their contents
4. **Pagination**: Use pagination controls for buckets with many files
### Downloading Files
To download a file:
1. Browse to the desired file
2. Select the file for download
3. The plugin retrieves the file content and metadata
## Supported Operations
| Operation | Description |
|-----------|-------------|
| List Buckets | View all S3 buckets in your AWS account |
| Browse Files | Navigate through files and folders in a bucket |
| Download File | Retrieve file content from S3 |
| Pagination | Handle large file lists with continuation tokens |
## Security Considerations
- **Credentials**: Store your AWS credentials securely and never share them
- **Permissions**: Use the principle of least privilege - grant only necessary S3 permissions
- **Region**: Ensure you're using the correct AWS region for your buckets
- **Encryption**: Consider using S3 server-side encryption for sensitive data
## Troubleshooting
### Common Issues
1. **"Credentials not found" error**:
- Verify that Access Key ID and Secret Access Key are correctly configured
- Check that the IAM user is active
2. **"Access Denied" error**:
- Ensure the IAM user has proper S3 permissions
- Verify the bucket policy allows access from your IAM user
3. **"Bucket not found" error**:
- Check that the bucket name is correct
- Ensure the bucket exists in the specified region
4. **Region connectivity issues**:
- Verify the region name is correct (e.g., `us-east-1`)
- Ensure the bucket exists in the specified region
## AWS Regions
Common AWS regions include:
- `us-east-1` (US East - N. Virginia)
- `us-west-2` (US West - Oregon)
- `eu-west-1` (EU - Ireland)
- `ap-southeast-1` (Asia Pacific - Singapore)
- `ap-northeast-1` (Asia Pacific - Tokyo)
For a complete list, refer to [AWS Regions and Endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html).
## Privacy
Please refer to the [Privacy Policy](PRIVACY.md) for information on how your data is handled when using this plugin.
## Support
For issues or questions:
- Contact: [hello@dify.ai](mailto:hello@dify.ai)
- Repository: Check the project repository for updates and documentation
Last updated: December 2024
datasources/aws_s3_storage/_assets/icon.svg
+20
View File
@@ -0,0 +1,20 @@
<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_16537_32446)">
<g clip-path="url(#clip1_16537_32446)">
<path fill-rule="evenodd" clip-rule="evenodd" d="M0 0H40V40H0V0Z" fill="url(#paint0_linear_16537_32446)"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M30.418 21.4465L30.61 20.0945C32.38 21.1545 32.4035 21.593 32.403 21.6045C32.4 21.6075 32.098 21.8595 30.418 21.4465ZM29.4465 21.1765C26.3865 20.25 22.125 18.295 20.4 17.48C20.4 17.473 20.402 17.4665 20.402 17.4595C20.4016 17.1409 20.2748 16.8354 20.0495 16.6102C19.8241 16.3849 19.5186 16.2583 19.2 16.258C18.538 16.258 17.999 16.797 17.999 17.4595C17.999 18.122 18.538 18.661 19.2 18.661C19.491 18.661 19.755 18.5525 19.9635 18.38C21.9925 19.34 26.221 21.267 29.3035 22.177L28.085 30.78C28.0816 30.8032 28.0799 30.8266 28.08 30.85C28.08 31.608 24.7265 32.9995 19.247 32.9995C13.7095 32.9995 10.3205 31.608 10.3205 30.8505C10.3205 30.8275 10.319 30.805 10.3155 30.7825L7.769 12.179C9.9735 13.6965 14.715 14.499 19.25 14.499C23.778 14.499 28.5115 13.699 30.72 12.1865L29.4465 21.1765ZM7.5 10.239C7.536 9.581 11.317 7 19.25 7C27.182 7 30.9635 9.58 31 10.239V10.4635C30.565 11.9385 25.665 13.5 19.25 13.5C12.824 13.5 7.9215 11.934 7.5 10.4565V10.239ZM32 10.25C32 8.5175 27.033 6 19.25 6C11.467 6 6.5 8.5175 6.5 10.25L6.547 10.627L9.321 30.889C9.3875 33.155 15.43 34 19.247 34C23.983 34 29.0145 32.911 29.0795 30.89L30.2775 22.442C30.944 22.6015 31.4925 22.683 31.9325 22.683C32.5245 22.683 32.9245 22.538 33.167 22.2495C33.2642 22.1369 33.3344 22.0036 33.3722 21.8598C33.41 21.7159 33.4144 21.5653 33.385 21.4195C33.255 20.728 32.434 19.982 30.761 19.0275L31.949 10.6465L32 10.25Z" fill="white"/>
</g>
</g>
<defs>
<linearGradient id="paint0_linear_16537_32446" x1="1.11759e-05" y1="3200" x2="40" y2="5.96978e-06" gradientUnits="userSpaceOnUse">
<stop stop-color="#1B660F"/>
<stop offset="1" stop-color="#6CAE3E"/>
</linearGradient>
<clipPath id="clip0_16537_32446">
<rect width="40" height="40" fill="white"/>
</clipPath>
<clipPath id="clip1_16537_32446">
<rect width="40" height="40" fill="white"/>
</clipPath>
</defs>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.1 KiB

datasources/aws_s3_storage/datasources/aws_s3_storage.py
+82
View File
@@ -0,0 +1,82 @@
import os
from collections.abc import Generator
import boto3
from botocore.client import Config
from dify_plugin.entities.datasource import (
DatasourceMessage,
OnlineDriveBrowseFilesRequest,
OnlineDriveBrowseFilesResponse,
OnlineDriveDownloadFileRequest,
OnlineDriveFile,
OnlineDriveFileBucket,
)
from dify_plugin.interfaces.datasource.online_drive import OnlineDriveDatasource
class AWSS3StorageDataSource(OnlineDriveDatasource):
def _browse_files(
self, request: OnlineDriveBrowseFilesRequest
) -> OnlineDriveBrowseFilesResponse:
credentials = self.runtime.credentials
bucket_name = request.bucket
prefix = request.prefix or ""
max_keys = request.max_keys or 100
next_page_parameters = request.next_page_parameters or {}
print(credentials)
if not credentials:
raise ValueError("Credentials not found")
client = boto3.client(
"s3",
aws_secret_access_key=credentials.get("secret_access_key"),
aws_access_key_id=credentials.get("access_key_id"),
endpoint_url=f"https://s3.{credentials.get('region_name')}.amazonaws.com",
region_name=credentials.get("region_name"),
config=Config(s3={"addressing_style": "path"}),
)
if not bucket_name:
response = client.list_buckets()
file_buckets = [OnlineDriveFileBucket(bucket=bucket["Name"], files=[], is_truncated=False, next_page_parameters={}) for bucket in response["Buckets"]]
return OnlineDriveBrowseFilesResponse(result=file_buckets)
else:
if not next_page_parameters and prefix:
max_keys = max_keys + 1
continuation_token = next_page_parameters.get("continuation_token")
if continuation_token:
response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix, MaxKeys=max_keys, ContinuationToken=continuation_token, Delimiter="/")
else:
response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix, MaxKeys=max_keys, Delimiter="/")
is_truncated = response.get("IsTruncated", False)
next_continuation_token = response.get("NextContinuationToken")
next_page_parameters = {"continuation_token": next_continuation_token} if next_continuation_token else {}
files = []
files.extend([OnlineDriveFile(id=blob["Key"], name=os.path.basename(blob["Key"]), size=blob["Size"], type="file") for blob in response.get("Contents", []) if blob["Key"]!=prefix])
for prefix in response.get("CommonPrefixes", []):
files.append(OnlineDriveFile(id=prefix["Prefix"], name=os.path.basename(prefix["Prefix"].rstrip('/')), size=0, type="folder"))
file_bucket = OnlineDriveFileBucket(bucket=bucket_name, files=sorted(files, key=lambda x: x.id), is_truncated=is_truncated, next_page_parameters=next_page_parameters)
return OnlineDriveBrowseFilesResponse(result=[file_bucket])
def _download_file(self, request: OnlineDriveDownloadFileRequest) -> Generator[DatasourceMessage, None, None]:
credentials = self.runtime.credentials
bucket_name = request.bucket
key = request.id
if not credentials:
raise ValueError("Credentials not found")
if not bucket_name:
raise ValueError("Bucket name not found")
client = boto3.client(
"s3",
aws_secret_access_key=credentials.get("secret_access_key"),
aws_access_key_id=credentials.get("access_key_id"),
endpoint_url=f"https://s3.{credentials.get('region_name')}.amazonaws.com",
region_name=credentials.get("region_name"),
config=Config(s3={"addressing_style": "path"}),
)
response = client.get_object(Bucket=bucket_name, Key=key)
b64bytes = response["Body"].read()
yield self.create_blob_message(b64bytes, meta={"file_name": key, "mime_type": response["ContentType"]})
datasources/aws_s3_storage/datasources/aws_s3_storage.yaml
+20
View File
@@ -0,0 +1,20 @@
identity:
name: aws_s3_storage
author: langgenius
label:
en_US: AWS S3 Storage
zh_Hans: AWS S3 Storage
description:
en_US: Get files from your AWS S3 Storage bucket
zh_Hans: 从您的 AWS S3 Storage 存储桶中获取文件
parameters:
output_schema:
type: object
properties:
file:
$ref: "https://dify.ai/schemas/v1/file.json"
extra:
python:
source: datasources/aws_s3_storage.py
datasources/aws_s3_storage/main.py
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import Plugin, DifyPluginEnv
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/aws_s3_storage/manifest.yaml
+30
View File
@@ -0,0 +1,30 @@
version: 0.3.3
type: plugin
author: langgenius
name: aws_s3_storage
label:
en_US: AWS S3 Storage
zh_Hans: AWS S3 Storage
description:
en_US: AWS S3 Storage Datasource
zh_Hans: AWS S3 Storage 数据源
icon: icon.svg
resource:
memory: 268435456
permission: {}
plugins:
datasources:
- provider/aws_s3_storage.yaml
meta:
version: 0.0.1
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-05-13T14:54:47.062188+08:00
privacy:
verified: false
datasources/aws_s3_storage/provider/aws_s3_storage.py
+35
View File
@@ -0,0 +1,35 @@
from typing import Any, Mapping
from dify_plugin.errors.tool import ToolProviderCredentialValidationError
from dify_plugin.interfaces.datasource import DatasourceProvider
import boto3 # type: ignore
from botocore.client import Config # type: ignore
class GoogleCloudStorageDatasourceProvider(DatasourceProvider):
def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
try:
if not credentials or not credentials.get("secret_access_key"):
raise ToolProviderCredentialValidationError(
"AWS S3 Storage credentials are required."
)
if not credentials.get("access_key_id"):
raise ToolProviderCredentialValidationError(
"AWS S3 Storage access key is required."
)
if not credentials.get("region_name"):
raise ToolProviderCredentialValidationError(
"AWS S3 Storage region is required."
)
client = boto3.client(
"s3",
aws_secret_access_key=credentials.get("secret_access_key"),
aws_access_key_id=credentials.get("access_key_id"),
endpoint_url=f"https://s3.{credentials.get('region_name')}.amazonaws.com",
region_name=credentials.get("region_name"),
config=Config(s3={"addressing_style": "path"}),
)
client.list_buckets()
except Exception as e:
raise ToolProviderCredentialValidationError(str(e))
datasources/aws_s3_storage/provider/aws_s3_storage.yaml
+51
View File
@@ -0,0 +1,51 @@
identity:
author: langgenius
name: aws_s3_storage
label:
en_US: AWS S3 Storage
zh_Hans: AWS S3 Storage
description:
en_US: AWS S3 Storage Datasource
zh_Hans: AWS S3 Storage 数据源
icon: icon.svg
provider_type: online_drive
help:
title:
en_US: Get your credentials from AWS Console
zh_Hans: 从 AWS Console 获取您的凭证
url:
en_US: https://console.aws.amazon.com/
credentials_schema:
- name: secret_access_key
type: secret-input
required: true
label:
en_US: Secret Access Key
zh_Hans: 密钥
placeholder:
en_US: Enter your AWS S3 Storage secret access key
zh_Hans: 输入您的 AWS S3 Storage 密钥
- name: access_key_id
type: secret-input
required: true
label:
en_US: Access Key ID
zh_Hans: 访问密钥 ID
placeholder:
en_US: Enter your AWS S3 Storage access key ID
zh_Hans: 输入您的 AWS S3 Storage 访问密钥 ID
- name: region_name
type: text-input
required: true
label:
en_US: Region Name
zh_Hans: 区域名称
placeholder:
en_US: Enter your AWS S3 Storage region name
zh_Hans: 输入您的 AWS S3 Storage 区域名称
datasources:
- datasources/aws_s3_storage.yaml
extra:
python:
source: provider/aws_s3_storage.py
datasources/aws_s3_storage/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
dify_plugin==0.5.0b14
boto3
datasources/azure_blob/.difyignore
+69
View File
@@ -0,0 +1,69 @@
# Dify Plugin Ignore File
# Python cache
__pycache__/
*.py[cod]
*$py.class
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# Virtual environments
venv/
env/
ENV/
.venv/
.env/
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
# Logs
*.log
# Test files
test_*.py
*_test.py
tests/
.pytest_cache/
.coverage
htmlcov/
# Documentation build
docs/_build/
# Temporary files
*.tmp
*.temp
.cache/
# Local configuration
local_config.py
secrets.yaml
.env.local
datasources/azure_blob/PRIVACY.md
+158
View File
@@ -0,0 +1,158 @@
# Azure Blob Storage Datasource Plugin Privacy Policy
## Data Collection and Usage
### Information Collected
This plugin requires the following information to provide services:
#### Required Authentication Information
- **Storage Account Name**: Used to identify your Azure storage account
- **Authentication Credentials** (one of the following):
- Account access key
- SAS (Shared Access Signature) token
- Connection string
- **Endpoint Configuration**: Cloud environment endpoint suffix
#### Optional Configuration Information
- Default container name
- Custom endpoint settings
### Purpose of Information Usage
The collected information is only used for:
- Establishing secure connections to your Azure Blob Storage
- Verifying access permissions
- Browsing and retrieving stored file content
- Providing datasource service functionality
## Data Storage and Security
### Local Storage
- Authentication credentials are securely encrypted and stored in the Dify platform
- Your actual file content is not stored
- Only necessary metadata information is cached
### Transmission Security
- All communications with Azure use HTTPS encryption
- Supports Azure enterprise-grade security standards
- Follows Azure data transmission best practices
### Access Control
- Only authorized Dify users can access configured datasources
- Supports role-based access control
- Follows the principle of least privilege
## Data Access Scope
### Content We Access
- Storage container lists
- Blob file lists and metadata
- File content (only when explicitly requested)
- Basic storage account information
### Content We Do Not Access
- Unauthorized containers or files
- Storage accounts outside the configured scope
- Any personal identification information (unless stored in files)
## Third-Party Services
### Microsoft Azure
- This plugin communicates directly with Microsoft Azure Blob Storage service
- Follows Microsoft Azure's terms of service and privacy policy
- Azure privacy policy: https://privacy.microsoft.com/privacystatement
### Data Location
- Data remains within your selected Azure region
- No data transfer to other geographic locations
- Follows Azure data residency policies
## User Rights
### Data Control
- You retain full control over all data stored in Azure Blob Storage
- Can revoke plugin access permissions at any time
- Can delete or modify stored authentication information
### Access and Deletion
- Can view the scope of data accessed by the plugin
- Can disable or delete datasource configuration at any time
- Deleting configuration will clear all related authentication information
## Compliance
### Data Protection Regulations
- Supports GDPR (General Data Protection Regulation) compliance
- Follows applicable data protection laws
- Supports data subject rights requests
### Industry Standards
- Follows ISO 27001 information security management standards
- Meets SOC 2 Type II control requirements
- Supports Azure compliance certifications
## Security Best Practices
### Authentication Security
- Use SAS tokens instead of account keys (recommended)
- Set appropriate token expiration times
- Regularly rotate access credentials
- Enable access logs and monitoring
### Permission Management
- Grant only necessary read and list permissions
- Avoid using overly broad permissions
- Regularly audit access permissions
- Use Azure RBAC for fine-grained control
### Monitoring and Auditing
- Enable Azure storage account logs
- Monitor abnormal access patterns
- Set security alerts
- Regularly check access logs
## Incident Response
### Security Incidents
If security incidents are discovered, we will:
- Immediately investigate and assess impact
- Promptly notify affected users
- Take measures to prevent further data exposure
- Cooperate with relevant authorities for investigation
### Data Breach Notification
- Notify within 72 hours of discovering a data breach
- Provide incident details and impact assessment
- Explain remedial measures taken
- Provide protection recommendations
## Contact Information
### Privacy Issues
For privacy-related questions, please contact:
- Email: privacy@dify.ai
- Address: Dify Data Protection Officer
### Technical Support
For technical issues, please contact:
- Email: support@dify.ai
- Documentation: https://docs.dify.ai
## Policy Updates
### Update Notifications
- Privacy policy changes will be notified to users in advance
- Major changes require explicit user consent
- Update history is maintained in this document
### Version History
- **v1.0** (2025-01-27): Initial version
- Privacy protection support for three authentication methods
- Clear scope of data collection and usage
---
Last Updated: January 27, 2025
Version: 1.0
datasources/azure_blob/README.md
+67
View File
@@ -0,0 +1,67 @@
# Azure Blob Storage Datasource Plugin
Access Azure Blob Storage containers and blobs as a datasource for Dify with multiple authentication methods.
## Features
- **Multiple Authentication Methods**: Account key, SAS token, connection string, Azure AD OAuth
- **Container Browsing**: List all accessible storage containers
- **Blob Management**: Browse and download blob files from containers
- **Directory Simulation**: Virtual directory structure based on prefixes
- **Large File Support**: Automatic chunked download for large blob files
- **Multi-Cloud Support**: Global cloud, China cloud, Government cloud, Germany cloud
- **Complete OAuth Support**: Automatic access token refresh without re-authorization
- **Rich Metadata**: Complete blob properties and metadata
## Supported Authentication Methods
### 1. Account Key (Recommended for Development)
- Use storage account name and access key
- Provides full storage account access permissions
### 2. SAS Token (Recommended for Production)
- Use Shared Access Signature token
- Support fine-grained permission control and time limits
- Minimum permissions: read and list permissions
### 3. Connection String
- Use complete connection string
- Contains all required connection information
### 4. Azure AD OAuth (Recommended for Enterprise)
- Use Azure Active Directory authentication
- Support automatic access token refresh
- Multi-cloud environment support (Global, China, Government clouds)
- Principle of least privilege: only requires Storage user impersonation permissions
## Supported Content Types
- All types of blob files
- Automatic MIME type detection
- Support for text, images, documents, archives, etc.
- Large file chunked download (>50MB)
## Azure Cloud Support
- **Global Azure**: core.windows.net (default)
- **Azure China**: core.chinacloudapi.cn
- **Azure Government**: core.usgovcloudapi.net
## Version: 0.2.0
### Features in v0.2.0
- ✅ Account key authentication
- ✅ SAS token authentication
- ✅ Connection string authentication
- ✅ Container and blob browsing
- ✅ Large file chunked download
- ✅ Multi-cloud environment support
- ✅ Archive tier detection and alerts
- ✅ Complete Azure AD OAuth support
- ✅ Automatic access token refresh
### Security Features
- Secure storage of sensitive credentials
- Permission validation and error handling
- Principle of least privilege
- Secure authentication flows
datasources/azure_blob/_assets/icon.svg
+20
View File
@@ -0,0 +1,20 @@
<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="40" height="40" fill="white"/>
<path d="M8 10C8 8.89543 8.89543 8 10 8H30C31.1046 8 32 8.89543 32 10V12H8V10Z" fill="#0078D4"/>
<path d="M8 14V28C8 29.1046 8.89543 30 10 30H30C31.1046 30 32 29.1046 32 28V14H8Z" fill="#40E0D0"/>
<rect x="10" y="16" width="20" height="2" rx="1" fill="white"/>
<rect x="10" y="20" width="15" height="2" rx="1" fill="white"/>
<rect x="10" y="24" width="18" height="2" rx="1" fill="white"/>
<circle cx="28" cy="25" r="2" fill="#0078D4"/>
<path d="M6 32C6 33.1046 6.89543 34 8 34H32C33.1046 34 34 33.1046 34 32V31C34 30.4477 33.5523 30 33 30H7C6.44772 30 6 30.4477 6 31V32Z" fill="#005A9E"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 746 B

datasources/azure_blob/datasources/azure_blob.py
+538
View File
@@ -0,0 +1,538 @@
from collections.abc import Generator
from typing import Dict, List, Optional, Any
import mimetypes
import os
import logging
from datetime import datetime
import requests
from azure.storage.blob import BlobServiceClient, ContainerClient
from azure.core.exceptions import AzureError, ResourceNotFoundError
# Setup logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
from dify_plugin.entities.datasource import (
DatasourceMessage,
OnlineDriveBrowseFilesRequest,
OnlineDriveBrowseFilesResponse,
OnlineDriveDownloadFileRequest,
OnlineDriveFile,
OnlineDriveFileBucket,
)
from dify_plugin.interfaces.datasource.online_drive import OnlineDriveDatasource
class AzureBlobDataSource(OnlineDriveDatasource):
def invoke(self, request: Any) -> Generator[DatasourceMessage, None, None]:
"""Only use OnlineDrive standard browse/download process."""
yield from super().invoke(request)
def _get_blob_service_client(self) -> BlobServiceClient:
"""Get Blob service client"""
if not hasattr(self, '_blob_service_client') or self._blob_service_client is None:
credentials = self.runtime.credentials
auth_method = credentials.get("auth_method", "account_key")
account_name = credentials.get("account_name")
endpoint_suffix = credentials.get("endpoint_suffix", "core.windows.net")
if auth_method == "account_key":
account_key = credentials.get("account_key")
account_url = f"https://{account_name}.blob.{endpoint_suffix}"
self._blob_service_client = BlobServiceClient(
account_url=account_url,
credential=account_key
)
elif auth_method == "sas_token":
sas_token = credentials.get("sas_token")
if not sas_token.startswith('?'):
sas_token = '?' + sas_token
account_url = f"https://{account_name}.blob.{endpoint_suffix}"
self._blob_service_client = BlobServiceClient(
account_url=account_url + sas_token
)
elif auth_method == "connection_string":
connection_string = credentials.get("connection_string")
self._blob_service_client = BlobServiceClient.from_connection_string(
connection_string
)
elif auth_method == "oauth":
access_token = credentials.get("access_token")
account_url = f"https://{account_name}.blob.{endpoint_suffix}"
# Create simple token credential
from azure.core.credentials import AccessToken
from datetime import datetime, timezone
class SimpleTokenCredential:
def __init__(self, token, expires_in=3600):
self.token = token
self.expires_at = int(datetime.now(timezone.utc).timestamp()) + expires_in
def get_token(self, *scopes, **kwargs):
current_time = int(datetime.now(timezone.utc).timestamp())
if current_time >= self.expires_at - 300: # Refresh 5 minutes early
from azure.core.exceptions import ClientAuthenticationError
raise ClientAuthenticationError("Access token has expired, refresh required")
return AccessToken(self.token, self.expires_at)
credential = SimpleTokenCredential(access_token)
self._blob_service_client = BlobServiceClient(
account_url=account_url,
credential=credential
)
else:
raise ValueError(f"Unsupported authentication method: {auth_method}")
return self._blob_service_client
def _browse_files(self, request: OnlineDriveBrowseFilesRequest) -> OnlineDriveBrowseFilesResponse:
"""Browse Azure Blob Storage files"""
bucket_name = request.bucket # Container name
prefix = request.prefix or "" # Blob prefix
max_keys = request.max_keys or 100
next_page_parameters = request.next_page_parameters or {}
# Fix: If bucket_name is empty but prefix contains container path, parse from prefix
if not bucket_name and prefix:
# Check if prefix contains container name (format like "container-name/" or "container-name/path/")
prefix_parts = prefix.strip('/').split('/')
if len(prefix_parts) >= 1:
# First part might be container name
potential_container = prefix_parts[0]
remaining_prefix = '/'.join(prefix_parts[1:]) if len(prefix_parts) > 1 else ""
# Try to verify if this is a valid container name
blob_service_client = self._get_blob_service_client()
try:
container_client = blob_service_client.get_container_client(potential_container)
# Simple check if container exists (no exception means container exists)
container_client.get_container_properties()
# Container exists, use parsed values
bucket_name = potential_container
prefix = remaining_prefix
except Exception:
# Parsing failed, continue with original values
pass
try:
blob_service_client = self._get_blob_service_client()
if not bucket_name:
# List all containers
return self._list_containers(blob_service_client, max_keys, next_page_parameters)
else:
# List blobs in specified container
return self._list_blobs_in_container(
blob_service_client, bucket_name, prefix, max_keys, next_page_parameters
)
except ResourceNotFoundError:
if bucket_name:
raise ValueError(f"Container '{bucket_name}' not found")
else:
raise ValueError("Storage account not accessible")
except AzureError as e:
raise ValueError(f"Azure Blob Storage error: {str(e)}")
except Exception as e:
raise ValueError(f"Failed to browse Azure Blob Storage: {str(e)}")
def _list_containers(self, blob_service_client: BlobServiceClient, max_keys: int,
next_page_parameters: Dict) -> OnlineDriveBrowseFilesResponse:
"""List all containers"""
continuation_token = next_page_parameters.get("continuation_token")
# According to Azure Blob SDK documentation, list_containers doesn't support results_per_page parameter
# Use pagination iterator to control page size
if continuation_token:
containers_page = blob_service_client.list_containers().by_page(
continuation_token=continuation_token
)
else:
containers_page = blob_service_client.list_containers().by_page()
page = next(containers_page)
files = []
for container in page:
# Get container properties
try:
container_client = blob_service_client.get_container_client(container.name)
container_properties = container_client.get_container_properties()
files.append(OnlineDriveFile(
id=container.name,
name=container.name,
size=0, # Container itself has no size
type="folder",
metadata={
"container_name": container.name,
"last_modified": container.last_modified.isoformat() if container.last_modified else "",
"etag": container.etag or "",
"public_access": getattr(container_properties, "public_access", "none"),
"has_immutability_policy": getattr(container_properties, "has_immutability_policy", False),
"has_legal_hold": getattr(container_properties, "has_legal_hold", False)
}
))
except Exception:
# If unable to get container properties, use basic info
files.append(OnlineDriveFile(
id=container.name,
name=container.name,
size=0,
type="folder",
metadata={
"container_name": container.name,
"last_modified": container.last_modified.isoformat() if container.last_modified else "",
"etag": container.etag or ""
}
))
# Check if there are more pages
new_continuation_token = getattr(page, 'continuation_token', None)
is_truncated = new_continuation_token is not None
next_page_params = {"continuation_token": new_continuation_token} if is_truncated else {}
return OnlineDriveBrowseFilesResponse(
result=[OnlineDriveFileBucket(
bucket="", # bucket is empty when listing containers
files=files,
is_truncated=is_truncated,
next_page_parameters=next_page_params
)]
)
def _list_blobs_in_container(self, blob_service_client: BlobServiceClient, container_name: str,
prefix: str, max_keys: int, next_page_parameters: Dict) -> OnlineDriveBrowseFilesResponse:
"""List blobs in container"""
continuation_token = next_page_parameters.get("continuation_token")
try:
container_client = blob_service_client.get_container_client(container_name)
items_iter = container_client.walk_blobs(
name_starts_with=prefix if prefix else None
)
# Pagination
items_page_iter = items_iter.by_page(continuation_token) if continuation_token else items_iter.by_page()
page = next(items_page_iter)
files = []
seen_dirs = set()
# BlobPrefix represents directory, BlobProperties represents file
for item in page:
item_name = getattr(item, "name", None)
if not item_name:
continue
# Calculate relative name for display
display_name = item_name[len(prefix):] if prefix and item_name.startswith(prefix) else item_name
# Fix folder judgment logic: only explicit directory markers are folders
is_folder = (item_name.endswith("/") and getattr(item, "size", 0) == 0) or type(item).__name__ == "BlobPrefix"
if is_folder:
# Directory: only show first-level directories in current layer
first_dir = display_name.rstrip("/")
if "/" in first_dir:
first_dir = first_dir.split("/", 1)[0]
if first_dir and first_dir not in seen_dirs:
seen_dirs.add(first_dir)
dir_path = f"{prefix}{first_dir}/" if prefix else f"{first_dir}/"
# Construct correct directory ID format: container_name/dir_path
dir_id = f"{container_name}/{dir_path}"
files.append(OnlineDriveFile(
id=dir_id, # Use container_name/dir_path format
name=first_dir,
size=0,
type="folder",
metadata={
"container_name": container_name,
"blob_path": dir_path,
"is_directory": True
}
))
else:
# File: only show current level (no further /)
if "/" in display_name:
# Deeper level files not shown in this layer, carried by directory items
continue
content_type = self._get_content_type(item_name, getattr(item, "content_settings", None))
size_val = getattr(item, "size", 0) or 0
last_modified = getattr(item, "last_modified", None)
etag = getattr(item, "etag", "") or ""
creation_time = getattr(item, "creation_time", None)
blob_tier = getattr(item, "blob_tier", "Unknown")
metadata_val = getattr(item, "metadata", None) or {}
# Construct correct file ID format: container_name/blob_path
file_id = f"{container_name}/{item_name}"
files.append(OnlineDriveFile(
id=file_id, # Use container_name/blob_path format
name=display_name,
size=size_val,
type="file",
metadata={
"container_name": container_name,
"blob_path": item_name,
"content_type": content_type,
"last_modified": last_modified.isoformat() if last_modified else "",
"etag": etag,
"blob_tier": blob_tier,
"creation_time": creation_time.isoformat() if creation_time else "",
"server_encrypted": getattr(item, "server_encrypted", False),
"metadata": metadata_val,
}
))
# Check if there are more pages
new_continuation_token = getattr(page, 'continuation_token', None)
is_truncated = new_continuation_token is not None
next_page_params = {"continuation_token": new_continuation_token} if is_truncated else {}
return OnlineDriveBrowseFilesResponse(
result=[OnlineDriveFileBucket(
bucket=container_name,
files=files,
is_truncated=is_truncated,
next_page_parameters=next_page_params
)]
)
except ResourceNotFoundError:
raise ValueError(f"Container '{container_name}' not found")
except AzureError as e:
raise ValueError(f"Failed to list blobs in container '{container_name}': {str(e)}")
def _get_content_type(self, blob_name: str, content_settings) -> str:
"""Get content type"""
if content_settings and content_settings.content_type:
return content_settings.content_type
# Infer MIME type based on file extension
mime_type, _ = mimetypes.guess_type(blob_name)
return mime_type or "application/octet-stream"
def _download_file(self, request: OnlineDriveDownloadFileRequest) -> Generator[DatasourceMessage, None, None]:
"""Download file content"""
file_id = request.id # Format: container_name/blob_path
if '/' not in file_id:
raise ValueError("Invalid file ID format. Expected: container_name/blob_path")
parts = file_id.split('/', 1)
container_name = parts[0]
blob_path = parts[1]
try:
logger.info(f"[Azure Blob] Starting download process for file: {file_id}")
blob_service_client = self._get_blob_service_client()
blob_client = blob_service_client.get_blob_client(
container=container_name,
blob=blob_path
)
# Get Blob properties
blob_properties = blob_client.get_blob_properties()
logger.info(f"[Azure Blob] Blob properties retrieved: size={blob_properties.size}, container={container_name}")
# Check Blob tier, special handling needed for archive tier
blob_tier = getattr(blob_properties, 'blob_tier', '')
if blob_tier and blob_tier.lower() == 'archive':
logger.error(f"[Azure Blob] Blob is in archive tier: {blob_path}")
raise ValueError(f"Blob '{blob_path}' is in Archive tier and needs to be rehydrated before download")
# Verify file exists and is valid
blob_size = blob_properties.size
if blob_size is None or blob_size < 0:
raise ValueError(f"Invalid blob size: {blob_size}")
content_type = self._get_content_type(blob_path, blob_properties.content_settings)
logger.info(f"[Azure Blob] Blob metadata: size={blob_size}, type={content_type}, tier={blob_tier}")
# Prefer SAS direct HTTP download (avoid SDK limitations)
credentials = self.runtime.credentials
auth_method = (credentials or {}).get("auth_method", "account_key")
if auth_method == "sas_token":
logger.info("[Azure Blob] Using SAS HTTP download path")
yield from self._download_via_sas_http(container_name, blob_path)
else:
# For large files, use streaming download (SDK)
if blob_size > 50 * 1024 * 1024: # 50MB
logger.info(f"[Azure Blob] Using large file download for {blob_size} bytes")
yield from self._download_large_blob(blob_client, blob_path, content_type, blob_size)
else:
logger.info(f"[Azure Blob] Using small file download for {blob_size} bytes")
yield from self._download_small_blob(blob_client, blob_path, content_type, blob_size)
logger.info(f"[Azure Blob] Download process completed successfully for: {file_id}")
except ResourceNotFoundError:
logger.error(f"[Azure Blob] Blob not found: {blob_path} in container {container_name}")
raise ValueError(f"Blob '{blob_path}' not found in container '{container_name}'")
except AzureError as e:
logger.error(f"[Azure Blob] Azure error during download: {str(e)}")
raise ValueError(f"Failed to download blob '{blob_path}': {str(e)}")
except Exception as e:
logger.error(f"[Azure Blob] Unexpected error during download: {str(e)}")
raise ValueError(f"Error downloading file: {str(e)}")
def _download_via_sas_http(self, container_name: str, blob_path: str) -> Generator[DatasourceMessage, None, None]:
"""Download via HTTP using SAS URL (not dependent on SDK data stream)."""
credentials = self.runtime.credentials or {}
account = credentials.get("account_name")
suffix = credentials.get("endpoint_suffix", "core.windows.net")
sas = credentials.get("sas_token") or ""
if not account:
raise ValueError("account_name not configured")
if not sas:
raise ValueError("sas_token not configured for SAS HTTP download")
if not sas.startswith("?"):
sas = "?" + sas
url = f"https://{account}.blob.{suffix}/{container_name}/{blob_path}{sas}"
with requests.get(url, stream=True, timeout=60) as resp:
resp.raise_for_status()
content_type = resp.headers.get("Content-Type", "application/octet-stream")
file_name = os.path.basename(blob_path)
content_length_header = resp.headers.get("Content-Length")
try:
content_length = int(content_length_header) if content_length_header else 0
except Exception:
content_length = 0
# Small files return at once
if 0 < content_length <= 50 * 1024 * 1024:
data = resp.content
yield self.create_blob_message(data, meta={
"file_name": file_name,
"mime_type": content_type,
"size": len(data),
})
return
# Large files return in chunks
chunk_size = 8 * 1024 * 1024
buffer = bytearray()
for chunk in resp.iter_content(chunk_size=chunk_size):
if not chunk:
continue
buffer.extend(chunk)
if len(buffer) >= 100 * 1024 * 1024: # 100MB batch output
yield self.create_blob_message(bytes(buffer), meta={
"file_name": file_name,
"mime_type": content_type,
"is_partial": True,
})
buffer = bytearray()
if buffer:
yield self.create_blob_message(bytes(buffer), meta={
"file_name": file_name,
"mime_type": content_type,
"is_partial": False,
})
def _download_small_blob(self, blob_client, blob_path: str, content_type: str,
blob_size: int) -> Generator[DatasourceMessage, None, None]:
"""Download small file"""
try:
logger.info(f"[Azure Blob] Starting download of small file: {blob_path}")
download_stream = blob_client.download_blob()
content = download_stream.readall()
# Verify download success
actual_size = len(content)
if actual_size != blob_size:
logger.warning(f"[Azure Blob] Size mismatch for {blob_path}: expected {blob_size}, got {actual_size}")
else:
logger.info(f"[Azure Blob] Successfully downloaded {blob_path}: {actual_size} bytes")
# Verify content is not empty
if not content:
raise ValueError(f"Downloaded content is empty for blob: {blob_path}")
# Extract file name and MIME type
file_name = os.path.basename(blob_path)
logger.info(f"[Azure Blob] Creating blob message for {file_name} with {actual_size} bytes")
yield self.create_blob_message(
blob=content,
meta={
"file_name": file_name,
"mime_type": content_type,
"size": actual_size,
"download_success": True
}
)
logger.info(f"[Azure Blob] Successfully yielded blob message for {file_name}")
except Exception as e:
raise ValueError(f"Failed to download blob content: {str(e)}")
def _download_large_blob(self, blob_client, blob_path: str, content_type: str,
blob_size: int) -> Generator[DatasourceMessage, None, None]:
"""Download large file in chunks"""
try:
logger.info(f"[Azure Blob] Starting download of large file: {blob_path} ({blob_size} bytes)")
# Extract file name
file_name = os.path.basename(blob_path)
chunk_size = 8 * 1024 * 1024 # 8MB chunks
downloaded_content = bytearray()
total_downloaded = 0
# Download in chunks
for i in range(0, blob_size, chunk_size):
end_range = min(i + chunk_size - 1, blob_size - 1)
download_stream = blob_client.download_blob(offset=i, length=end_range - i + 1)
chunk = download_stream.readall()
downloaded_content.extend(chunk)
total_downloaded += len(chunk)
logger.debug(f"[Azure Blob] Downloaded chunk {i//chunk_size + 1}: {len(chunk)} bytes (total: {total_downloaded}/{blob_size})")
# If accumulated content is too large, can yield in batches
if len(downloaded_content) > 100 * 1024 * 1024: # 100MB
yield self.create_blob_message(
blob=bytes(downloaded_content),
meta={
"file_name": file_name,
"mime_type": content_type,
"size": len(downloaded_content),
"is_partial": True
}
)
downloaded_content = bytearray()
# Verify download integrity
if total_downloaded != blob_size:
logger.error(f"[Azure Blob] Download incomplete: expected {blob_size}, got {total_downloaded}")
raise ValueError(f"Download incomplete: expected {blob_size}, got {total_downloaded}")
logger.info(f"[Azure Blob] Large file download completed: {total_downloaded} bytes")
# Output remaining content
if downloaded_content:
yield self.create_blob_message(
blob=bytes(downloaded_content),
meta={
"file_name": file_name,
"mime_type": content_type,
"size": len(downloaded_content),
"download_success": True,
"is_partial": False
}
)
except Exception as e:
raise ValueError(f"Failed to download large blob: {str(e)}")
datasources/azure_blob/datasources/azure_blob.yaml
+28
View File
@@ -0,0 +1,28 @@
identity:
name: azure_blob
author: langgenius
label:
en_US: Azure Blob Storage
zh_Hans: Azure Blob 存储
icon: icon.svg
description:
en_US: Access Azure Blob Storage containers and blobs with support for multiple authentication methods
zh_Hans: 访问 Azure Blob 存储容器和 Blob，支持多种认证方式
output_schema:
type: object
properties:
file:
$ref: "https://dify.ai/schemas/v1/file.json"
extra:
python:
source: datasources/azure_blob.py
datasources/azure_blob/main.py
+7
View File
@@ -0,0 +1,7 @@
from dify_plugin import DifyPluginEnv, Plugin
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/azure_blob/manifest.yaml
+39
View File
@@ -0,0 +1,39 @@
version: 0.2.4
type: plugin
author: langgenius
name: azure_blob_datasource
label:
en_US: Azure Blob Storage
zh_Hans: Azure Blob 存储
description:
en_US: Azure Blob Storage Datasource - Access containers, blobs, and files with multiple authentication methods
zh_Hans: Azure Blob 存储数据源 - 支持多种认证方式访问容器、Blob 和文件
icon: icon.svg
resource:
memory: 268435456
permission: {}
plugins:
datasources:
- provider/azure_blob.yaml
meta:
version: 0.2.0
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-01-27T16:00:00.000000+08:00
privacy: PRIVACY.md
verified: false
datasources/azure_blob/provider/azure_blob.py
+453
View File
@@ -0,0 +1,453 @@
from typing import Any, Mapping
from azure.storage.blob import BlobServiceClient
from azure.core.exceptions import AzureError, ClientAuthenticationError
from azure.identity import ClientSecretCredential
import re
import requests
import urllib.parse
from flask import Request
from dify_plugin.interfaces.datasource import DatasourceProvider, DatasourceOAuthCredentials
from dify_plugin.errors.tool import DatasourceOAuthError
class AzureBlobDatasourceProvider(DatasourceProvider):
def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
"""Validate credential validity"""
auth_method = credentials.get("auth_method", "account_key")
account_name = credentials.get("account_name")
endpoint_suffix = credentials.get("endpoint_suffix", "core.windows.net")
if not account_name:
raise ValueError("Storage account name is required")
# Validate account name format
if not self._is_valid_account_name(account_name):
raise ValueError("Invalid storage account name. Must be 3-24 characters, lowercase letters and numbers only")
# Validate corresponding credentials based on authentication method
if auth_method == "account_key":
account_key = credentials.get("account_key")
if not account_key or account_key.strip() == "":
raise ValueError("Account key is required when using account key authentication")
self._validate_account_key_access(account_name, account_key, endpoint_suffix)
elif auth_method == "sas_token":
sas_token = credentials.get("sas_token")
if not sas_token or sas_token.strip() == "":
raise ValueError("SAS token is required when using SAS token authentication")
self._validate_sas_token_access(account_name, sas_token, endpoint_suffix)
elif auth_method == "connection_string":
connection_string = credentials.get("connection_string")
if not connection_string or connection_string.strip() == "":
raise ValueError("Connection string is required when using connection string authentication")
self._validate_connection_string_access(connection_string)
elif auth_method == "oauth":
access_token = credentials.get("access_token")
if not access_token or access_token.strip() == "":
raise ValueError("Access token is required. Please complete OAuth authorization first")
self._validate_oauth_access(credentials)
else:
raise ValueError(f"Unsupported authentication method: {auth_method}")
def _is_valid_account_name(self, account_name: str) -> bool:
"""Validate Azure storage account name format"""
if not account_name or len(account_name) < 3 or len(account_name) > 24:
return False
return bool(re.match(r'^[a-z0-9]+$', account_name))
def _validate_account_key_access(self, account_name: str, account_key: str, endpoint_suffix: str) -> None:
"""Validate account key access"""
try:
account_url = f"https://{account_name}.blob.{endpoint_suffix}"
blob_service_client = BlobServiceClient(
account_url=account_url,
credential=account_key
)
# Try to list containers to verify access permissions
containers_iter = blob_service_client.list_containers()
# Only get first container to verify access permissions
containers = []
for container in containers_iter:
containers.append(container)
break # Only get one for verification
# Successfully listing containers or getting empty list indicates successful authentication
except ClientAuthenticationError as e:
raise ValueError(f"Invalid account key or insufficient permissions: {str(e)}")
except AzureError as e:
if "AuthenticationFailed" in str(e):
raise ValueError(f"Authentication failed: {str(e)}")
elif "AccountIsDisabled" in str(e):
raise ValueError("Storage account is disabled")
elif "InvalidResourceName" in str(e):
raise ValueError("Invalid storage account name")
else:
raise ValueError(f"Azure Storage error: {str(e)}")
except Exception as e:
raise ValueError(f"Failed to validate Azure Blob Storage credentials: {str(e)}")
def _validate_sas_token_access(self, account_name: str, sas_token: str, endpoint_suffix: str) -> None:
"""Validate SAS token access"""
try:
# Ensure SAS token starts with ?
if not sas_token.startswith('?'):
sas_token = '?' + sas_token
account_url = f"https://{account_name}.blob.{endpoint_suffix}"
blob_service_client = BlobServiceClient(
account_url=account_url + sas_token
)
# Try to list containers to verify SAS permissions
containers_iter = blob_service_client.list_containers()
# Only get first container to verify SAS permissions
containers = []
for container in containers_iter:
containers.append(container)
break # Only get one for verification
except ClientAuthenticationError as e:
raise ValueError(f"Invalid SAS token or insufficient permissions: {str(e)}")
except AzureError as e:
if "AuthenticationFailed" in str(e):
raise ValueError("SAS token authentication failed. Please check token validity and permissions")
elif "AuthorizationPermissionMismatch" in str(e):
raise ValueError("SAS token does not have required permissions (read and list)")
elif "InvalidQueryParameterValue" in str(e):
raise ValueError("Invalid SAS token format")
else:
raise ValueError(f"SAS token validation error: {str(e)}")
except Exception as e:
raise ValueError(f"Failed to validate SAS token: {str(e)}")
def _validate_connection_string_access(self, connection_string: str) -> None:
"""Validate connection string access"""
try:
blob_service_client = BlobServiceClient.from_connection_string(connection_string)
# Try to list containers to verify connection
containers_iter = blob_service_client.list_containers()
# Only get first container to verify connection
containers = []
for container in containers_iter:
containers.append(container)
break # Only get one for verification
except ClientAuthenticationError as e:
raise ValueError(f"Invalid connection string or insufficient permissions: {str(e)}")
except ValueError as e:
if "connection string" in str(e).lower():
raise ValueError("Invalid connection string format")
raise e
except AzureError as e:
if "AuthenticationFailed" in str(e):
raise ValueError("Connection string authentication failed")
else:
raise ValueError(f"Connection string validation error: {str(e)}")
except Exception as e:
raise ValueError(f"Failed to validate connection string: {str(e)}")
def _get_account_url(self, account_name: str, endpoint_suffix: str) -> str:
"""Build storage account URL"""
return f"https://{account_name}.blob.{endpoint_suffix}"
def _create_blob_service_client(self, credentials: Mapping[str, Any]) -> BlobServiceClient:
"""Create Blob service client"""
auth_method = credentials.get("auth_method", "account_key")
account_name = credentials.get("account_name")
endpoint_suffix = credentials.get("endpoint_suffix", "core.windows.net")
if auth_method == "account_key":
account_key = credentials.get("account_key")
account_url = self._get_account_url(account_name, endpoint_suffix)
return BlobServiceClient(account_url=account_url, credential=account_key)
elif auth_method == "sas_token":
sas_token = credentials.get("sas_token")
if not sas_token.startswith('?'):
sas_token = '?' + sas_token
account_url = self._get_account_url(account_name, endpoint_suffix)
return BlobServiceClient(account_url=account_url + sas_token)
elif auth_method == "connection_string":
connection_string = credentials.get("connection_string")
return BlobServiceClient.from_connection_string(connection_string)
elif auth_method == "oauth":
access_token = credentials.get("access_token")
account_url = self._get_account_url(account_name, endpoint_suffix)
# Create simple token credential
from azure.core.credentials import AccessToken
from datetime import datetime, timezone
class SimpleTokenCredential:
def __init__(self, token, expires_in=3600):
self.token = token
self.expires_at = int(datetime.now(timezone.utc).timestamp()) + expires_in
def get_token(self, *scopes, **kwargs):
current_time = int(datetime.now(timezone.utc).timestamp())
if current_time >= self.expires_at - 300: # Refresh 5 minutes early
raise ClientAuthenticationError("Access token has expired, refresh required")
return AccessToken(self.token, self.expires_at)
credential = SimpleTokenCredential(access_token)
return BlobServiceClient(account_url=account_url, credential=credential)
else:
raise ValueError(f"Unsupported authentication method: {auth_method}")
def _get_storage_info(self, credentials: Mapping[str, Any]) -> dict:
"""Get storage account information (for validation and metadata)"""
try:
blob_service_client = self._create_blob_service_client(credentials)
account_info = blob_service_client.get_account_information()
return {
"account_kind": account_info.get("account_kind", "Unknown"),
"sku_name": account_info.get("sku_name", "Unknown"),
"is_hns_enabled": account_info.get("is_hns_enabled", False)
}
except Exception:
# If unable to get account info, return basic info
return {
"account_kind": "StorageV2",
"sku_name": "Standard_LRS",
"is_hns_enabled": False
}
def _validate_oauth_access(self, credentials: Mapping[str, Any]) -> None:
"""Validate OAuth access"""
try:
account_name = credentials.get("account_name")
endpoint_suffix = credentials.get("endpoint_suffix", "core.windows.net")
access_token = credentials.get("access_token")
if not account_name:
raise ValueError("Storage account name is required for OAuth authentication")
# Create BlobServiceClient using access token
account_url = f"https://{account_name}.blob.{endpoint_suffix}"
# Use TokenCredential approach
from azure.core.credentials import AccessToken
from azure.identity import AccessTokenCredential
from datetime import datetime, timezone
# Create a simple token credential
class SimpleTokenCredential:
def __init__(self, token, expires_in=3600):
self.token = token
self.expires_at = int(datetime.now(timezone.utc).timestamp()) + expires_in
def get_token(self, *scopes, **kwargs):
current_time = int(datetime.now(timezone.utc).timestamp())
if current_time >= self.expires_at - 300: # Refresh 5 minutes early
raise ClientAuthenticationError("Access token has expired, refresh required")
return AccessToken(self.token, self.expires_at)
credential = SimpleTokenCredential(access_token)
blob_service_client = BlobServiceClient(account_url=account_url, credential=credential)
# Try to list containers to verify permissions
containers_iter = blob_service_client.list_containers()
# Only get first container to verify permissions
containers = []
for container in containers_iter:
containers.append(container)
break # Only get one for verification
except ClientAuthenticationError as e:
raise ValueError(f"Invalid OAuth token or insufficient permissions: {str(e)}")
except AzureError as e:
if "AuthenticationFailed" in str(e):
raise ValueError("OAuth authentication failed. Please check access token validity")
else:
raise ValueError(f"OAuth validation error: {str(e)}")
except Exception as e:
raise ValueError(f"Error validating OAuth access: {str(e)}")
def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str:
"""Get OAuth authorization URL"""
client_id = system_credentials.get("client_id")
tenant_id = system_credentials.get("tenant_id")
cloud_environment = system_credentials.get("cloud_environment", "global")
if not client_id or not tenant_id:
raise ValueError("Incomplete OAuth configuration: missing client_id or tenant_id")
# Get login endpoint and scope based on cloud environment
login_endpoint, storage_scope = self._get_cloud_endpoints(cloud_environment)
auth_url = f"https://{login_endpoint}/{tenant_id}/oauth2/v2.0/authorize"
params = {
"client_id": client_id,
"response_type": "code",
"redirect_uri": redirect_uri,
"scope": storage_scope,
"response_mode": "query",
"prompt": "consent" # Ensure user consents to permissions
}
return f"{auth_url}?{urllib.parse.urlencode(params)}"
def _oauth_get_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request
) -> DatasourceOAuthCredentials:
"""Get OAuth credentials"""
code = request.args.get("code")
if not code:
raise ValueError("Authorization code not found")
client_id = system_credentials.get("client_id")
client_secret = system_credentials.get("client_secret")
tenant_id = system_credentials.get("tenant_id")
cloud_environment = system_credentials.get("cloud_environment", "global")
if not all([client_id, client_secret, tenant_id]):
raise ValueError("Incomplete OAuth configuration")
# Get login endpoint based on cloud environment
login_endpoint, storage_scope = self._get_cloud_endpoints(cloud_environment)
# Exchange authorization code for access token
token_url = f"https://{login_endpoint}/{tenant_id}/oauth2/v2.0/token"
token_data = {
"client_id": client_id,
"client_secret": client_secret,
"code": code,
"redirect_uri": redirect_uri,
"grant_type": "authorization_code",
"scope": storage_scope
}
headers = {
"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded",
}
try:
response = requests.post(token_url, data=token_data, headers=headers, timeout=30)
if response.status_code != 200:
error_detail = response.text
raise ValueError(f"Failed to obtain access token: {error_detail}")
token_json = response.json()
access_token = token_json.get("access_token")
refresh_token = token_json.get("refresh_token")
if not access_token:
raise ValueError("Failed to obtain access token")
# Note: client_secret is not returned here
return DatasourceOAuthCredentials(
credentials={
"access_token": access_token,
"refresh_token": refresh_token,
"client_id": client_id,
"tenant_id": tenant_id
}
)
except requests.RequestException as e:
raise ValueError(f"Error requesting access token: {str(e)}")
except Exception as e:
raise ValueError(f"Error in OAuth authentication process: {str(e)}")
def _refresh_access_token(self, credentials: Mapping[str, Any]) -> Mapping[str, Any]:
"""Refresh access token"""
refresh_token = credentials.get("refresh_token")
client_id = credentials.get("client_id")
tenant_id = credentials.get("tenant_id")
# Note: client_secret needs to be obtained from system configuration, not from credentials
# In actual implementation, client_secret may need to be obtained through other means
if not refresh_token or not client_id or not tenant_id:
raise ValueError("Incomplete credentials required for token refresh")
# Need to get client_secret here, but cannot get it from user credentials
raise ValueError("Access token has expired, please re-authorize OAuth")
def _oauth_refresh_credentials(self, redirect_uri: str, system_credentials: Mapping[str, Any], credentials: Mapping[str, Any]) -> DatasourceOAuthCredentials:
"""OAuth refresh credentials method - interface required by Dify framework"""
refresh_token = credentials.get("refresh_token")
client_id = credentials.get("client_id") or system_credentials.get("client_id")
client_secret = system_credentials.get("client_secret") # Get from system configuration
tenant_id = credentials.get("tenant_id") or system_credentials.get("tenant_id")
cloud_environment = system_credentials.get("cloud_environment", "global")
if not refresh_token:
raise DatasourceOAuthError("No refresh token available, please re-authorize OAuth")
if not all([client_id, client_secret, tenant_id]):
raise DatasourceOAuthError("Incomplete OAuth refresh configuration")
# Get login endpoint based on cloud environment
login_endpoint, storage_scope = self._get_cloud_endpoints(cloud_environment)
# Refresh access token
token_url = f"https://{login_endpoint}/{tenant_id}/oauth2/v2.0/token"
token_data = {
"client_id": client_id,
"client_secret": client_secret,
"refresh_token": refresh_token,
"grant_type": "refresh_token",
"scope": storage_scope
}
headers = {
"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded",
}
try:
response = requests.post(token_url, data=token_data, headers=headers, timeout=30)
if response.status_code != 200:
error_detail = response.text
raise DatasourceOAuthError(f"Failed to refresh access token: {error_detail}")
token_json = response.json()
new_access_token = token_json.get("access_token")
new_refresh_token = token_json.get("refresh_token", refresh_token) # Use new or keep existing
if not new_access_token:
raise DatasourceOAuthError("Failed to obtain new access token")
return DatasourceOAuthCredentials(
credentials={
"access_token": new_access_token,
"refresh_token": new_refresh_token,
"client_id": client_id,
"tenant_id": tenant_id
}
)
except requests.RequestException as e:
raise DatasourceOAuthError(f"Token refresh request failed: {str(e)}")
except Exception as e:
raise DatasourceOAuthError(f"Error in OAuth refresh process: {str(e)}")
def _get_cloud_endpoints(self, cloud_environment: str) -> tuple[str, str]:
"""Get endpoints and scope for different cloud environments"""
if cloud_environment == "china":
return "login.chinacloudapi.cn", "https://storage.azure.cn/user_impersonation"
elif cloud_environment == "government":
return "login.microsoftonline.us", "https://storage.azure.us/user_impersonation"
elif cloud_environment == "germany":
return "login.microsoftonline.de", "https://storage.azure.de/user_impersonation"
else: # global (default)
return "login.microsoftonline.com", "https://storage.azure.com/user_impersonation"
datasources/azure_blob/provider/azure_blob.yaml
+221
View File
@@ -0,0 +1,221 @@
identity:
author: langgenius
name: azure_blob
label:
en_US: Azure Blob Storage
zh_Hans: Azure Blob 存储
description:
en_US: Azure Blob Storage Datasource - Access containers and blobs with multiple authentication methods
zh_Hans: Azure Blob 存储数据源 - 支持多种认证方式访问容器和 Blob
icon: icon.svg
provider_type: online_drive
help:
title:
en_US: Configure Azure Blob Storage credentials
zh_Hans: 配置 Azure Blob 存储凭证
url:
en_US: https://portal.azure.com/#view/Microsoft_Azure_Storage/StorageMenuBlade/~/AccessKeys
zh_Hans: https://portal.azure.com/#view/Microsoft_Azure_Storage/StorageMenuBlade/~/AccessKeys
oauth_schema:
client_schema:
- name: client_id
type: secret-input
required: true
label:
en_US: Application (Client) ID
zh_Hans: 应用程序（客户端）ID
placeholder:
en_US: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
zh_Hans: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
help:
en_US: Azure AD Application (Client) ID from app registration
zh_Hans: 来自应用注册的 Azure AD 应用程序（客户端）ID
url: https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade
- name: client_secret
type: secret-input
required: true
label:
en_US: Client Secret
zh_Hans: 客户端密钥
placeholder:
en_US: Client secret value
zh_Hans: 客户端密钥值
help:
en_US: Client secret from Azure AD app registration
zh_Hans: 来自 Azure AD 应用注册的客户端密钥
url: https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationsListBlade
- name: tenant_id
type: secret-input
required: true
label:
en_US: Tenant ID
zh_Hans: 租户 ID
placeholder:
en_US: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
zh_Hans: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
help:
en_US: Azure AD Tenant ID
zh_Hans: Azure AD 租户 ID
url: https://portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/Properties
- name: cloud_environment
type: select
required: false
label:
en_US: Cloud Environment
zh_Hans: 云环境
options:
- value: global
label:
en_US: Global Azure (login.microsoftonline.com)
zh_Hans: 全球 Azure (login.microsoftonline.com)
- value: china
label:
en_US: Azure China (login.chinacloudapi.cn)
zh_Hans: Azure 中国 (login.chinacloudapi.cn)
- value: government
label:
en_US: Azure Government (login.microsoftonline.us)
zh_Hans: Azure 政府 (login.microsoftonline.us)
- value: germany
label:
en_US: Azure Germany (login.microsoftonline.de)
zh_Hans: Azure 德国 (login.microsoftonline.de)
default: global
help:
en_US: Select the correct Azure cloud environment for your tenant
zh_Hans: 选择租户所在的正确 Azure 云环境
credentials_schema:
- name: access_token
type: secret-input
label:
en_US: Access Token
zh_Hans: 访问令牌
- name: refresh_token
type: secret-input
label:
en_US: Refresh Token
zh_Hans: 刷新令牌
- name: client_id
type: secret-input
label:
en_US: Application (Client) ID
zh_Hans: 应用程序（客户端）ID
- name: client_secret
type: secret-input
label:
en_US: Client Secret
zh_Hans: 客户端密钥
- name: tenant_id
type: secret-input
label:
en_US: Tenant ID
zh_Hans: 租户 ID
credentials_schema:
- name: auth_method
type: select
required: true
label:
en_US: Authentication Method
zh_Hans: 认证方式
options:
- value: account_key
label:
en_US: Account Key
zh_Hans: 账户密钥
- value: sas_token
label:
en_US: SAS Token
zh_Hans: SAS 令牌
- value: connection_string
label:
en_US: Connection String
zh_Hans: 连接字符串
- value: oauth
label:
en_US: Azure AD OAuth
zh_Hans: Azure AD OAuth
default: account_key
help:
en_US: Choose your preferred authentication method
zh_Hans: 选择您偏好的认证方式
- name: account_name
type: text-input
required: true
label:
en_US: Storage Account Name
zh_Hans: 存储账户名称
placeholder:
en_US: mystorageaccount
zh_Hans: mystorageaccount
help:
en_US: Your Azure Storage account name
zh_Hans: 您的 Azure 存储账户名称
- name: account_key
type: secret-input
required: false
label:
en_US: Account Key
zh_Hans: 账户密钥
placeholder:
en_US: Primary or secondary access key
zh_Hans: 主要或次要访问密钥
help:
en_US: Required when using "Account Key" authentication method. Leave empty for other methods.
zh_Hans: 选择"账户密钥"认证方式时必填。使用其他认证方式时请留空。
url: https://portal.azure.com/#view/Microsoft_Azure_Storage/StorageMenuBlade/~/AccessKeys
- name: sas_token
type: secret-input
required: false
label:
en_US: SAS Token
zh_Hans: SAS 令牌
placeholder:
en_US: ?sv=2022-11-02&ss=b&srt=co&sp=rl&se=...
zh_Hans: ?sv=2022-11-02&ss=b&srt=co&sp=rl&se=...
help:
en_US: Required when using "SAS Token" authentication method. SAS token with read and list permissions for blob service.
zh_Hans: 选择"SAS 令牌"认证方式时必填。需要具有 Blob 服务读取和列表权限的 SAS 令牌。
url: https://docs.microsoft.com/azure/storage/common/storage-sas-overview
- name: connection_string
type: secret-input
required: false
label:
en_US: Connection String
zh_Hans: 连接字符串
placeholder:
en_US: DefaultEndpointsProtocol=https;AccountName=...
zh_Hans: DefaultEndpointsProtocol=https;AccountName=...
help:
en_US: Required when using "Connection String" authentication method. Complete connection string for the storage account.
zh_Hans: 选择"连接字符串"认证方式时必填。存储账户的完整连接字符串。
url: https://portal.azure.com/#view/Microsoft_Azure_Storage/StorageMenuBlade/~/AccessKeys
- name: endpoint_suffix
type: text-input
required: false
label:
en_US: Endpoint Suffix
zh_Hans: 端点后缀
placeholder:
en_US: core.windows.net
zh_Hans: core.windows.net
default: core.windows.net
help:
en_US: 'Endpoint suffix: core.windows.net (Global), core.chinacloudapi.cn (China), core.usgovcloudapi.net (Government)'
zh_Hans: '端点后缀：core.windows.net（全球）、core.chinacloudapi.cn（中国）、core.usgovcloudapi.net（政府）'
- name: default_container
type: text-input
required: false
label:
en_US: Default Container
zh_Hans: 默认容器
placeholder:
en_US: my-container
zh_Hans: my-container
help:
en_US: Optional default container to browse (leave empty to list all containers)
zh_Hans: 可选的默认浏览容器（留空则列出所有容器）
datasources:
- datasources/azure_blob.yaml
extra:
python:
source: provider/azure_blob.py
datasources/azure_blob/requirements.txt
+6
View File
@@ -0,0 +1,6 @@
dify-plugin>=0.5.0b14
azure-storage-blob>=12.20.0
azure-identity>=1.14.0
azure-core>=1.29.0
python-dateutil>=2.8.0
requests>=2.28.0
datasources/box_datasource/.difyignore
+179
View File
@@ -0,0 +1,179 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
# Git
.git/
.gitignore
.github/
# Mac
.DS_Store
# Windows
Thumbs.db
datasources/box_datasource/PRIVACY.md
+98
View File
@@ -0,0 +1,98 @@
# Privacy Policy
This privacy policy outlines how the Dify Box Datasource Plugin handles your information when you use it to interact with your Box cloud storage.
## Data Collection
The Dify Box Datasource Plugin requires the following information to function:
1. **Box OAuth Credentials**: Your Box OAuth 2.0 client ID and client secret are required to authenticate with the Box API. These credentials are securely stored by Dify's credential management system and are used solely for making API requests to your Box account on your behalf.
2. **OAuth Tokens**: During the OAuth flow, the plugin obtains and stores:
- Access tokens for API authentication
- Refresh tokens for automatic token renewal
- Token expiration information
3. **User Information**: The plugin retrieves basic user information from Box API:
- User name
- User email address
- User ID (for API operations)
4. **File Operations Data**: When you use the plugin to browse or download files:
- File metadata (names, IDs, sizes, types)
- Folder paths and structures
- File content (only when explicitly downloading)
## Data Processing
- **OAuth Credentials**: Your Box OAuth credentials are used exclusively to authenticate API requests sent from the plugin to your Box account. They are not used for any other purpose.
- **File Operations**: The plugin processes file data only when you explicitly request file browsing or downloading:
- File browsing: Retrieves file and folder metadata from your Box account
- File downloading: Downloads file content temporarily for processing
- No file content is stored persistently by the plugin
- **Real-time Processing**: All operations are performed in real-time. File content is processed on-demand and not cached or stored locally.
## Third-party Services
This plugin interacts directly with the **Box API** and **Box OAuth 2.0** services. Your use of Box is subject to Box's privacy policy and terms of service. The plugin acts as an intermediary, sending your requests and data to the Box API and returning Box's responses. No data is shared with any other third-party services.
### Box API Scopes
This plugin requests the following Box API scopes:
- `root_readwrite`: Required for reading and browsing files and folders in your Box account
## Data Retention
The Dify Box Datasource Plugin itself does not store your file content persistently. The plugin follows these data retention practices:
- **OAuth Tokens**: Stored securely in your Dify workspace and automatically refreshed when needed
- **File Content**: Downloaded temporarily for processing and not retained after the operation completes
- **File Metadata**: Retrieved in real-time and not cached or stored locally
- **User Information**: Retrieved during OAuth flow for display purposes only
All file data resides within your Box account and is subject to Box's data retention policies. Logs generated by Dify during plugin execution might contain operational data but are handled according to Dify's platform policies.
## Data Security
- **Encryption**: All communication with Box API is encrypted using HTTPS
- **Token Security**: OAuth tokens are stored securely in your Dify workspace
- **No Logging**: File content and sensitive metadata are not logged
- **Access Control**: The plugin only accesses files and folders that you explicitly request
- **Scope Limitation**: The plugin requests only the minimum required API scopes
## User Rights
Your rights regarding the data stored within Box (files, folders, etc.) are governed by your Box account settings and Box's policies. You can manage your data directly within Box. For data handled by the Dify platform (like stored OAuth credentials), please refer to Dify's privacy policy and user settings.
You have the right to:
- Revoke access to your Box account at any time through Box Account settings
- Delete your Box OAuth application credentials
- Request information about how your data is being used
- Control which Box API scopes are granted to the application
- Stop using the plugin at any time, which will prevent further data access
## File Access and Privacy
- **Selective Access**: The plugin only accesses files and folders that you explicitly browse or download
- **No Background Scanning**: The plugin does not scan or index your Box content automatically
- **Temporary Processing**: File content is processed only when you request a download and is not stored after processing
- **Metadata Only**: File browsing operations only retrieve metadata (names, sizes, types) without accessing file content
## Compliance
This plugin is designed to comply with:
- Box's API terms of service and privacy policy
- Dify's platform privacy and security standards
- OAuth 2.0 security best practices
## Contact Information
If you have any questions or concerns about the privacy practices of this plugin, please contact the plugin author at hello@dify.ai
## Changes to This Policy
This privacy policy may be updated from time to time. Users will be notified of any material changes through the plugin's update mechanism.
Last updated: 2025-08-28
datasources/box_datasource/README.md
+92
View File
@@ -0,0 +1,92 @@
# Box Datasource Plugin
**Author:** langgenius
**Version:** 0.1.0
**Type:** datasource
## Description
The Box Datasource Plugin is a Dify plugin that enables seamless integration with Box cloud storage. This plugin allows you to browse, search, and download files from your Box account, making it easy to access and process your cloud-stored documents within Dify applications.
## Setup
## OAuth Configuration in Box
To use this plugin, you need to configure OAuth 2.0 in your Box developer account:
### Step 1: Create a Box App
1. Go to the [Box Developer Console](https://app.box.com/developers)
2. Click **"Create New App"** at https://app.box.com/developers/console/newapp
3. Select **"Custom App"** and click **"Next"**
4. Choose **"Standard OAuth 2.0"** authentication method
5. Give your app a name (e.g., "Dify Box Integration")
6. Click **"Create App"**
<img src="_assets/box_app.png" alt="Box App Creation" width="300">
### Step 2: Configure OAuth Settings
1. In your app dashboard, go to the **"Configuration"** tab
2. Under **"OAuth 2.0 Redirect URIs"**, add your Dify callback URL:
```
https://your-dify-instance.com/console/api/oauth/plugin/langgenius/box_datasource/box/datasource/callback
```
3. Under **"Application Scopes"**, ensure the following scopes are enabled:
- `root_readwrite` (required for file operations)
4. Click **"Save Changes"**
### Step 3: Get Your Credentials
1. In the **"Configuration"** tab, find the **"OAuth 2.0 Credentials"** section
2. Copy your **Client ID** and **Client Secret**
3. These will be used in the plugin configuration
### Step 4: Authorize Your App
1. Go to the **"Authorization"** tab
2. Click **"Review and Submit"** if this is a production app
3. For development/testing, you can use the app immediately
## Usage
### Plugin Configuration
1. **Add the plugin** to your Dify workspace
2. **Configure OAuth credentials**:
- Enter your Box **Client ID**
- Enter your Box **Client Secret**
3. **Authorize the connection**:
- Click the authorization button
- You'll be redirected to Box to grant permissions
- After authorization, you'll receive access and refresh tokens
## Troubleshooting
### Common Issues
1. **Authentication Errors (401)**
- Verify your Client ID and Client Secret are correct
- Ensure your app has the required scopes
- Check if your access token has expired
2. **File Not Found Errors**
- Verify the folder path exists in your Box account
- Check file permissions in Box
- Ensure the file ID is valid
3. **Network Errors**
- Check your internet connection
- Verify Box API is accessible from your network
- Check for firewall restrictions
### Getting Help
- Check the [Box API Documentation](https://developer.box.com/)
- Review the plugin logs for detailed error information
- Ensure your Box app is properly configured with the correct redirect URIs
## License
This plugin is part of the Dify ecosystem and follows the same licensing terms as the Dify platform.
datasources/box_datasource/_assets/box_app.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 558 KiB

datasources/box_datasource/_assets/icon.svg
+3
View File
@@ -0,0 +1,3 @@
<svg width="512" height="521" viewBox="0 0 512 521" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M270.887 215.884C241.205 215.884 215.429 232.573 202.402 257.075C189.377 232.573 163.602 215.884 133.916 215.884C116.469 215.884 100.37 221.649 87.4156 231.378V166.077H87.4078C87.2289 157.661 80.3625 150.89 71.9023 150.89C63.4422 150.89 56.5773 157.661 56.3984 166.077H56.3906V294.698H56.4234C57.1125 336.913 91.5305 370.935 133.916 370.935C163.602 370.935 189.376 354.246 202.402 329.746C215.429 354.245 241.205 370.935 270.887 370.935C313.703 370.935 348.413 336.224 348.413 293.412C348.413 250.594 313.702 215.884 270.887 215.884ZM133.916 339.912C108.234 339.912 87.4156 319.094 87.4156 293.411C87.4156 267.725 108.234 246.908 133.916 246.908C159.598 246.908 180.419 267.725 180.419 293.411C180.419 319.093 159.598 339.912 133.916 339.912ZM270.887 339.912C245.206 339.912 224.387 319.094 224.387 293.411C224.387 267.725 245.205 246.908 270.887 246.908C296.569 246.908 317.391 267.725 317.391 293.411C317.391 319.093 296.569 339.912 270.887 339.912ZM452.692 346.546C457.605 353.384 456.181 362.92 449.418 368.002C442.654 373.083 433.095 371.793 427.897 365.172H427.894L393.629 319.363L359.359 365.172L359.352 365.165C354.155 371.789 344.599 373.079 337.832 367.998C331.068 362.916 329.645 353.381 334.557 346.542H334.551L374.255 293.469L334.551 240.395L334.557 240.392C329.645 233.556 331.068 224.02 337.832 218.937C344.599 213.855 354.155 215.147 359.352 221.77L359.359 221.763L393.628 267.573L427.893 221.763L427.896 221.766C433.095 215.145 442.653 213.854 449.417 218.934C456.18 224.016 457.604 233.553 452.691 240.391L452.702 240.395L412.998 293.469L452.702 346.542L452.692 346.546Z" fill="black"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.7 KiB

datasources/box_datasource/datasources/box.py
+227
View File
@@ -0,0 +1,227 @@
import logging
from collections.abc import Generator
import requests
from dify_plugin.entities.datasource import (
DatasourceMessage,
OnlineDriveBrowseFilesRequest,
OnlineDriveBrowseFilesResponse,
OnlineDriveDownloadFileRequest,
OnlineDriveFile,
OnlineDriveFileBucket,
)
from dify_plugin.interfaces.datasource.online_drive import OnlineDriveDatasource
logger = logging.getLogger(__name__)
class BoxDataSource(OnlineDriveDatasource):
_BASE_URL = "https://api.box.com/2.0"
def _browse_files(
self, request: OnlineDriveBrowseFilesRequest
) -> OnlineDriveBrowseFilesResponse:
credentials = self.runtime.credentials
bucket_name = request.bucket
prefix = request.prefix or "" # Allow empty prefix for root folder
max_keys = request.max_keys or 10
next_page_parameters = request.next_page_parameters or {}
if not credentials:
raise ValueError("Credentials not found")
access_token = credentials.get("access_token")
if not access_token:
raise ValueError("Access token not found in credentials")
# Resolve prefix to folder ID if it's a path
try:
# folder_id = self._resolve_folder_path_to_id(prefix, access_token)
# logger.debug(f"Resolved prefix '{prefix}' to folder ID: {folder_id}")
if not prefix or prefix.strip() == "":
folder_id = "0"
else:
folder_id = prefix
except Exception as e:
logger.error(f"Failed to resolve folder path '{prefix}': {str(e)}")
raise ValueError(f"Failed to resolve folder path '{prefix}': {str(e)}")
# Prepare headers for HTTP request
headers = {
"Authorization": f"Bearer {access_token}",
"Accept": "application/json"
}
# Build query parameters for Box API
params = {
"limit": max_keys,
"fields": "id,name,size,type,modified_at"
}
# Add offset for pagination if exists
if next_page_parameters and next_page_parameters.get("offset"):
params["offset"] = next_page_parameters.get("offset")
try:
# Make HTTP request to Box API
url = f"{self._BASE_URL}/folders/{folder_id}/items"
response = requests.get(url, headers=headers, params=params, timeout=30)
# Check for authentication errors
if response.status_code == 401:
raise ValueError(
"Authentication failed (401 Unauthorized). The access token may have expired. "
"Please refresh or reauthorize the connection."
)
elif response.status_code == 404:
raise ValueError(f"Folder with ID '{folder_id}' not found.")
elif response.status_code != 200:
raise ValueError(f"Failed to list files: {response.status_code} - {response.text[:200]}")
# Parse response
results = response.json()
items = results.get("entries", [])
if not items:
return OnlineDriveBrowseFilesResponse(result=[])
files = []
for item in items:
# Check if it's a folder (Box folders have type 'folder')
is_folder = item.get("type") == "folder"
file_type = "folder" if is_folder else "file"
size = 0 if is_folder else int(item.get("size", 0))
files.append(OnlineDriveFile(
id=item.get("id", ""),
name=item.get("name", ""),
size=size,
type=file_type
))
# Handle pagination - Box uses offset-based pagination
total_count = results.get("total_count", 0)
offset = results.get("offset", 0)
limit = results.get("limit", max_keys)
# Calculate if there are more items
has_more = offset + limit < total_count
next_offset = offset + limit if has_more else None
next_page_parameters = {"offset": next_offset} if next_offset else {}
is_truncated = has_more
return OnlineDriveBrowseFilesResponse(result=[
OnlineDriveFileBucket(
bucket=bucket_name,
files=files,
is_truncated=is_truncated,
next_page_parameters=next_page_parameters
)
])
except requests.exceptions.RequestException as e:
raise ValueError(f"Network error when accessing Box: {str(e)}") from e
except Exception as e:
if "401" in str(e) or "Unauthorized" in str(e):
raise ValueError(
"Authentication failed. The access token may have expired. "
"Please refresh or reauthorize the connection."
) from e
return OnlineDriveBrowseFilesResponse(result=[])
def _download_file(self, request: OnlineDriveDownloadFileRequest) -> Generator[DatasourceMessage, None, None]:
credentials = self.runtime.credentials
file_id = request.id
if not credentials:
raise ValueError("Credentials not found")
access_token = credentials.get("access_token")
if not access_token:
raise ValueError("Access token not found in credentials")
logger.debug(f"Downloading file with ID: {file_id}")
# Prepare headers for HTTP request
headers = {
"Authorization": f"Bearer {access_token}",
"Accept": "application/json"
}
try:
# First, get file metadata
metadata_url = f"{self._BASE_URL}/files/{file_id}"
metadata_params = {"fields": "id,name,size,type"}
metadata_response = requests.get(
metadata_url,
headers=headers,
params=metadata_params,
timeout=30
)
if metadata_response.status_code == 401:
logger.error(f"Authentication failed: {metadata_response.text[:200]}")
raise ValueError(
"Authentication failed (401 Unauthorized). The access token may have expired. "
"Please refresh or reauthorize the connection."
)
elif metadata_response.status_code == 404:
logger.error(f"File not found: {file_id}")
raise ValueError(f"File with ID '{file_id}' not found.")
elif metadata_response.status_code != 200:
logger.error(f"Failed to get file metadata: {metadata_response.status_code}")
raise ValueError(f"Failed to get file metadata: {metadata_response.status_code}")
file_metadata = metadata_response.json()
file_name = file_metadata.get("name", "unknown")
file_type = file_metadata.get("type", "file")
# Check if it's a folder
if file_type == "folder":
raise ValueError(f"Cannot download folder '{file_name}'. Please select a file.")
# Get file content URL
content_url = f"{self._BASE_URL}/files/{file_id}/content"
# Download file content
content_response = requests.get(
content_url,
headers=headers,
timeout=60, # Longer timeout for file downloads
stream=True # Stream the response for large files
)
if content_response.status_code == 401:
logger.error("Authentication failed during file download")
raise ValueError(
"Authentication failed during file download. "
"Please refresh or reauthorize the connection."
)
elif content_response.status_code != 200:
logger.error(f"Failed to download file: {content_response.status_code}")
raise ValueError(f"Failed to download file: {content_response.status_code}")
# Get the content
file_content = content_response.content
# Determine MIME type from file extension or use default
mime_type = self._get_mime_type_from_filename(file_name)
yield self.create_blob_message(file_content, meta={
"file_name": file_name,
"mime_type": mime_type
})
except requests.exceptions.RequestException as e:
logger.error(f"Network error: {e}")
raise ValueError(f"Network error when downloading file: {str(e)}") from e
except Exception as e:
if "already" not in str(e).lower(): # Avoid re-raising our own errors
logger.error(f"Unexpected error: {e}")
raise
def _get_mime_type_from_filename(self, filename: str) -> str:
"""Determine MIME type from file extension."""
import mimetypes
mime_type, _ = mimetypes.guess_type(filename)
return mime_type or "application/octet-stream"
datasources/box_datasource/datasources/box.yaml
+20
View File
@@ -0,0 +1,20 @@
identity:
name: box
author: langgenius
label:
en_US: Box
zh_Hans: Box
description:
en_US: Get files from your Box
zh_Hans: 从您的 Box 中获取文件
parameters:
output_schema:
type: object
properties:
file:
$ref: "https://dify.ai/schemas/v1/file.json"
extra:
python:
source: datasources/box.py
datasources/box_datasource/main.py
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import Plugin, DifyPluginEnv
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/box_datasource/manifest.yaml
+30
View File
@@ -0,0 +1,30 @@
version: 0.1.1
type: plugin
author: langgenius
name: box_datasource
label:
en_US: Box Datasource
zh_Hans: Box 数据源
description:
en_US: Box Datasource
zh_Hans: Box 数据源
icon: icon.svg
resource:
memory: 268435456
permission: {}
plugins:
datasources:
- provider/box.yaml
meta:
version: 0.0.1
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-07-31T14:54:47.062188+08:00
privacy: PRIVACY.md
verified: false
datasources/box_datasource/provider/box.py
+258
View File
@@ -0,0 +1,258 @@
from typing import Any, Mapping
import certifi
import time
import secrets
import urllib.parse
from dify_plugin.errors.tool import ToolProviderCredentialValidationError, DatasourceOAuthError
from dify_plugin.interfaces.datasource import DatasourceProvider, DatasourceOAuthCredentials
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from flask import Request
class BoxDatasourceProvider(DatasourceProvider):
_AUTH_URL = "https://account.box.com/api/oauth2/authorize"
_TOKEN_URL = "https://api.box.com/oauth2/token"
_API_BASE_URL = "https://api.box.com/2.0"
_REQUIRED_SCOPES = "root_readwrite" # Required for complete file operations
def _get_requests_session(self) -> requests.Session:
session = requests.Session()
retry_strategy = Retry(
total=3,
backoff_factor=1,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE", "POST"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
session.mount("http://", adapter)
session.mount("https://", adapter)
session.verify = certifi.where()
return session
# def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
# try:
# access_token = credentials.get("access_token")
# if not access_token:
# raise ToolProviderCredentialValidationError("Access token is required.")
# headers = {
# "Authorization": f"Bearer {access_token}",
# "Content-Type": "application/json",
# }
# session = self._get_requests_session()
# # Test API call to get user information
# response = session.get(f"{self._API_BASE_URL}/users/me", headers=headers, timeout=30)
# if response.status_code == 401:
# raise ToolProviderCredentialValidationError(
# "Access token is invalid or expired. Please refresh or re-authorize."
# )
# elif response.status_code != 200:
# raise ToolProviderCredentialValidationError(
# f"Failed to validate credentials: {response.status_code} {response.text}"
# )
# return None
# except requests.RequestException as e:
# raise ToolProviderCredentialValidationError(
# f"Network error when validating Box credentials: {str(e)}"
# )
# except Exception as e:
# raise ToolProviderCredentialValidationError(str(e))
def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str:
"""
Generate the authorization URL for Box OAuth 2.0.
Args:
redirect_uri: The redirect URI after authorization
system_credentials: System credentials containing client_id and client_secret
Returns:
Authorization URL string
"""
state = secrets.token_urlsafe(32)
params = {
"client_id": system_credentials["client_id"],
"redirect_uri": redirect_uri,
"response_type": "code",
"scope": self._REQUIRED_SCOPES,
"state": state,
}
return f"{self._AUTH_URL}?{urllib.parse.urlencode(params)}"
def _oauth_get_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request
) -> DatasourceOAuthCredentials:
"""
Exchange authorization code for access token.
Args:
redirect_uri: The redirect URI
system_credentials: System credentials containing client_id and client_secret
request: The request object containing the authorization code
Returns:
DatasourceOAuthCredentials object containing access and refresh tokens
"""
code = request.args.get("code")
if not code:
raise DatasourceOAuthError("Authorization code not provided")
# Validate state parameter for security
state = request.args.get("state")
if not state:
raise DatasourceOAuthError("State parameter missing")
data = {
"grant_type": "authorization_code",
"code": code,
"client_id": system_credentials["client_id"],
"client_secret": system_credentials["client_secret"],
"redirect_uri": redirect_uri,
}
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "application/json"
}
session = self._get_requests_session()
try:
response = session.post(self._TOKEN_URL, data=data, headers=headers, timeout=30)
response.raise_for_status()
response_data = response.json()
access_token = response_data.get("access_token")
refresh_token = response_data.get("refresh_token")
expires_in = int(time.time()) + response_data.get("expires_in", 3600)
if not access_token:
raise DatasourceOAuthError(f"Failed to obtain access token: {response_data}")
# Get user information
userinfo_headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
}
try:
userinfo_response = session.get(f"{self._API_BASE_URL}/users/me", headers=userinfo_headers, timeout=30)
userinfo_response.raise_for_status()
userinfo_json = userinfo_response.json()
except requests.exceptions.RequestException as e:
raise DatasourceOAuthError(f"Failed to get user information: {str(e)}")
user_name = userinfo_json.get("name")
user_email = userinfo_json.get("login")
user_picture = None # Box API doesn't provide profile picture in user info
return DatasourceOAuthCredentials(
name=user_name or user_email,
avatar_url=user_picture,
expires_at=expires_in,
credentials={
"access_token": access_token,
"refresh_token": refresh_token,
"token_type": response_data.get("token_type", "bearer")
},
)
except requests.RequestException as e:
raise DatasourceOAuthError(f"Failed to exchange code for token: {str(e)}")
def _oauth_refresh_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], credentials: Mapping[str, Any]
) -> DatasourceOAuthCredentials:
"""
Refresh the access token using refresh token.
Args:
redirect_uri: The redirect URI
system_credentials: System credentials containing client_id and client_secret
credentials: Current credentials containing refresh_token
Returns:
DatasourceOAuthCredentials object with new access token
"""
refresh_token = credentials.get("refresh_token")
if not refresh_token:
raise DatasourceOAuthError("Refresh token not available")
data = {
"grant_type": "refresh_token",
"refresh_token": refresh_token,
"client_id": system_credentials["client_id"],
"client_secret": system_credentials["client_secret"],
}
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "application/json"
}
session = self._get_requests_session()
try:
response = session.post(self._TOKEN_URL, data=data, headers=headers, timeout=30)
response.raise_for_status()
response_data = response.json()
access_token = response_data.get("access_token")
new_refresh_token = response_data.get("refresh_token", refresh_token)
expires_in = int(time.time()) + response_data.get("expires_in", 3600)
if not access_token:
raise DatasourceOAuthError(f"Failed to refresh access token: {response_data}")
# Get user information
userinfo_headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
}
try:
userinfo_response = session.get(f"{self._API_BASE_URL}/users/me", headers=userinfo_headers, timeout=30)
userinfo_response.raise_for_status()
userinfo_json = userinfo_response.json()
except requests.exceptions.RequestException as e:
raise DatasourceOAuthError(f"Failed to get user information: {str(e)}")
user_name = userinfo_json.get("name")
user_email = userinfo_json.get("login")
user_picture = None # Box API doesn't provide profile picture in user info
updated_credentials = {
"access_token": access_token,
"refresh_token": new_refresh_token,
"token_type": response_data.get("token_type", "bearer"),
"client_id": system_credentials.get("client_id") or credentials.get("client_id"),
"client_secret": system_credentials.get("client_secret") or credentials.get("client_secret"),
"user_email": user_email,
}
return DatasourceOAuthCredentials(
name=user_name or user_email,
avatar_url=user_picture,
expires_at=expires_in,
credentials=updated_credentials,
)
except requests.exceptions.SSLError as e:
raise DatasourceOAuthError(
f"SSL error when refreshing token. This might be due to network proxy or firewall settings: {str(e)}"
)
except requests.exceptions.ConnectionError as e:
raise DatasourceOAuthError(
f"Connection error when refreshing token. Please check your network connection: {str(e)}"
)
except requests.exceptions.Timeout as e:
raise DatasourceOAuthError(
f"Timeout when refreshing token. The Box OAuth server might be slow or unreachable: {str(e)}"
)
except requests.RequestException as e:
raise DatasourceOAuthError(f"Failed to refresh token: {str(e)}")
datasources/box_datasource/provider/box.yaml
+50
View File
@@ -0,0 +1,50 @@
identity:
author: langgenius
name: box
label:
en_US: Box
zh_Hans: Box
description:
en_US: Box Datasource
zh_Hans: Box 数据源
icon: icon.svg
provider_type: online_drive
help:
title:
en_US: Get your credentials from Box OAuth
zh_Hans: 从 Box OAuth 获取您的凭证
url:
en_US: https://app.box.com/developers
zh_Hans: https://app.box.com/developers
oauth_schema:
client_schema:
- name: client_id
type: secret-input
label:
zh_Hans: Client ID
en_US: Client ID
- name: client_secret
type: secret-input
label:
zh_Hans: Client Secret
en_US: Client Secret
help:
zh_Hans: 从 Box OAuth 集成页面获取您的接口令牌
en_US: Get your integration token from the Box OAuth Integrations page
url: https://app.box.com/developers
credentials_schema:
- name: access_token
type: secret-input
label:
zh_Hans: Access Token
en_US: Access Token
- name: refresh_token
type: secret-input
label:
zh_Hans: Refresh Token
en_US: Refresh Token
datasources:
- datasources/box.yaml
extra:
python:
source: provider/box.py
datasources/box_datasource/requirements.txt
+4
View File
@@ -0,0 +1,4 @@
# This file is @generated by PDM.
# Please do not edit it manually.
dify-plugin==0.5.0b14
datasources/brightdata_datasource/.difyignore
+180
View File
@@ -0,0 +1,180 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
# Git
.git/
.gitignore
# Mac
.DS_Store
# Windows
Thumbs.db
*.difypkg
datasources/brightdata_datasource/.env.example
+4
View File
@@ -0,0 +1,4 @@
INSTALL_METHOD=remote
REMOTE_INSTALL_URL=debug.dify.ai
REMOTE_INSTALL_PORT=5003
REMOTE_INSTALL_KEY=********-****-****-****-************
datasources/brightdata_datasource/.gitignore
+175
View File
@@ -0,0 +1,175 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
#uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
*.difypkg
test.py
datasources/brightdata_datasource/GUIDE.md
+149
View File
@@ -0,0 +1,149 @@
## User Guide of how to develop a Dify Plugin
Hi there, looks like you have already created a Plugin, now let's get you started with the development!
### Choose a Plugin type you want to develop
Before start, you need some basic knowledge about the Plugin types, Plugin supports to extend the following abilities in Dify:
- **Tool**: Tool Providers like Google Search, Stable Diffusion, etc. it can be used to perform a specific task.
- **Model**: Model Providers like OpenAI, Anthropic, etc. you can use their models to enhance the AI capabilities.
- **Endpoint**: Like Service API in Dify and Ingress in Kubernetes, you can extend a http service as an endpoint and control its logics using your own code.
Based on the ability you want to extend, we have divided the Plugin into three types: **Tool**, **Model**, and **Extension**.
- **Tool**: It's a tool provider, but not only limited to tools, you can implement an endpoint there, for example, you need both `Sending Message` and `Receiving Message` if you are building a Discord Bot, **Tool** and **Endpoint** are both required.
- **Model**: Just a model provider, extending others is not allowed.
- **Extension**: Other times, you may only need a simple http service to extend the functionalities, **Extension** is the right choice for you.
I believe you have chosen the right type for your Plugin while creating it, if not, you can change it later by modifying the `manifest.yaml` file.
### Manifest
Now you can edit the `manifest.yaml` file to describe your Plugin, here is the basic structure of it:
- version(version, required)：Plugin's version
- type(type, required)：Plugin's type, currently only supports `plugin`, future support `bundle`
- author(string, required)：Author, it's the organization name in Marketplace and should also equals to the owner of the repository
- label(label, required)：Multi-language name
- created_at(RFC3339, required)：Creation time, Marketplace requires that the creation time must be less than the current time
- icon(asset, required)：Icon path
- resource (object)：Resources to be applied
- memory (int64)：Maximum memory usage, mainly related to resource application on SaaS for serverless, unit bytes
- permission(object)：Permission application
- tool(object)：Reverse call tool permission
- enabled (bool)
- model(object)：Reverse call model permission
- enabled(bool)
- llm(bool)
- text_embedding(bool)
- rerank(bool)
- tts(bool)
- speech2text(bool)
- moderation(bool)
- node(object)：Reverse call node permission
- enabled(bool)
- endpoint(object)：Allow to register endpoint permission
- enabled(bool)
- app(object)：Reverse call app permission
- enabled(bool)
- storage(object)：Apply for persistent storage permission
- enabled(bool)
- size(int64)：Maximum allowed persistent memory, unit bytes
- plugins(object, required)：Plugin extension specific ability yaml file list, absolute path in the plugin package, if you need to extend the model, you need to define a file like openai.yaml, and fill in the path here, and the file on the path must exist, otherwise the packaging will fail.
- Format
- tools(list[string]): Extended tool suppliers, as for the detailed format, please refer to [Tool Guide](https://docs.dify.ai/plugins/schema-definition/tool)
- models(list[string])：Extended model suppliers, as for the detailed format, please refer to [Model Guide](https://docs.dify.ai/plugins/schema-definition/model)
- endpoints(list[string])：Extended Endpoints suppliers, as for the detailed format, please refer to [Endpoint Guide](https://docs.dify.ai/plugins/schema-definition/endpoint)
- Restrictions
- Not allowed to extend both tools and models
- Not allowed to have no extension
- Not allowed to extend both models and endpoints
- Currently only supports up to one supplier of each type of extension
- meta(object)
- version(version, required)：manifest format version, initial version 0.0.1
- arch(list[string], required)：Supported architectures, currently only supports amd64 arm64
- runner(object, required)：Runtime configuration
- language(string)：Currently only supports python
- version(string)：Language version, currently only supports 3.12
- entrypoint(string)：Program entry, in python it should be main
### Install Dependencies
- First of all, you need a Python 3.11+ environment, as our SDK requires that.
- Then, install the dependencies:
```bash
pip install -r requirements.txt
```
- If you want to add more dependencies, you can add them to the `requirements.txt` file, once you have set the runner to python in the `manifest.yaml` file, `requirements.txt` will be automatically generated and used for packaging and deployment.
### Implement the Plugin
Now you can start to implement your Plugin, by following these examples, you can quickly understand how to implement your own Plugin:
- [OpenAI](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/openai): best practice for model provider
- [Google Search](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/google): a simple example for tool provider
- [Neko](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/neko): a funny example for endpoint group
### Test and Debug the Plugin
You may already noticed that a `.env.example` file in the root directory of your Plugin, just copy it to `.env` and fill in the corresponding values, there are some environment variables you need to set if you want to debug your Plugin locally.
- `INSTALL_METHOD`: Set this to `remote`, your plugin will connect to a Dify instance through the network.
- `REMOTE_INSTALL_HOST`: The host of your Dify instance, you can use our SaaS instance `https://debug.dify.ai`, or self-hosted Dify instance.
- `REMOTE_INSTALL_PORT`: The port of your Dify instance, default is 5003
- `REMOTE_INSTALL_KEY`: You should get your debugging key from the Dify instance you used, at the right top of the plugin management page, you can see a button with a `debug` icon, click it and you will get the key.
Run the following command to start your Plugin:
```bash
python -m main
```
Refresh the page of your Dify instance, you should be able to see your Plugin in the list now, but it will be marked as `debugging`, you can use it normally, but not recommended for production.
### Publish and Update the Plugin
To streamline your plugin update workflow, you can configure GitHub Actions to automatically create PRs to the Dify plugin repository whenever you create a release.
##### Prerequisites
- Your plugin source repository
- A fork of the dify-plugins repository
- Proper plugin directory structure in your fork
#### Configure GitHub Action
1. Create a Personal Access Token with write permissions to your forked repository
2. Add it as a secret named `PLUGIN_ACTION` in your source repository settings
3. Create a workflow file at `.github/workflows/plugin-publish.yml`
#### Usage
1. Update your code and the version in your `manifest.yaml`
2. Create a release in your source repository
3. The action automatically packages your plugin and creates a PR to your forked repository
#### Benefits
- Eliminates manual packaging and PR creation steps
- Ensures consistency in your release process
- Saves time during frequent updates
---
For detailed setup instructions and example configuration, visit: [GitHub Actions Workflow Documentation](https://docs.dify.ai/plugins/publish-plugins/plugin-auto-publish-pr)
### Package the Plugin
After all, just package your Plugin by running the following command:
```bash
dify-plugin plugin package ./ROOT_DIRECTORY_OF_YOUR_PLUGIN
```
you will get a `plugin.difypkg` file, that's all, you can submit it to the Marketplace now, look forward to your Plugin being listed!
## User Privacy Policy
Please fill in the privacy policy of the plugin if you want to make it published on the Marketplace, refer to [PRIVACY.md](PRIVACY.md) for more details.
datasources/brightdata_datasource/PRIVACY.md
+96
View File
@@ -0,0 +1,96 @@
# Privacy Policy
## Overview
This privacy policy explains how the Dify platform handles data when using the Bright Data Datasource plugin. We are committed to protecting your privacy and ensuring transparency about data collection and usage.
## Data Collection and Storage
### What We Store
**API Key Only**: The Dify platform stores only your Bright Data API key to enable the functionality of this plugin. This is the minimal data required for the plugin to operate.
### What We Do NOT Store
The Dify platform does not store any of the following user data when using this plugin:
- **Query Content**: Your actual search queries and data requests are not stored
- **Response Data**: Data retrieved from Bright Data services is not stored on Dify servers
- **User Information**: Personal information, account details, or usage patterns are not collected
- **Metadata**: Search history, timestamps, or other usage metadata are not retained
- **Configuration**: Plugin settings and preferences are not stored
## Data Processing
### Real-time Processing
All data processing occurs in real-time:
- Your queries are sent directly to Bright Data services
- Responses are returned to you immediately
- No intermediate storage or caching of data occurs
### Data Flow
1. **Query Submission**: Your query is sent to Bright Data via their API
2. **Processing**: Bright Data processes your request using their services
3. **Response**: Results are returned directly to you through the plugin
4. **Cleanup**: No data remains on Dify servers after the request is complete
## Security
### API Key Protection
Your Bright Data API key is:
- Encrypted when stored
- Used only for legitimate plugin operations
- Never shared with third parties
- Accessible only to authorized Dify services
### Data Transmission
All data transmission is secured using:
- HTTPS encryption
- Secure API endpoints
- Industry-standard security protocols
## Third-Party Services
### Bright Data
This plugin integrates with Bright Data services. Please refer to [Bright Data's Privacy Policy](https://brightdata.com/privacy-policy) for information about how they handle your data.
### No Additional Third Parties
Dify does not share your data with any other third-party services beyond what is necessary for the Bright Data integration.
## Your Rights
You have the right to:
- **Access**: Request information about what data is stored
- **Deletion**: Remove your API key from the platform
- **Control**: Manage your API key settings at any time
- **Transparency**: Understand exactly how your data is used
## Data Retention
- **API Key**: Stored until you remove it or delete your account
- **Usage Data**: Not stored or retained
- **Query Data**: Not stored or retained
## Contact Information
If you have questions about this privacy policy or how your data is handled, please contact us through the Dify platform support channels.
## Updates to This Policy
This privacy policy may be updated from time to time. We will notify users of any material changes through the Dify platform.
## Last Updated
This privacy policy was last updated on the date of plugin publication.
---
**Note**: This privacy policy applies specifically to the Bright Data Datasource plugin for Dify. For general Dify platform privacy information, please refer to the main Dify privacy policy.
Last Updated: 2025-08-27
datasources/brightdata_datasource/README.md
+50
View File
@@ -0,0 +1,50 @@
## brightdata_datasource
**Author:** langgenius
**Version:** 0.1.3
**Type:** datasource
# Bright Data Web Scraper
A Dify plugin that enables enterprise-grade web scraping with Bright Data's Unlocker API. Extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
## Features
- **Universal Web Access**: Scrape any webpage, even those with bot detection or CAPTCHA
- **Multiple Formats**: Output in Markdown or HTML format
- **PDF Support**: Extract content from PDF files
- **Enterprise-Grade**: Powered by Bright Data's reliable infrastructure
## Configuration
### Prerequisites
1. **Bright Data Account**: Sign up at [Bright Data](https://brightdata.com)
2. **Unlocker API Zone**: Create a new zone in your Bright Data dashboard at https://brightdata.com/cp/zones/new
3. **API Token**: Generate an API token from your user settings
### Setup
1. Install the plugin in your Dify workspace
2. Configure the following credentials:
- **Unlocker API Zone**: Your Bright Data zone identifier
- **Bright Data API Token**: Your API authentication token
## Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `url` | string | Yes | The full URL of the webpage to scrape |
| `format` | select | Yes | Output format: `markdown` or `html` (default: markdown) |
### Output Schema
The plugin returns an array of objects containing:
- `source_url`: The original URL that was scraped
- `content`: The extracted content in the specified format
- `title`: The title of the webpage
- `description`: A description of the content
## Usage
Simply provide a URL and the plugin will return the scraped content in your preferred format, ready for use with LLMs or other applications.
datasources/brightdata_datasource/_assets/icon.svg
+5
View File
@@ -0,0 +1,5 @@
<svg width="512" height="512" viewBox="0 0 512 512" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M0 63.2099C0 28.3 28.3 0 63.2099 0H448.79C483.7 0 512 28.3 512 63.2099V448.79C512 483.7 483.7 512 448.79 512H63.2099C28.3 512 0 483.7 0 448.79V63.2099Z" fill="#3D7FFC"/>
<path d="M220.363 115.876C225.342 125.336 222.853 133.302 219.368 141.766C213.891 155.706 215.385 169.148 225.342 180.599C235.299 192.548 249.24 194.042 263.678 191.055C283.593 186.574 295.044 167.655 292.056 146.744C290.563 136.787 284.588 126.83 290.065 115.876C285.086 118.366 285.086 118.366 279.112 128.323C273.635 121.851 268.657 114.881 262.682 107.911C257.704 102.434 252.725 97.4553 255.712 88.4937C249.737 91.4809 246.75 95.4638 244.759 100.442C243.265 105.421 242.269 110.4 241.274 115.379C240.278 120.357 237.789 124.838 232.81 127.825C229.823 122.349 226.836 118.366 220.363 115.876Z" fill="white"/>
<path d="M208.593 422.618V398.267L234.975 394.837V255.594L210.649 247.706V220.612L277.117 211.695L291.165 210.306V219.583V394.494L317.204 398.267V422.618H208.593Z" fill="white"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

datasources/brightdata_datasource/datasources/brightdata.py
+81
View File
@@ -0,0 +1,81 @@
import time
from typing import Any, Generator
import requests
from dify_plugin.entities.datasource import WebSiteInfo, WebSiteInfoDetail
from dify_plugin.entities.tool import ToolInvokeMessage
from dify_plugin.interfaces.datasource.website import WebsiteCrawlDatasource
class BrightdataProvider(WebsiteCrawlDatasource):
def _get_website_crawl(
self, datasource_parameters: dict[str, Any]
) -> Generator[ToolInvokeMessage, None, None]:
try:
api_token = self.runtime.credentials["api_token"]
except KeyError:
raise Exception("Bright Data API token is required.")
url = datasource_parameters.get("url", "").strip()
if not url:
raise Exception("URL cannot be empty.")
try:
crawl_res = WebSiteInfo(web_info_list=[], status="", total=0, completed=0)
crawl_res.status = "processing"
yield self.create_crawl_message(crawl_res)
format = datasource_parameters.get("format", "markdown")
markdown_content = self._scrape_as_markdown(url, api_token, format)
crawl_res.status = "completed"
crawl_res.web_info_list = [
WebSiteInfoDetail(
title=url, # Any Better Ways?
source_url=url,
description=markdown_content,
content=markdown_content,
)
]
crawl_res.total = 1
crawl_res.completed = 1
yield self.create_crawl_message(crawl_res)
except Exception as e:
raise Exception(f"Web scraping failed: {str(e)}")
def _scrape_as_markdown(self, url: str, api_token: str, format: str) -> str:
"""Use exact same API call as Bright Data MCP server"""
headers = {
"user-agent": "dify-plugin/1.0.0", # todo check
"authorization": f"Bearer {api_token}",
"content-type": "application/json",
}
# Use the same zone as MCP server (hardcoded default)
payload = {
"url": url,
"zone": "dify_plugin",
"format": "raw",
"data_format": format,
}
try:
response = requests.post(
"https://api.brightdata.com/request",
json=payload,
headers=headers,
timeout=180,
)
if response.status_code != 200:
raise Exception(
f"Bright Data API error {response.status_code}: {response.text}"
)
return response.text
except requests.exceptions.RequestException as e:
raise Exception(f"Request failed: {str(e)}")
datasources/brightdata_datasource/datasources/brightdata.yaml
+76
View File
@@ -0,0 +1,76 @@
identity:
name: brightdata
author: langgenius
label:
en_US: Bright Data Web Scraper
zh_Hans: Bright Data 网页抓取器
pt_BR: Bright Data Web Scraper
description:
en_US: Scrape a single webpage URL and get back the results in MarkDown format. This tool can unlock any webpage even if it uses bot detection or CAPTCHA.
zh_Hans: 抓取单个网页URL并以MarkDown语言返回结果。此工具可以解锁任何网页，即使它使用机器人检测或验证码。
pt_BR: Scrape a single webpage URL and get back the results in MarkDown format. This tool can unlock any webpage even if it uses bot detection or CAPTCHA.
icon: icon.svg
description:
en_US: Fetch the target URL (can be a PDF) and convert it into a LLM-friendly
markdown.
pt_BR: Busque a URL de destino (que pode ser um PDF) e converta em um Markdown
LLM-friendly.
zh_Hans: 获取目标网址（可以是 PDF），并将其转换为适合大模型处理的 Markdown 格式。
parameters:
- name: url
type: string
required: true
label:
en_US: Website URL
zh_Hans: 网站 URL
pt_BR: Website URL
description:
en_US: The full URL of the webpage to scrape
zh_Hans: 要抓取的网页的完整 URL
pt_BR: The full URL of the webpage to scrape
form: llm
- name: format
type: select
required: true
label:
en_US: Format
zh_Hans: 格式
pt_BR: Formato
description:
en_US: The format of the output
zh_Hans: 输出的格式
pt_BR: O formato da saída
form: llm
default: markdown
options:
- label:
en_US: Markdown
zh_Hans: Markdown
pt_BR: Markdown
value: markdown
- label:
en_US: HTML
zh_Hans: HTML
pt_BR: HTML
value: html
output_schema:
type: object
properties:
source_url:
type: string
description: the source url of the file
content:
type: string
description: the content extracted from the file
title:
type: string
description: the title of the file
description:
type: string
description: the description of the file
extra:
python:
source: datasources/brightdata.py
provider_type: website_crawl
datasources/brightdata_datasource/main.py
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import DifyPluginEnv, Plugin
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/brightdata_datasource/manifest.yaml
+43
View File
@@ -0,0 +1,43 @@
version: 0.1.3
type: plugin
author: langgenius
name: brightdata_datasource
label:
en_US: Bright Data Web Scraper
ja_JP: Bright Data Web Scraper
zh_Hans: Bright Data Web Scraper
pt_BR: Bright Data Web Scraper
description:
en_US: Unlock the entire web with enterprise-grade scraping - extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
ja_JP: Unlock the entire web with enterprise-grade scraping - extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
zh_Hans: Unlock the entire web with enterprise-grade scraping - extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
pt_BR: Unlock the entire web with enterprise-grade scraping - extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
icon: icon.svg
resource:
memory: 268435456
permission:
model:
enabled: true
llm: false
text_embedding: false
rerank: false
tts: false
speech2text: false
moderation: false
plugins:
datasources:
- provider/brightdata.yaml
meta:
version: 0.0.1
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-05-18T14:29:38.188882+08:00
privacy: PRIVACY.md
verified: false
provider_type: website_crawl
datasources/brightdata_datasource/provider/brightdata.py
+26
View File
@@ -0,0 +1,26 @@
from typing import Any
from dify_plugin.errors.tool import ToolProviderCredentialValidationError
from dify_plugin.interfaces.datasource import DatasourceProvider
class BrightdataProvider(DatasourceProvider):
def _validate_credentials(self, credentials: dict[str, Any]) -> None:
api_token = credentials.get("api_token")
# Basic validation only - no network calls
if not api_token:
raise ToolProviderCredentialValidationError(
"Bright Data API token is required."
)
# Check if token has basic expected format
api_token = str(api_token).strip()
if len(api_token) < 10:
raise ToolProviderCredentialValidationError(
"API token appears too short to be valid."
)
pass
datasources/brightdata_datasource/provider/brightdata.yaml
+52
View File
@@ -0,0 +1,52 @@
identity:
author: langgenius
name: brightdata
label:
en_US: Bright Data Web Scraper
zh_Hans: Bright Data 网页抓取器
pt_BR: Bright Data Web Scraper
description:
en_US: Unlock the entire web with enterprise-grade scraping - extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
zh_Hans: 使用企业级网络抓取解锁整个网络 - 从任何网站提取数据，绕过反机器人保护，并访问来自 50 多个平台（包括 Amazon、LinkedIn、Instagram 等）的结构化数据。
pt_BR: Unlock the entire web with enterprise-grade scraping - extract data from any website, bypass anti-bot protection, and access structured data from 50+ platforms including Amazon, LinkedIn, Instagram, and more.
icon: icon.svg
credentials_schema:
- name: zone
type: text-input
required: true
label:
en_US: Unlocker API Zone
zh_Hans: Unlocker API 区域
pt_BR: Unlocker API Zona
placeholder:
en_US: Enter your Unlocker API Zone
zh_Hans: 请输入您的 Unlocker API Zone
pt_BR: Enter your Unlocker API Zone
help:
en_US: Get your Unlocker API Zone from Bright Data user settings.
zh_Hans: 从 Bright Data 用户设置中获取您的 Unlocker API 区域。
pt_BR: Get your Unlocker API Zone from Bright Data user settings.
url: https://brightdata.com/cp/zones/new
- name: api_token
type: secret-input
required: true
label:
en_US: Bright Data API Token
zh_Hans: Bright Data API 令牌
pt_BR: Bright Data API Token
placeholder:
en_US: Enter your Bright Data API token
zh_Hans: 请输入您的 Bright Data API 令牌
pt_BR: Enter your Bright Data API token
help:
en_US: Get your API token from Bright Data user settings.
zh_Hans: 从 Bright Data 用户设置中获取您的 API 令牌。令牌应以您的区域凭据开头。
pt_BR: Get your API token from Bright Data user settings.
url: https://brightdata.com/cp/setting/users
extra:
python:
source: provider/brightdata.py
datasources:
- datasources/brightdata.yaml
provider_type: website_crawl
datasources/brightdata_datasource/requirements.txt
+4
View File
@@ -0,0 +1,4 @@
# This file is @generated by PDM.
# Please do not edit it manually.
dify-plugin==0.5.0b14
datasources/confluence_datasource/.difyignore
+179
View File
@@ -0,0 +1,179 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
# Git
.git/
.gitignore
.github/
# Mac
.DS_Store
# Windows
Thumbs.db
datasources/confluence_datasource/.env.example
+4
View File
@@ -0,0 +1,4 @@
INSTALL_METHOD=remote
REMOTE_INSTALL_HOST=debug.dify.ai
REMOTE_INSTALL_PORT=5003
REMOTE_INSTALL_KEY=********-****-****-****-************
datasources/confluence_datasource/GUIDE.md
+149
View File
@@ -0,0 +1,149 @@
## User Guide of how to develop a Dify Plugin
Hi there, looks like you have already created a Plugin, now let's get you started with the development!
### Choose a Plugin type you want to develop
Before start, you need some basic knowledge about the Plugin types, Plugin supports to extend the following abilities in Dify:
- **Tool**: Tool Providers like Google Search, Stable Diffusion, etc. it can be used to perform a specific task.
- **Model**: Model Providers like OpenAI, Anthropic, etc. you can use their models to enhance the AI capabilities.
- **Endpoint**: Like Service API in Dify and Ingress in Kubernetes, you can extend a http service as an endpoint and control its logics using your own code.
Based on the ability you want to extend, we have divided the Plugin into three types: **Tool**, **Model**, and **Extension**.
- **Tool**: It's a tool provider, but not only limited to tools, you can implement an endpoint there, for example, you need both `Sending Message` and `Receiving Message` if you are building a Discord Bot, **Tool** and **Endpoint** are both required.
- **Model**: Just a model provider, extending others is not allowed.
- **Extension**: Other times, you may only need a simple http service to extend the functionalities, **Extension** is the right choice for you.
I believe you have chosen the right type for your Plugin while creating it, if not, you can change it later by modifying the `manifest.yaml` file.
### Manifest
Now you can edit the `manifest.yaml` file to describe your Plugin, here is the basic structure of it:
- version(version, required)：Plugin's version
- type(type, required)：Plugin's type, currently only supports `plugin`, future support `bundle`
- author(string, required)：Author, it's the organization name in Marketplace and should also equals to the owner of the repository
- label(label, required)：Multi-language name
- created_at(RFC3339, required)：Creation time, Marketplace requires that the creation time must be less than the current time
- icon(asset, required)：Icon path
- resource (object)：Resources to be applied
- memory (int64)：Maximum memory usage, mainly related to resource application on SaaS for serverless, unit bytes
- permission(object)：Permission application
- tool(object)：Reverse call tool permission
- enabled (bool)
- model(object)：Reverse call model permission
- enabled(bool)
- llm(bool)
- text_embedding(bool)
- rerank(bool)
- tts(bool)
- speech2text(bool)
- moderation(bool)
- node(object)：Reverse call node permission
- enabled(bool)
- endpoint(object)：Allow to register endpoint permission
- enabled(bool)
- app(object)：Reverse call app permission
- enabled(bool)
- storage(object)：Apply for persistent storage permission
- enabled(bool)
- size(int64)：Maximum allowed persistent memory, unit bytes
- plugins(object, required)：Plugin extension specific ability yaml file list, absolute path in the plugin package, if you need to extend the model, you need to define a file like openai.yaml, and fill in the path here, and the file on the path must exist, otherwise the packaging will fail.
- Format
- tools(list[string]): Extended tool suppliers, as for the detailed format, please refer to [Tool Guide](https://docs.dify.ai/plugins/schema-definition/tool)
- models(list[string])：Extended model suppliers, as for the detailed format, please refer to [Model Guide](https://docs.dify.ai/plugins/schema-definition/model)
- endpoints(list[string])：Extended Endpoints suppliers, as for the detailed format, please refer to [Endpoint Guide](https://docs.dify.ai/plugins/schema-definition/endpoint)
- Restrictions
- Not allowed to extend both tools and models
- Not allowed to have no extension
- Not allowed to extend both models and endpoints
- Currently only supports up to one supplier of each type of extension
- meta(object)
- version(version, required)：manifest format version, initial version 0.0.1
- arch(list[string], required)：Supported architectures, currently only supports amd64 arm64
- runner(object, required)：Runtime configuration
- language(string)：Currently only supports python
- version(string)：Language version, currently only supports 3.12
- entrypoint(string)：Program entry, in python it should be main
### Install Dependencies
- First of all, you need a Python 3.11+ environment, as our SDK requires that.
- Then, install the dependencies:
```bash
pip install -r requirements.txt
```
- If you want to add more dependencies, you can add them to the `requirements.txt` file, once you have set the runner to python in the `manifest.yaml` file, `requirements.txt` will be automatically generated and used for packaging and deployment.
### Implement the Plugin
Now you can start to implement your Plugin, by following these examples, you can quickly understand how to implement your own Plugin:
- [OpenAI](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/openai): best practice for model provider
- [Google Search](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/google): a simple example for tool provider
- [Neko](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/neko): a funny example for endpoint group
### Test and Debug the Plugin
You may already noticed that a `.env.example` file in the root directory of your Plugin, just copy it to `.env` and fill in the corresponding values, there are some environment variables you need to set if you want to debug your Plugin locally.
- `INSTALL_METHOD`: Set this to `remote`, your plugin will connect to a Dify instance through the network.
- `REMOTE_INSTALL_HOST`: The host of your Dify instance, you can use our SaaS instance `https://debug.dify.ai`, or self-hosted Dify instance.
- `REMOTE_INSTALL_PORT`: The port of your Dify instance, default is 5003
- `REMOTE_INSTALL_KEY`: You should get your debugging key from the Dify instance you used, at the right top of the plugin management page, you can see a button with a `debug` icon, click it and you will get the key.
Run the following command to start your Plugin:
```bash
python -m main
```
Refresh the page of your Dify instance, you should be able to see your Plugin in the list now, but it will be marked as `debugging`, you can use it normally, but not recommended for production.
### Publish and Update the Plugin
To streamline your plugin update workflow, you can configure GitHub Actions to automatically create PRs to the Dify plugin repository whenever you create a release.
##### Prerequisites
- Your plugin source repository
- A fork of the dify-plugins repository
- Proper plugin directory structure in your fork
#### Configure GitHub Action
1. Create a Personal Access Token with write permissions to your forked repository
2. Add it as a secret named `PLUGIN_ACTION` in your source repository settings
3. Create a workflow file at `.github/workflows/plugin-publish.yml`
#### Usage
1. Update your code and the version in your `manifest.yaml`
2. Create a release in your source repository
3. The action automatically packages your plugin and creates a PR to your forked repository
#### Benefits
- Eliminates manual packaging and PR creation steps
- Ensures consistency in your release process
- Saves time during frequent updates
---
For detailed setup instructions and example configuration, visit: [GitHub Actions Workflow Documentation](https://docs.dify.ai/plugins/publish-plugins/plugin-auto-publish-pr)
### Package the Plugin
After all, just package your Plugin by running the following command:
```bash
dify-plugin plugin package ./ROOT_DIRECTORY_OF_YOUR_PLUGIN
```
you will get a `plugin.difypkg` file, that's all, you can submit it to the Marketplace now, look forward to your Plugin being listed!
## User Privacy Policy
Please fill in the privacy policy of the plugin if you want to make it published on the Marketplace, refer to [PRIVACY.md](PRIVACY.md) for more details.
datasources/confluence_datasource/PRIVACY.md
+3
View File
@@ -0,0 +1,3 @@
## Privacy
!!! Please fill in the privacy policy of the plugin.
datasources/confluence_datasource/README.md
+10
View File
@@ -0,0 +1,10 @@
## Confluence
**Author:** langgenius
**Version:** 0.0.1
**Type:** datasource
### Description
datasources/confluence_datasource/_assets/icon.svg
+9
View File
@@ -0,0 +1,9 @@
<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="40" height="40" fill="#1868DB"/>
<mask id="mask0_15278_70070" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="9" y="10" width="22" height="20">
<path d="M30.2674 10.1978H9.6355V29.9227H30.2674V10.1978Z" fill="white"/>
</mask>
<g mask="url(#mask0_15278_70070)">
<path d="M28.9497 23.8166C22.7775 20.8321 20.9744 20.3862 18.3737 20.3862C15.3223 20.3862 12.7216 21.6554 10.3984 25.223L10.0169 25.8062C9.70486 26.2865 9.6355 26.458 9.6355 26.6638C9.6355 26.8697 9.7395 27.0412 10.1209 27.2814L14.0393 29.717C14.2473 29.8542 14.4207 29.9228 14.5941 29.9228C14.8021 29.9228 14.9409 29.8198 15.1489 29.5111L15.773 28.5506C16.744 27.0755 17.6109 26.5953 18.7205 26.5953C19.6913 26.5953 20.8357 26.8697 22.2573 27.5558L26.349 29.4768C26.7652 29.6826 27.216 29.5798 27.424 29.0994L29.3658 24.8458C29.5739 24.3654 29.4352 24.0567 28.9497 23.8166ZM10.9532 16.3039C17.1254 19.2884 18.9285 19.7344 21.5292 19.7344C24.5806 19.7344 27.1813 18.4651 29.5045 14.8974L29.886 14.3143C30.1981 13.834 30.2674 13.6625 30.2674 13.4567C30.2674 13.2509 30.1633 13.0794 29.7819 12.8392L25.8636 10.4036C25.6556 10.2664 25.4821 10.1978 25.3088 10.1978C25.1008 10.1978 24.9621 10.3007 24.754 10.6094L24.1298 11.5699C23.1589 13.045 22.2921 13.5253 21.1824 13.5253C20.2115 13.5253 19.0672 13.2509 17.6455 12.5648L13.5538 10.6438C13.1377 10.4379 12.6869 10.5408 12.4789 11.021L10.537 15.2748C10.329 15.755 10.4677 16.0638 10.9532 16.3039Z" fill="white"/>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.5 KiB

datasources/confluence_datasource/datasources/confluence_datasource.py
+260
View File
@@ -0,0 +1,260 @@
import logging
import re
import urllib.parse
from collections.abc import Generator
from typing import Any
import requests
from bs4 import BeautifulSoup
from dify_plugin.entities.datasource import (
DatasourceGetPagesResponse,
DatasourceMessage,
GetOnlineDocumentPageContentRequest,
OnlineDocumentInfo,
OnlineDocumentPage,
)
from dify_plugin.interfaces.datasource.online_document import OnlineDocumentDatasource
logger = logging.getLogger(__name__)
class ConfluenceDataSource(OnlineDocumentDatasource):
_API_BASE = "https://api.atlassian.com/ex/confluence"
def _get_pages(self, datasource_parameters: dict[str, Any]) -> DatasourceGetPagesResponse:
access_token = self.runtime.credentials.get("access_token")
workspace_id = self.runtime.credentials.get("workspace_id")
if not access_token:
raise ValueError("Access token not found in credentials")
headers = {
"Authorization": f"Bearer {access_token}",
"Accept": "application/json",
}
# Use Confluence API v2 for better performance and cleaner structure
url = f"{self._API_BASE}/{workspace_id}/wiki/api/v2/pages"
params = {
"limit": 100,
"sort": "-modified-date",
}
print(f"Fetching Confluence pages from: {url}")
all_pages = []
next_cursor = None
# Handle pagination
while True:
if next_cursor:
params["cursor"] = next_cursor
try:
response = requests.get(url, headers=headers, params=params, timeout=30)
response.raise_for_status()
data = response.json()
except requests.exceptions.HTTPError as e:
if response.status_code == 401:
print(f"Authentication failed with 401. URL: {url}")
print(f"Response: {response.text[:200]}")
raise ValueError(
"Authentication failed (401 Unauthorized). The access token may have expired. "
"Please refresh the connection or reauthorize. If the problem persists, "
"the OAuth app may need reconfiguration."
) from e
else:
print(f"Request failed: {response.status_code} - {response.text[:200]}")
raise ValueError(f"Failed to fetch pages: {response.status_code} {response.text[:200]}") from e
# Parse v2 response structure
for item in data.get("results", []):
# Extract relevant fields from v2 response
page = OnlineDocumentPage(
page_name=item.get("title", ""),
page_id=item.get("id", ""),
type="page", # v2 only returns pages in this endpoint
last_edited_time=item.get("version", {}).get("createdAt", ""),
parent_id=item.get("parentId", ""),
page_icon=None, # v2 doesn't provide icon in list response
)
all_pages.append(page)
# Check if there are more pages
links = data.get("_links", {})
if links.get("next"):
# Extract cursor from next link
next_url = links["next"]
parsed = urllib.parse.urlparse(next_url)
query_params = urllib.parse.parse_qs(parsed.query)
next_cursor = query_params.get("cursor", [None])[0]
if not next_cursor:
break
else:
break
# Get workspace info from credentials
workspace_name = self.runtime.credentials.get("workspace_name", "Confluence")
workspace_id = self.runtime.credentials.get("workspace_id", "unknown")
workspace_icon = self.runtime.credentials.get("workspace_icon", "")
online_document_info = OnlineDocumentInfo(
workspace_name=workspace_name,
workspace_icon=workspace_icon,
workspace_id=workspace_id,
pages=all_pages,
total=len(all_pages),
)
return DatasourceGetPagesResponse(result=[online_document_info])
def _get_content(self, page: GetOnlineDocumentPageContentRequest) -> Generator[DatasourceMessage, None, None]:
access_token = self.runtime.credentials.get("access_token")
workspace_id = self.runtime.credentials.get("workspace_id")
if not access_token:
raise ValueError("Access token not found in credentials")
if not workspace_id:
raise ValueError("Workspace ID not found in credentials")
headers = {
"Authorization": f"Bearer {access_token}",
"Accept": "application/json",
}
page_id = page.page_id
# Use API v2 endpoint for getting page content
url = f"{self._API_BASE}/{workspace_id}/wiki/api/v2/pages/{page_id}"
# v2 allows specifying body format and other details
params = {
"body-format": "storage", # Get the storage format (HTML)
}
try:
logger.debug(f"Fetching page content from: {url}")
response = requests.get(url, headers=headers, params=params, timeout=30)
response.raise_for_status()
data = response.json()
# v2 response structure is different
page_title = data.get("title", "")
page_status = data.get("status", "current")
page_version = data.get("version", {}).get("number", 1)
# Get the body content - v2 structure
body = data.get("body", {})
content_html = ""
# v2 can have multiple body representations
if "storage" in body:
content_html = body["storage"].get("value", "")
elif "atlas_doc_format" in body:
# Handle Atlas Doc Format if present (newer format)
content_html = body["atlas_doc_format"].get("value", "")
else:
logger.warning(f"No body content found for page {page_id}")
content_html = "<p>No content available</p>"
# Convert HTML to text
content_text = self._html_to_text(content_html)
# Return content and metadata
yield self.create_variable_message("content", content_text)
yield self.create_variable_message("page_id", page_id)
yield self.create_variable_message("workspace_id", page.workspace_id or workspace_id)
yield self.create_variable_message("title", page_title)
except requests.exceptions.HTTPError as e:
if response.status_code == 404:
logger.error(f"Page not found: {page_id}")
raise ValueError(f"Page with ID '{page_id}' not found") from e
elif response.status_code == 401:
logger.error(f"Authentication failed for page: {page_id}")
raise ValueError("Authentication failed. Please refresh or reauthorize the connection.") from e
else:
logger.error(f"Failed to fetch page content: {response.status_code}")
raise ValueError(f"Failed to fetch page content: {response.status_code} {response.text[:200]}") from e
except Exception as e:
logger.error(f"Unexpected error fetching page content: {e}")
raise ValueError(f"Error fetching page content: {str(e)}") from e
def _html_to_text(self, html: str) -> str:
"""Convert Confluence HTML to plain text with improved formatting."""
if not html:
return ""
soup = BeautifulSoup(html, "html.parser")
# Remove unwanted tags
for tag in soup(["script", "style", "meta", "noscript", "link"]):
tag.decompose()
# Handle Confluence-specific elements
# Convert ac:structured-macro to readable format
for macro in soup.find_all("ac:structured-macro"):
macro_name = macro.get("ac:name", "") # type: ignore
if macro_name == "code":
# Extract code content
code_body = macro.find("ac:plain-text-body") # type: ignore
if code_body:
code_text = code_body.get_text() # type: ignore
new_tag = soup.new_tag("p")
new_tag.string = f"\n```\n{code_text}\n```\n"
macro.replace_with(new_tag) # type: ignore
elif macro_name == "info" or macro_name == "note":
# Extract info/note content
rich_body = macro.find("ac:rich-text-body") # type: ignore
if rich_body:
note_text = rich_body.get_text(strip=True) # type: ignore
new_tag = soup.new_tag("p")
new_tag.string = f"\n[{str(macro_name).upper()}]: {note_text}\n"
macro.replace_with(new_tag) # type: ignore
else:
macro.decompose() # type: ignore
# Handle tables
for table in soup.find_all("table"):
rows = []
for tr in table.find_all("tr"): # type: ignore
cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])] # type: ignore
if cells:
rows.append(" | ".join(cells))
if rows:
table_text = "\n".join(rows)
new_tag = soup.new_tag("p")
new_tag.string = f"\n{table_text}\n"
table.replace_with(new_tag) # type: ignore
# Extract text with better formatting
text_parts = []
# Process different block elements with appropriate spacing
for element in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "div", "blockquote", "pre"]):
text = element.get_text(strip=True) # type: ignore
if text:
# Add appropriate formatting based on tag
element_name = element.name # type: ignore
if element_name and element_name.startswith("h"):
level = int(element_name[1])
text = f"\n{'#' * level} {text}\n"
elif element_name == "li":
# Check if it's part of ordered or unordered list
parent = element.parent # type: ignore
if parent and hasattr(parent, 'name') and parent.name == "ol":
text = f" * {text}"
else:
text = f" - {text}"
elif element_name == "blockquote":
text = f"\n> {text}\n"
elif element_name == "pre":
text = f"\n```\n{text}\n```\n"
text_parts.append(text)
# Join and clean up excessive whitespace
result = "\n".join(text_parts)
# Remove multiple consecutive newlines
result = re.sub(r'\n{3,}', '\n\n', result)
return result.strip()
datasources/confluence_datasource/datasources/confluence_datasource.yaml
+32
View File
@@ -0,0 +1,32 @@
identity:
name: confluence_datasource
author: langgenius
label:
en_US: Confluence
zh_Hans: Confluence
pt_BR: Confluence
ja_JP: Confluence
description:
en_US: Search for pages and databases in your Confluence workspace
zh_Hans: 在您的 Confluence 工作区中搜索页面和数据库
pt_BR: Pesquise páginas e bancos de dados em seu workspace do Confluence
ja_JP: Confluence ワークスペース内のページとデータベースを検索します
zh_Hant: 在您的 Notion 工作區中搜尋頁面和資料庫
parameters:
output_schema:
type: object
properties:
workspace_id:
type: string
description: workspace id
page_id:
type: string
description: page id
content:
type: string
description: page content
extra:
python:
source: datasources/confluence_datasource.py
datasources/confluence_datasource/main.py
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import DifyPluginEnv, Plugin
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/confluence_datasource/manifest.yaml
+34
View File
@@ -0,0 +1,34 @@
version: 0.2.2
type: plugin
author: langgenius
name: confluence_datasource
label:
en_US: Confluence
ja_JP: Confluence
zh_Hans: Confluence
pt_BR: Confluence
description:
en_US: Confluence Datasource
ja_JP: Confluence Datasource
zh_Hans: Confluence Datasource
pt_BR: Confluence Datasource
icon: icon.svg
resource:
memory: 268435456
permission: {}
plugins:
datasources:
- provider/confluence_datasource.yaml
meta:
version: 0.1.8
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.0.0
created_at: 2025-05-13T14:54:47.062188+08:00
privacy: PRIVACY.md
verified: false
datasources/confluence_datasource/provider/confluence_datasource.py
+130
View File
@@ -0,0 +1,130 @@
import urllib.parse
import time
from collections.abc import Mapping
from typing import Any
import requests
from dify_plugin.entities.datasource import DatasourceOAuthCredentials
from dify_plugin.errors.tool import (
DatasourceOAuthError,
ToolProviderCredentialValidationError,
)
from dify_plugin.interfaces.datasource import DatasourceProvider
from werkzeug import Request
__TIMEOUT_SECONDS__ = 60 * 60
class ConfluenceDatasourceProvider(DatasourceProvider):
_AUTH_URL = "https://auth.atlassian.com/authorize"
_TOKEN_URL = "https://auth.atlassian.com/oauth/token"
_RESOURCE_URL = "https://api.atlassian.com/oauth/token/accessible-resources"
_API_BASE = "https://api.atlassian.com/ex/confluence"
def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str:
params = {
"audience": "api.atlassian.com",
"client_id": system_credentials["client_id"],
"scope": "offline_access read:confluence-content.all read:confluence-space.summary read:confluence-props read:confluence-user read:page:confluence",
"redirect_uri": redirect_uri,
"response_type": "code",
"prompt": "consent",
}
return f"{self._AUTH_URL}?{urllib.parse.urlencode(params)}"
def _oauth_get_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request
) -> DatasourceOAuthCredentials:
code = request.args.get("code")
if not code:
raise DatasourceOAuthError("No code provided")
data = {
"grant_type": "authorization_code",
"client_id": system_credentials["client_id"],
"client_secret": system_credentials["client_secret"],
"code": code,
"redirect_uri": redirect_uri,
}
response = requests.post(self._TOKEN_URL, data=data, timeout=__TIMEOUT_SECONDS__)
if response.status_code != 200:
raise DatasourceOAuthError(f"Token request failed: {response.status_code} {response.text}")
response_json = response.json()
access_token = response_json.get("access_token")
refresh_token = response_json.get("refresh_token")
expires_in = response_json.get("expires_in")
print(f"Expires in: {expires_in}")
if not access_token:
raise DatasourceOAuthError(f"OAuth failed: {response_json}")
headers = {
"Authorization": f"Bearer {access_token}",
"Accept": "application/json",
}
res = requests.get(self._RESOURCE_URL, headers=headers, timeout=10)
if res.status_code != 200:
raise DatasourceOAuthError(f"Failed to get resources: {res.status_code} {res.text}")
resources = res.json()
if not resources or len(resources) == 0:
raise DatasourceOAuthError("No Confluence workspace found for this account.")
resource = resources[0]
cloud_id = resource["id"]
workspace_name = resource["name"]
workspace_url = resource.get("url")
return DatasourceOAuthCredentials(
name=workspace_name,
avatar_url=workspace_url,
credentials={
"access_token": access_token,
"refresh_token": refresh_token,
"workspace_id": cloud_id,
"workspace_name": workspace_name,
"workspace_icon": workspace_url,
},
expires_at=int(time.time()) + expires_in if expires_in else -1,
)
def _oauth_refresh_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], credentials: Mapping[str, Any]
):
refresh_token = credentials.get("refresh_token")
if not refresh_token:
raise DatasourceOAuthError("No refresh token available. Please reauthorize.")
data = {
"grant_type": "refresh_token",
"client_id": system_credentials["client_id"],
"client_secret": system_credentials["client_secret"],
"refresh_token": refresh_token,
}
response = requests.post(self._TOKEN_URL, data=data, timeout=__TIMEOUT_SECONDS__)
if response.status_code != 200:
raise DatasourceOAuthError(f"Token refresh failed: {response.status_code} {response.text}")
response_json = response.json()
new_access_token = response_json.get("access_token")
new_refresh_token = response_json.get("refresh_token", refresh_token) # Some providers return new refresh token
if not new_access_token:
raise DatasourceOAuthError(f"Token refresh failed: {response_json}")
# Keep existing workspace info
return DatasourceOAuthCredentials(
name=credentials.get("workspace_name", "Confluence"),
avatar_url=credentials.get("workspace_icon"),
credentials={
"access_token": new_access_token,
"refresh_token": new_refresh_token,
"workspace_id": credentials.get("workspace_id"),
"workspace_name": credentials.get("workspace_name"),
"workspace_icon": credentials.get("workspace_icon"),
},
)
def _validate_credentials(self, credentials: Mapping[str, Any]):
pass
datasources/confluence_datasource/provider/confluence_datasource.yaml
+69
View File
@@ -0,0 +1,69 @@
identity:
author: langgenius
name: confluence_datasource
label:
en_US: Confluence
zh_Hans: Confluence
pt_BR: Confluence
ja_JP: Confluence
description:
en_US: Confluence Datasource
zh_Hans: Confluence Datasource
pt_BR: Confluence Datasource
ja_JP: Confluence Datasource
icon: icon.svg
oauth_schema:
client_schema:
- name: client_id
type: secret-input
label:
zh_Hans: Client ID
en_US: Client ID
- name: client_secret
type: secret-input
label:
zh_Hans: Client Secret
en_US: Client Secret
help:
zh_Hans: 从 Confluence 集成页面获取您的接口令牌
en_US: Get your integration token from the Confluence Integrations page
url: https://developer.atlassian.com/console/myapps/
credentials_schema:
- name: access_token
type: secret-input
label:
zh_Hans: Access Token
en_US: Access Token
- name: client_id
type: secret-input
label:
zh_Hans: Client ID
en_US: Client ID
- name: client_secret
type: secret-input
label:
zh_Hans: Client Secret
en_US: Client Secret
- name: workspace_id
type: text-input
label:
zh_Hans: Workspace ID
en_US: Workspace ID
- name: workspace_name
type: text-input
label:
zh_Hans: Workspace Name
en_US: Workspace Name
- name: workspace_icon
type: text-input
label:
zh_Hans: Workspace Icon
en_US: Workspace Icon
provider_type: online_document
datasources:
- datasources/confluence_datasource.yaml
extra:
python:
source: provider/confluence_datasource.py
datasources/confluence_datasource/requirements.txt
+4
View File
@@ -0,0 +1,4 @@
dify_plugin==0.5.0b14
requests
werkzeug
beautifulsoup4
datasources/dropbox_datasource/.difyignore
Executable
+179
View File
@@ -0,0 +1,179 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
# Git
.git/
.gitignore
.github/
# Mac
.DS_Store
# Windows
Thumbs.db
datasources/dropbox_datasource/.env.example
Executable
+4
View File
@@ -0,0 +1,4 @@
INSTALL_METHOD=remote
REMOTE_INSTALL_HOST=debug.dify.ai
REMOTE_INSTALL_PORT=5003
REMOTE_INSTALL_KEY=********-****-****-****-************
datasources/dropbox_datasource/.gitattributes
Executable
+2
View File
@@ -0,0 +1,2 @@
# Auto detect text files and perform LF normalization
* text=auto
datasources/dropbox_datasource/.gitignore
Executable
+164
View File
@@ -0,0 +1,164 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# .cursor
.DS_Store
datasources/dropbox_datasource/PRIVACY.md
Executable
+209
View File
@@ -0,0 +1,209 @@
# Dropbox Datasource Plugin Privacy Policy
## Data Collection and Use
### Information We Collect
This plugin collects the following information to provide datasource services:
#### Required Authentication Information
- **OAuth Access Token**: Used to authenticate requests to Dropbox API
- **OAuth Client Credentials**: Client ID and Client Secret for OAuth flow
- **User Account Information**: Basic Dropbox account details (name, email)
#### Optional Configuration Information
- **File Access Patterns**: Which files and folders you browse or download
- **Usage Analytics**: Basic usage statistics for plugin improvement
### Purpose of Information Collection
The collected information is used solely for:
- Establishing secure connections to your Dropbox account
- Authenticating API requests to Dropbox
- Browsing and downloading your Dropbox files
- Providing datasource functionality within Dify
- Maintaining service reliability and performance
## Data Storage and Security
### Local Storage
- OAuth tokens are securely encrypted and stored within Dify platform
- No actual file content is permanently stored by the plugin
- Only temporary file metadata is cached during active sessions
- All sensitive credentials are protected using industry-standard encryption
### Data Transmission Security
- All communication with Dropbox uses HTTPS/TLS encryption
- OAuth 2.0 protocol ensures secure authentication flows
- API requests follow Dropbox security best practices
- No credentials are transmitted in plain text
### Access Control
- Only authorized Dify users can access configured Dropbox datasources
- Role-based access control within Dify platform
- Principle of least privilege is enforced
- OAuth tokens can be revoked at any time
## Data Access Scope
### What We Access
- **File and Folder Metadata**: Names, sizes, modification dates, paths
- **File Content**: Only when explicitly requested for download
- **Account Information**: Basic profile information for authentication
- **Directory Structure**: Folder hierarchy for navigation
### What We Don't Access
- **Files Outside Authorized Scope**: Only files you explicitly grant access to
- **Deleted or Trashed Files**: No access to Dropbox trash
- **Other Users' Files**: Only your own files unless explicitly shared
- **Sensitive Metadata**: No access to edit history or collaboration data
## Third-Party Services
### Dropbox Integration
- This plugin directly communicates with Dropbox API services
- Subject to Dropbox Terms of Service and Privacy Policy
- Dropbox Privacy Policy: https://www.dropbox.com/privacy
- Data remains within your Dropbox account boundaries
### Data Location
- Files remain stored in your Dropbox account
- No files are transferred to third-party servers
- Temporary downloads occur only during active plugin use
- Geographic data location follows your Dropbox settings
## User Rights and Control
### Data Control
- You maintain full control over all files in your Dropbox account
- OAuth authorization can be revoked at any time in Dropbox settings
- Plugin access can be disabled or removed from Dify at any time
- No persistent access to files after plugin removal
### Access Transparency
- Clear visibility into which files are accessed
- Audit trail of plugin activities within Dify
- Real-time permission management
- Granular control over file and folder access
### Data Deletion
- Removing the plugin configuration clears all stored credentials
- No residual file content remains after plugin removal
- OAuth tokens are invalidated upon plugin deletion
- Cache data is automatically purged after sessions end
## Compliance and Standards
### Data Protection Regulations
- Compliant with GDPR (General Data Protection Regulation)
- Adheres to applicable data protection laws
- Supports data subject rights requests
- Implements privacy by design principles
### Industry Standards
- Follows ISO 27001 information security standards
- Complies with SOC 2 Type II controls
- Adheres to OAuth 2.0 security specifications
- Implements Dropbox API security guidelines
### Enterprise Compliance
- Supports enterprise data governance requirements
- Compatible with corporate security policies
- Audit logging for compliance monitoring
- Integration with enterprise identity management
## Security Best Practices
### Authentication Security
- OAuth 2.0 implementation with secure token handling
- Regular security reviews and updates
- Protection against common OAuth vulnerabilities
- Secure credential storage with encryption at rest
### API Security
- Rate limiting to prevent abuse
- Proper error handling without information disclosure
- Secure communication channels only
- Regular security patching and updates
### Monitoring and Auditing
- Security event logging
- Anomaly detection for suspicious activities
- Regular security assessments
- Incident response procedures
## Incident Response
### Security Incidents
In the event of a security incident, we will:
- Immediately investigate and assess the impact
- Notify affected users within 72 hours
- Take measures to prevent further data exposure
- Coordinate with relevant authorities as required
### Data Breach Notification
- Prompt notification of any potential data breaches
- Detailed incident reports with impact assessment
- Remediation steps and preventive measures
- Ongoing monitoring and support
## Contact Information
### Privacy Inquiries
For privacy-related questions, please contact:
- **Email**: privacy@dify.ai
- **Subject**: Dropbox Datasource Plugin Privacy Inquiry
### Technical Support
For technical issues and support:
- **Email**: support@dify.ai
- **Documentation**: https://docs.dify.ai
- **Issue Tracker**: Official Dify support channels
### Data Protection Officer
For formal data protection requests:
- **Email**: dpo@dify.ai
- **Address**: Dify Data Protection Office
## Policy Updates
### Update Notifications
- Users will be notified of material privacy policy changes
- Significant changes require explicit user consent
- Update history is maintained and accessible
- Regular policy reviews to ensure current compliance
### Version History
- **v1.0** (January 2025): Initial privacy policy
- OAuth 2.0 authentication privacy protections
- File access and download privacy safeguards
- Comprehensive data handling procedures
### Effective Date
This privacy policy is effective as of January 2025 and applies to all users of the Dropbox Datasource Plugin.
## Additional Protections
### Children's Privacy
- This plugin is not designed for use by children under 13
- No intentional collection of children's personal information
- Parents should supervise any use by minors
- Compliance with COPPA regulations where applicable
### International Transfers
- Data transfers comply with applicable international frameworks
- Adequate protection measures for cross-border data flows
- Compliance with regional data protection requirements
- User notification of any international data processing
### Data Minimization
- Collection limited to data necessary for plugin functionality
- Regular review and deletion of unnecessary data
- Purpose limitation for all data processing activities
- Transparency in data collection practices
---
**Last Updated**: January 2025
**Version**: 1.0
**Plugin Version**: 0.1.0
For the most current version of this privacy policy, please check the plugin documentation in your Dify installation.
datasources/dropbox_datasource/README.md
Executable
+198
View File
@@ -0,0 +1,198 @@
# Dropbox Datasource Plugin
Access your Dropbox files and folders as a datasource for Dify with OAuth 2.0 authentication.
## Features
- **🔐 OAuth 2.0 Authentication**: Secure authentication using Dropbox OAuth 2.0 flow
- **📂 File & Folder Browsing**: Navigate through your Dropbox folder structure
- **⬇️ File Downloads**: Download files directly into Dify workflows
- **🔄 Pagination Support**: Handle large directories with efficient pagination
- **🎯 Smart File ID Handling**: Automatic conversion between file IDs and paths
- **📝 MIME Type Detection**: Automatic file type detection for proper handling
- **🛡️ Error Handling**: Comprehensive error handling with user-friendly messages
- **🚀 High Performance**: Optimized for speed and reliability
## Supported File Types
This plugin supports downloading all file types from Dropbox:
### Documents
- `.pdf` - PDF documents
- `.doc`, `.docx` - Microsoft Word documents
- `.xls`, `.xlsx` - Microsoft Excel spreadsheets
- `.ppt`, `.pptx` - Microsoft PowerPoint presentations
- `.txt` - Plain text files
### Images
- `.jpg`, `.jpeg` - JPEG images
- `.png` - PNG images
- `.gif` - GIF images
- `.bmp` - Bitmap images
- `.svg` - SVG vector graphics
### Media
- `.mp4` - MP4 videos
- `.avi` - AVI videos
- `.mov` - QuickTime videos
- `.mp3` - MP3 audio
- `.wav` - WAV audio
### Archives & Code
- `.zip` - ZIP archives
- `.rar` - RAR archives
- `.tar`, `.gz` - TAR archives
- `.json` - JSON files
- `.xml` - XML files
- `.html`, `.css`, `.js` - Web files
- `.py`, `.java`, `.cpp`, `.c` - Source code files
## Authentication Methods
### OAuth 2.0 (Recommended)
- Secure authentication through Dropbox OAuth flow
- Automatic token management
- Dropbox tokens don't expire (long-lived)
- Granular permission control
## Setup Instructions
### 1. Create Dropbox App
1. Go to [Dropbox App Console](https://www.dropbox.com/developers/apps)
2. Click "Create app"
3. Choose "Scoped access"
4. Choose "Full Dropbox" or "App folder" based on your needs
5. Enter your app name
6. Click "Create app"
### 2. Configure OAuth Settings
1. In your Dropbox app settings, find the "OAuth 2" section
2. Add your Dify redirect URI to "Redirect URIs":
```
https://your-dify-instance.com/console/api/workspaces/current/data-sources/oauth/callback
```
3. Note down your **App key** (Client ID) and **App secret** (Client Secret)
### 3. Set Required Permissions
In the "Permissions" tab, enable these scopes:
- `files.metadata.read` - Read file and folder metadata
- `files.content.read` - Download files
### 4. Install in Dify
1. In Dify, go to **Data Sources** → **Add Data Source**
2. Select **Dropbox**
3. Enter your **Client ID** and **Client Secret**
4. Complete the OAuth authorization flow
5. Grant permissions to your Dropbox account
## Usage
### Browse Files
1. After authentication, you can browse your Dropbox files and folders
2. Navigate through directories by clicking on folder names
3. Use the pagination controls for large directories
### Download Files
1. Select files you want to use as data sources
2. Files will be downloaded and made available to your Dify workflows
3. All file metadata (name, size, type) is preserved
### Integration with Dify
- Use downloaded files in Knowledge Base creation
- Process files in Dify workflows
- Extract content for AI analysis
- Support for all common file formats
## Security & Privacy
### Data Protection
- All communication with Dropbox uses HTTPS encryption
- OAuth tokens are securely stored in Dify
- No file content is cached permanently
- Only requested files are downloaded temporarily
### Permissions
- Plugin only accesses files you explicitly authorize
- Follows principle of least privilege
- OAuth tokens can be revoked at any time in Dropbox settings
- Full audit trail of accessed files
### Compliance
- Complies with Dropbox API Terms of Service
- Supports GDPR data protection requirements
- Enterprise-grade security standards
- Regular security updates
## Technical Specifications
### Requirements
- Python 3.12+
- Dify Plugin SDK 0.5.0b14+
- Dropbox Python SDK 12.0.2+
### API Limitations
- Follows Dropbox API rate limits
- Supports files up to Dropbox's maximum file size
- Efficient pagination for large directories
- Automatic retry on transient errors
### Performance
- Optimized for large file downloads
- Smart caching of folder structures
- Minimal memory footprint
- Asynchronous operations where possible
## Troubleshooting
### Common Issues
**Authentication Failed**
- Check your Client ID and Client Secret
- Verify redirect URI is correctly configured
- Ensure Dropbox app has required permissions
**File Not Found**
- Verify file still exists in Dropbox
- Check if file was moved or deleted
- Refresh the file list and try again
**Permission Denied**
- Check if you have access to the file/folder
- Verify OAuth permissions are granted
- Re-authorize if necessary
**Large File Downloads**
- Large files may take longer to download
- Check your network connection
- Consider breaking into smaller chunks
### Support
For technical support:
- Check Dify documentation
- Review Dropbox API documentation
- Contact support through Dify channels
## Version History
### v0.1.0 (Current)
- ✅ Initial release
- ✅ OAuth 2.0 authentication
- ✅ File and folder browsing
- ✅ File downloads with MIME detection
- ✅ Pagination support
- ✅ Comprehensive error handling
- ✅ Smart file ID/path conversion
## License
This plugin is released under the same license as Dify.
---
**Author**: langgenius
**Version**: 0.1.0
**Last Updated**: January 2025
datasources/dropbox_datasource/_assets/icon.svg
Executable
+5
View File
@@ -0,0 +1,5 @@
<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="40" height="40" fill="white"/>
<rect width="40" height="40" fill="#0061FF"/>
<path d="M12.3438 29.0625L20 33.9844L27.6562 29.0625L20 24.2188L12.3438 29.0625ZM12.3438 7.89062L4.60938 12.8125L27.6562 27.5L35.3906 22.5781L20 12.8125L27.6562 7.89062L35.3906 12.8125L12.3438 27.5L4.60938 22.5781L20 12.8125" fill="white"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 434 B

datasources/dropbox_datasource/datasources/dropbox.py
Executable
+261
View File
@@ -0,0 +1,261 @@
import logging
import os
from collections.abc import Generator
import dropbox
from dropbox.exceptions import AuthError, ApiError
from dropbox.files import FileMetadata, FolderMetadata
from dify_plugin.entities.datasource import (
DatasourceMessage,
OnlineDriveBrowseFilesRequest,
OnlineDriveBrowseFilesResponse,
OnlineDriveDownloadFileRequest,
OnlineDriveFile,
OnlineDriveFileBucket,
)
from dify_plugin.interfaces.datasource.online_drive import OnlineDriveDatasource
logger = logging.getLogger(__name__)
class DropboxDataSource(OnlineDriveDatasource):
def _get_dropbox_client(self) -> dropbox.Dropbox:
"""Get authenticated Dropbox client"""
credentials = self.runtime.credentials
if not credentials:
raise ValueError("Credentials not found")
access_token = credentials.get("access_token")
if not access_token:
raise ValueError("Access token not found in credentials")
try:
dbx = dropbox.Dropbox(access_token)
# Verify the connection
dbx.users_get_current_account()
return dbx
except AuthError as e:
raise ValueError(f"Authentication failed: {str(e)}") from e
def _browse_files(
self, request: OnlineDriveBrowseFilesRequest
) -> OnlineDriveBrowseFilesResponse:
bucket_name = request.bucket or "" # Dropbox doesn't have buckets, use empty string
prefix = request.prefix or "" # Start from root if no prefix
max_keys = request.max_keys or 100
next_page_parameters = request.next_page_parameters or {}
try:
dbx = self._get_dropbox_client()
# Handle file IDs vs paths
if prefix and prefix.startswith("id:"):
# Convert file ID to path
try:
metadata = dbx.files_get_metadata(prefix)
if isinstance(metadata, FolderMetadata):
# It's a folder, use its path
path = metadata.path_display
else:
# It's a file, can't list contents of a file
raise ValueError(f"Cannot browse contents of file: {metadata.name}")
except ApiError as e:
error_str = str(e)
if "not_found" in error_str:
raise ValueError(f"Item with ID '{prefix}' not found") from e
else:
raise ValueError(f"Failed to resolve ID '{prefix}': {error_str}") from e
else:
# Build the path to list
path = prefix if prefix else ""
if path and not path.startswith("/"):
path = "/" + path
if path == "/":
path = "" # Root folder should be empty string
# Handle pagination
cursor = next_page_parameters.get("cursor")
try:
if cursor:
# Continue from previous listing
result = dbx.files_list_folder_continue(cursor)
else:
# Start new listing
result = dbx.files_list_folder(path, limit=max_keys)
files = []
for entry in result.entries:
file_info = {
"id": entry.id,
"name": entry.name,
"path": entry.path_display,
}
if isinstance(entry, FileMetadata):
file_info.update({
"type": "file",
"size": entry.size,
"modified": entry.server_modified.isoformat() if entry.server_modified else None,
})
elif isinstance(entry, FolderMetadata):
file_info.update({
"type": "folder",
"size": 0,
})
files.append(OnlineDriveFile(
id=file_info["id"],
name=file_info["name"],
size=file_info["size"],
type=file_info["type"]
))
# Handle pagination
next_cursor = result.cursor if result.has_more else None
next_page_parameters = {"cursor": next_cursor} if next_cursor else {}
is_truncated = result.has_more
return OnlineDriveBrowseFilesResponse(result=[
OnlineDriveFileBucket(
bucket=bucket_name,
files=files,
is_truncated=is_truncated,
next_page_parameters=next_page_parameters
)
])
except ApiError as e:
error_str = str(e)
if "not_found" in error_str:
raise ValueError(f"Path '{path}' not found") from e
elif "malformed_path" in error_str:
raise ValueError(f"Invalid path format: '{path}'") from e
elif "invalid_cursor" in error_str:
raise ValueError("Invalid pagination cursor") from e
else:
raise ValueError(f"Dropbox API error: {error_str}") from e
except AuthError as e:
raise ValueError(
"Authentication failed. The access token may have expired. "
"Please refresh or reauthorize the connection."
) from e
except Exception as e:
if "Authentication failed" in str(e):
raise
logger.error(f"Error browsing Dropbox files: {e}")
raise ValueError(f"Failed to browse Dropbox files: {str(e)}") from e
def _download_file(self, request: OnlineDriveDownloadFileRequest) -> Generator[DatasourceMessage, None, None]:
file_id = request.id
if not file_id:
raise ValueError("File ID is required")
try:
dbx = self._get_dropbox_client()
# First, get file metadata using the file ID
try:
# Get file metadata by ID
metadata_result = dbx.files_get_metadata(file_id)
if not isinstance(metadata_result, FileMetadata):
raise ValueError("The specified ID is not a file")
file_name = metadata_result.name
file_size = metadata_result.size
# Use the path_display for downloading since Dropbox download API requires path
file_path = metadata_result.path_display
except ApiError as e:
error_str = str(e)
if "not_found" in error_str:
raise ValueError(f"File with ID '{file_id}' not found") from e
else:
raise ValueError(f"Failed to get file metadata: {error_str}") from e
# Download the file content
try:
_, response = dbx.files_download(file_path)
file_content = response.content
# Determine MIME type based on file extension
mime_type = self._get_mime_type(file_name)
logger.debug(f"Downloaded file: {file_name} (size: {file_size} bytes)")
yield self.create_blob_message(file_content, meta={
"file_name": file_name,
"mime_type": mime_type,
"file_size": file_size
})
except ApiError as e:
error_str = str(e)
if "not_found" in error_str:
raise ValueError(f"File '{file_path}' not found") from e
elif "insufficient_space" in error_str:
raise ValueError("Insufficient space to download file") from e
else:
raise ValueError(f"Failed to download file: {error_str}") from e
except AuthError as e:
raise ValueError(
"Authentication failed during file download. "
"Please refresh or reauthorize the connection."
) from e
except Exception as e:
if "Authentication failed" in str(e) or "not found" in str(e).lower():
raise
logger.error(f"Unexpected error downloading file: {e}")
raise ValueError(f"Error downloading file: {str(e)}") from e
def _get_mime_type(self, file_name: str) -> str:
"""Determine MIME type based on file extension"""
if not file_name:
return "application/octet-stream"
ext = os.path.splitext(file_name.lower())[1]
mime_types = {
'.txt': 'text/plain',
'.pdf': 'application/pdf',
'.doc': 'application/msword',
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'.xls': 'application/vnd.ms-excel',
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'.ppt': 'application/vnd.ms-powerpoint',
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.bmp': 'image/bmp',
'.svg': 'image/svg+xml',
'.mp4': 'video/mp4',
'.avi': 'video/x-msvideo',
'.mov': 'video/quicktime',
'.mp3': 'audio/mpeg',
'.wav': 'audio/wav',
'.zip': 'application/zip',
'.rar': 'application/x-rar-compressed',
'.tar': 'application/x-tar',
'.gz': 'application/gzip',
'.json': 'application/json',
'.xml': 'application/xml',
'.html': 'text/html',
'.css': 'text/css',
'.js': 'application/javascript',
'.py': 'text/x-python',
'.java': 'text/x-java-source',
'.cpp': 'text/x-c++src',
'.c': 'text/x-csrc',
'.h': 'text/x-chdr',
}
return mime_types.get(ext, 'application/octet-stream')
datasources/dropbox_datasource/datasources/dropbox.yaml
Executable
+27
View File
@@ -0,0 +1,27 @@
identity:
name: dropbox
author: langgenius
label:
en_US: "Dropbox"
ja_JP: "Dropbox"
zh_Hans: "Dropbox"
pt_BR: "Dropbox"
zh_Hant: "Dropbox"
description:
en_US: "Get files from your Dropbox"
ja_JP: "Dropboxからファイルを取得します"
zh_Hans: "从您的 Dropbox 中获取文件"
pt_BR: "Obtenha arquivos do seu Dropbox"
zh_Hant: "從您的 Dropbox 中獲取文件"
parameters:
output_schema:
type: object
properties:
file:
$ref: "https://dify.ai/schemas/v1/file.json"
extra:
python:
source: datasources/dropbox.py
datasources/dropbox_datasource/main.py
Executable
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import Plugin, DifyPluginEnv
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/dropbox_datasource/manifest.yaml
Executable
+36
View File
@@ -0,0 +1,36 @@
version: 0.2.1
type: plugin
author: langgenius
name: dropbox_datasource
label:
en_US: "Dropbox"
ja_JP: "Dropbox"
zh_Hans: "Dropbox"
pt_BR: "Dropbox"
zh_Hant: "Dropbox"
description:
en_US: "Dropbox Datasource"
ja_JP: "Dropboxデータソース"
zh_Hans: "Dropbox 数据源"
pt_BR: "Fonte de dados do Dropbox"
zh_Hant: "Dropbox 數據源"
icon: icon.svg
resource:
memory: 268435456
permission: {}
plugins:
datasources:
- provider/dropbox.yaml
meta:
version: 0.1.0
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-07-31T14:54:47.062188+08:00
privacy: ""
verified: false
datasources/dropbox_datasource/provider/dropbox.py
Executable
+132
View File
@@ -0,0 +1,132 @@
import secrets
import urllib.parse
from typing import Any, Mapping
import requests
import dropbox
from dropbox.exceptions import AuthError, ApiError
from werkzeug import Request
from dify_plugin.interfaces.datasource import DatasourceProvider, DatasourceOAuthCredentials
from dify_plugin.errors.tool import ToolProviderCredentialValidationError, DatasourceOAuthError
class DropboxDatasourceProvider(DatasourceProvider):
_AUTH_URL = "https://www.dropbox.com/oauth2/authorize"
_TOKEN_URL = "https://api.dropboxapi.com/oauth2/token"
def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
try:
# Check if access_token is provided in credentials
if "access_token" not in credentials or not credentials.get("access_token"):
raise ToolProviderCredentialValidationError("Dropbox access token is required.")
access_token = credentials.get("access_token")
# Try to authenticate with Dropbox using the access token
try:
dbx = dropbox.Dropbox(access_token)
# Verify the connection by getting current account
dbx.users_get_current_account()
except AuthError as e:
raise ToolProviderCredentialValidationError(f"Invalid Dropbox access token: {str(e)}")
except Exception as e:
raise ToolProviderCredentialValidationError(f"Failed to connect to Dropbox: {str(e)}")
except Exception as e:
raise ToolProviderCredentialValidationError(str(e))
def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str:
"""
Generate the authorization URL for the Dropbox OAuth 2.0 flow.
"""
state = secrets.token_urlsafe(16)
params = {
"client_id": system_credentials["client_id"],
"redirect_uri": redirect_uri,
"response_type": "code",
"state": state,
}
return f"{self._AUTH_URL}?{urllib.parse.urlencode(params)}"
def _oauth_get_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request
) -> DatasourceOAuthCredentials:
"""
Exchange authorization code for access_token.
"""
code = request.args.get("code")
if not code:
raise DatasourceOAuthError("No authorization code provided")
data = {
"client_id": system_credentials["client_id"],
"client_secret": system_credentials["client_secret"],
"code": code,
"grant_type": "authorization_code",
"redirect_uri": redirect_uri,
}
headers = {"Content-Type": "application/x-www-form-urlencoded"}
try:
response = requests.post(self._TOKEN_URL, data=data, headers=headers, timeout=30)
response.raise_for_status()
response_json = response.json()
access_token = response_json.get("access_token")
if not access_token:
raise DatasourceOAuthError(f"Error in Dropbox OAuth: {response_json}")
# Get user information
try:
dbx = dropbox.Dropbox(access_token)
account_info = dbx.users_get_current_account()
user_name = account_info.name.display_name
user_email = account_info.email
user_avatar = account_info.profile_photo_url if hasattr(account_info, 'profile_photo_url') else None
except Exception as e:
# If we can't get user info, use defaults
user_name = "Dropbox User"
user_email = None
user_avatar = None
return DatasourceOAuthCredentials(
name=user_name or user_email or "Dropbox User",
avatar_url=user_avatar,
credentials={"access_token": access_token},
expires_at=-1 # Dropbox tokens don't expire
)
except requests.RequestException as e:
raise DatasourceOAuthError(f"Failed to exchange code for token: {str(e)}")
except Exception as e:
raise DatasourceOAuthError(f"Unexpected error during OAuth: {str(e)}")
def _oauth_refresh_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], credentials: Mapping[str, Any]
) -> DatasourceOAuthCredentials:
"""
Dropbox access tokens don't expire, so we just return the existing credentials.
"""
access_token = credentials.get("access_token")
if not access_token:
raise DatasourceOAuthError("No access token available. Please re-authorize.")
# Validate the token is still working
try:
dbx = dropbox.Dropbox(access_token)
account_info = dbx.users_get_current_account()
user_name = account_info.name.display_name
user_email = account_info.email
user_avatar = account_info.profile_photo_url if hasattr(account_info, 'profile_photo_url') else None
except AuthError:
raise DatasourceOAuthError("Access token is no longer valid. Please re-authorize.")
except Exception as e:
raise DatasourceOAuthError(f"Failed to validate token: {str(e)}")
return DatasourceOAuthCredentials(
name=user_name or user_email or "Dropbox User",
avatar_url=user_avatar,
credentials={"access_token": access_token},
expires_at=-1 # Dropbox tokens don't expire
)
datasources/dropbox_datasource/provider/dropbox.yaml
Executable
+129
View File
@@ -0,0 +1,129 @@
identity:
author: langgenius
name: dropbox
label:
en_US: "Dropbox"
ja_JP: "Dropbox"
zh_Hans: "Dropbox"
pt_BR: "Dropbox"
zh_Hant: "Dropbox"
description:
en_US: "Dropbox Datasource"
ja_JP: "Dropboxデータソース"
zh_Hans: "Dropbox 数据源"
pt_BR: "Fonte de dados do Dropbox"
zh_Hant: "Dropbox 數據源"
icon: icon.svg
provider_type: online_drive
help:
title:
en_US: "Get your credentials from Dropbox App Console"
ja_JP: "Dropbox App Console から認証情報を取得します"
zh_Hans: "从 Dropbox 应用控制台获取您的凭证"
pt_BR: "Obtenha suas credenciais no Dropbox App Console"
zh_Hant: "從 Dropbox 應用程式控制台取得您的憑證"
url:
en_US: "https://www.dropbox.com/developers/apps"
ja_JP: "https://www.dropbox.com/developers/apps"
zh_Hans: "https://www.dropbox.com/developers/apps"
pt_BR: "https://www.dropbox.com/developers/apps"
zh_Hant: "https://www.dropbox.com/developers/apps"
oauth_schema:
client_schema:
- name: client_id
type: secret-input
required: true
label:
en_US: "Client ID"
ja_JP: "Client ID"
zh_Hans: "Client ID"
pt_BR: "Client ID"
zh_Hant: "Client ID"
placeholder:
en_US: "Please input your Client ID"
ja_JP: "クライアントIDを入力してください"
zh_Hans: "请输入你的 Client ID"
pt_BR: "Insira seu ID de cliente"
zh_Hant: "請輸入您的客戶端 ID"
help:
en_US: "Client ID is used to authenticate requests to the Dropbox API."
ja_JP: "クライアントIDは、Dropbox APIへのリクエストを認証するために使用されます。"
zh_Hans: "Client ID 用于认证请求到 Dropbox API。"
pt_BR: "O ID do cliente é usado para autenticar solicitações à API do Dropbox."
zh_Hant: "客戶端 ID 用於對 Dropbox API 的請求進行身份驗證。"
url: https://www.dropbox.com/developers/apps
- name: client_secret
type: secret-input
required: true
label:
en_US: "Client Secret"
ja_JP: "Client Secret"
zh_Hans: "Client Secret"
pt_BR: "Client Secret"
zh_Hant: "Client Secret"
placeholder:
en_US: "Please input your Client Secret"
ja_JP: "クライアントシークレットを入力してください"
zh_Hans: "请输入你的 Client Secret"
pt_BR: "Insira seu segredo de cliente"
zh_Hant: "請輸入您的客戶端密鑰"
help:
en_US: "Get your client secret from the Dropbox App Console"
ja_JP: "Dropbox App Console からクライアントシークレットを取得します"
zh_Hans: "从 Dropbox 应用控制台获取您的客户端密钥"
pt_BR: "Obtenha seu segredo de cliente no Dropbox App Console"
zh_Hant: "從 Dropbox 應用程式控制台取得您的用戶端密碼"
url: https://www.dropbox.com/developers/apps
credentials_schema:
- name: access_token
type: secret-input
label:
en_US: "Access Token"
ja_JP: "Access Token"
zh_Hans: "Access Token"
pt_BR: "Access Token"
zh_Hant: "Access Token"
- name: client_id
type: secret-input
label:
en_US: "Client ID"
ja_JP: "Client ID"
zh_Hans: "Client ID"
pt_BR: "Client ID"
zh_Hant: "Client ID"
- name: client_secret
type: secret-input
label:
en_US: "Client Secret"
ja_JP: "Client Secret"
zh_Hans: "Client Secret"
pt_BR: "Client Secret"
zh_Hant: "Client Secret"
credentials_schema:
- name: access_token
type: secret-input
required: true
label:
en_US: "Access Token"
ja_JP: "アクセストークン"
zh_Hans: "访问令牌"
pt_BR: "Token de acesso"
zh_Hant: "訪問令牌"
placeholder:
en_US: "Please input your Dropbox access token"
ja_JP: "Dropboxアクセストークンを入力してください"
zh_Hans: "请输入您的 Dropbox 访问令牌"
pt_BR: "Insira seu token de acesso do Dropbox"
zh_Hant: "請輸入您的 Dropbox 存取權杖"
help:
en_US: "Get your access token from Dropbox App Console"
ja_JP: "Dropbox App Console からアクセストークンを取得します"
zh_Hans: "从 Dropbox 应用控制台获取您的访问令牌"
pt_BR: "Obtenha seu token de acesso no Dropbox App Console"
zh_Hant: "從 Dropbox 應用程式控制台取得您的存取權杖"
url: https://www.dropbox.com/developers/apps
datasources:
- datasources/dropbox.yaml
extra:
python:
source: provider/dropbox.py
datasources/dropbox_datasource/requirements.txt
Executable
+2
View File
@@ -0,0 +1,2 @@
dify_plugin>=0.5.0b14
dropbox~=12.0.2
datasources/firecrawl_datasource/.difyignore
+179
View File
@@ -0,0 +1,179 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Vscode
.vscode/
# Git
.git/
.gitignore
.github/
# Mac
.DS_Store
# Windows
Thumbs.db
datasources/firecrawl_datasource/.env.example
+4
View File
@@ -0,0 +1,4 @@
INSTALL_METHOD=remote
REMOTE_INSTALL_HOST=debug.dify.ai
REMOTE_INSTALL_PORT=5003
REMOTE_INSTALL_KEY=********-****-****-****-************
datasources/firecrawl_datasource/GUIDE.md
+149
View File
@@ -0,0 +1,149 @@
## User Guide of how to develop a Dify Plugin
Hi there, looks like you have already created a Plugin, now let's get you started with the development!
### Choose a Plugin type you want to develop
Before start, you need some basic knowledge about the Plugin types, Plugin supports to extend the following abilities in Dify:
- **Tool**: Tool Providers like Google Search, Stable Diffusion, etc. it can be used to perform a specific task.
- **Model**: Model Providers like OpenAI, Anthropic, etc. you can use their models to enhance the AI capabilities.
- **Endpoint**: Like Service API in Dify and Ingress in Kubernetes, you can extend a http service as an endpoint and control its logics using your own code.
Based on the ability you want to extend, we have divided the Plugin into three types: **Tool**, **Model**, and **Extension**.
- **Tool**: It's a tool provider, but not only limited to tools, you can implement an endpoint there, for example, you need both `Sending Message` and `Receiving Message` if you are building a Discord Bot, **Tool** and **Endpoint** are both required.
- **Model**: Just a model provider, extending others is not allowed.
- **Extension**: Other times, you may only need a simple http service to extend the functionalities, **Extension** is the right choice for you.
I believe you have chosen the right type for your Plugin while creating it, if not, you can change it later by modifying the `manifest.yaml` file.
### Manifest
Now you can edit the `manifest.yaml` file to describe your Plugin, here is the basic structure of it:
- version(version, required)：Plugin's version
- type(type, required)：Plugin's type, currently only supports `plugin`, future support `bundle`
- author(string, required)：Author, it's the organization name in Marketplace and should also equals to the owner of the repository
- label(label, required)：Multi-language name
- created_at(RFC3339, required)：Creation time, Marketplace requires that the creation time must be less than the current time
- icon(asset, required)：Icon path
- resource (object)：Resources to be applied
- memory (int64)：Maximum memory usage, mainly related to resource application on SaaS for serverless, unit bytes
- permission(object)：Permission application
- tool(object)：Reverse call tool permission
- enabled (bool)
- model(object)：Reverse call model permission
- enabled(bool)
- llm(bool)
- text_embedding(bool)
- rerank(bool)
- tts(bool)
- speech2text(bool)
- moderation(bool)
- node(object)：Reverse call node permission
- enabled(bool)
- endpoint(object)：Allow to register endpoint permission
- enabled(bool)
- app(object)：Reverse call app permission
- enabled(bool)
- storage(object)：Apply for persistent storage permission
- enabled(bool)
- size(int64)：Maximum allowed persistent memory, unit bytes
- plugins(object, required)：Plugin extension specific ability yaml file list, absolute path in the plugin package, if you need to extend the model, you need to define a file like openai.yaml, and fill in the path here, and the file on the path must exist, otherwise the packaging will fail.
- Format
- tools(list[string]): Extended tool suppliers, as for the detailed format, please refer to [Tool Guide](https://docs.dify.ai/plugins/schema-definition/tool)
- models(list[string])：Extended model suppliers, as for the detailed format, please refer to [Model Guide](https://docs.dify.ai/plugins/schema-definition/model)
- endpoints(list[string])：Extended Endpoints suppliers, as for the detailed format, please refer to [Endpoint Guide](https://docs.dify.ai/plugins/schema-definition/endpoint)
- Restrictions
- Not allowed to extend both tools and models
- Not allowed to have no extension
- Not allowed to extend both models and endpoints
- Currently only supports up to one supplier of each type of extension
- meta(object)
- version(version, required)：manifest format version, initial version 0.0.1
- arch(list[string], required)：Supported architectures, currently only supports amd64 arm64
- runner(object, required)：Runtime configuration
- language(string)：Currently only supports python
- version(string)：Language version, currently only supports 3.12
- entrypoint(string)：Program entry, in python it should be main
### Install Dependencies
- First of all, you need a Python 3.11+ environment, as our SDK requires that.
- Then, install the dependencies:
```bash
pip install -r requirements.txt
```
- If you want to add more dependencies, you can add them to the `requirements.txt` file, once you have set the runner to python in the `manifest.yaml` file, `requirements.txt` will be automatically generated and used for packaging and deployment.
### Implement the Plugin
Now you can start to implement your Plugin, by following these examples, you can quickly understand how to implement your own Plugin:
- [OpenAI](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/openai): best practice for model provider
- [Google Search](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/google): a simple example for tool provider
- [Neko](https://github.com/langgenius/dify-plugin-sdks/tree/main/python/examples/neko): a funny example for endpoint group
### Test and Debug the Plugin
You may already noticed that a `.env.example` file in the root directory of your Plugin, just copy it to `.env` and fill in the corresponding values, there are some environment variables you need to set if you want to debug your Plugin locally.
- `INSTALL_METHOD`: Set this to `remote`, your plugin will connect to a Dify instance through the network.
- `REMOTE_INSTALL_HOST`: The host of your Dify instance, you can use our SaaS instance `https://debug.dify.ai`, or self-hosted Dify instance.
- `REMOTE_INSTALL_PORT`: The port of your Dify instance, default is 5003
- `REMOTE_INSTALL_KEY`: You should get your debugging key from the Dify instance you used, at the right top of the plugin management page, you can see a button with a `debug` icon, click it and you will get the key.
Run the following command to start your Plugin:
```bash
python -m main
```
Refresh the page of your Dify instance, you should be able to see your Plugin in the list now, but it will be marked as `debugging`, you can use it normally, but not recommended for production.
### Publish and Update the Plugin
To streamline your plugin update workflow, you can configure GitHub Actions to automatically create PRs to the Dify plugin repository whenever you create a release.
##### Prerequisites
- Your plugin source repository
- A fork of the dify-plugins repository
- Proper plugin directory structure in your fork
#### Configure GitHub Action
1. Create a Personal Access Token with write permissions to your forked repository
2. Add it as a secret named `PLUGIN_ACTION` in your source repository settings
3. Create a workflow file at `.github/workflows/plugin-publish.yml`
#### Usage
1. Update your code and the version in your `manifest.yaml`
2. Create a release in your source repository
3. The action automatically packages your plugin and creates a PR to your forked repository
#### Benefits
- Eliminates manual packaging and PR creation steps
- Ensures consistency in your release process
- Saves time during frequent updates
---
For detailed setup instructions and example configuration, visit: [GitHub Actions Workflow Documentation](https://docs.dify.ai/plugins/publish-plugins/plugin-auto-publish-pr)
### Package the Plugin
After all, just package your Plugin by running the following command:
```bash
dify-plugin plugin package ./ROOT_DIRECTORY_OF_YOUR_PLUGIN
```
you will get a `plugin.difypkg` file, that's all, you can submit it to the Marketplace now, look forward to your Plugin being listed!
## User Privacy Policy
Please fill in the privacy policy of the plugin if you want to make it published on the Marketplace, refer to [PRIVACY.md](PRIVACY.md) for more details.
datasources/firecrawl_datasource/PRIVACY.md
+109
View File
@@ -0,0 +1,109 @@
# Privacy Policy
This Privacy Policy explains how we collect, use, and protect your information when you use the Firecrawl Datasource Plugin for Dify.
## Information Collection
We do not collect, store, or share any personal information from users. All data processed by the plugin is handled locally or through secure connections to Firecrawl API services, as required for functionality.
## Use of Information
The plugin only accesses publicly available web content to provide the requested features:
- Recursively crawling websites and subdomains
- Extracting and structuring web page content
- Filtering content based on URL patterns
- Converting web content to clean, structured markdown format
No personal data is collected, stored, or transmitted to third parties by the plugin itself.
## Data Processing
When using this plugin:
- **URL Processing**: Target URLs are sent to Firecrawl's API for processing
- **Content Extraction**: Web page content is extracted and converted to structured format
- **Crawling Scope**: Only specified URLs and their subpages (based on your settings) are accessed
- **Temporary Processing**: Data is processed in real-time and not permanently stored by the plugin
- **Content Filtering**: Main content can be extracted while excluding headers, footers, and navigation
## Third-Party Services
This plugin interacts with Firecrawl API services:
- **Default Service Endpoint**: `https://api.firecrawl.dev`
- **Self-hosted Option**: You can use your own Firecrawl instance
- Please refer to [Firecrawl's Privacy Policy](https://firecrawl.dev/privacy) for details on how your data is handled by Firecrawl services
## API Key Security
- **Required Authentication**: API key is required for service access
- **Secure Transmission**: API keys are transmitted securely via HTTPS
- **Session-Only Storage**: API keys are only stored for the duration of your session
- **No Key Logging**: The plugin does not log or permanently store your API key
- **Self-hosted Support**: For self-hosted instances, you control your own authentication
## Data Security
We are committed to ensuring the security of your data:
- All communications with Firecrawl services use secure HTTPS protocols
- Bearer token authentication ensures secure API access
- No crawled content is permanently stored by the plugin
- Data is transmitted directly between your system and Firecrawl services
- Idempotency keys can be used to prevent duplicate operations
## Content Access Scope
The plugin accesses only:
- Publicly available web pages at the URLs you specify
- Subpages within the specified crawl depth
- Content matching your include patterns (if specified)
- Content not matching your exclude patterns (if specified)
The plugin does NOT:
- Access password-protected or private content
- Store crawled data permanently
- Share crawled content with unauthorized parties
- Exceed the page limits you specify
- Access content beyond the specified depth
## User Rights
You have the right to:
- Control which websites are crawled
- Set crawl depth and page limits
- Define URL patterns to include or exclude
- Choose between cloud or self-hosted service
- Cancel crawling operations at any time
- Request only main content extraction
## Compliance
This plugin respects:
- Website robots.txt files and crawling policies
- Rate limiting and ethical crawling practices
- Copyright and intellectual property rights
- Data protection regulations
- Website terms of service
## Data Retention
- **No Long-term Storage**: The plugin does not retain crawled content after processing
- **Session Data**: Temporary data is cleared after each session
- **Job Status**: Crawling job status is tracked only during active operations
- **Cache**: No caching of crawled content is performed by the plugin
## Self-Hosted Deployments
For self-hosted Firecrawl instances:
- You have complete control over data processing
- Data never leaves your infrastructure
- You manage your own security and privacy policies
- The plugin respects your self-hosted instance's configuration
## Changes to This Policy
We may update this Privacy Policy from time to time. Any changes will be posted in this document with an updated effective date.
## Contact
If you have any questions or concerns about this Privacy Policy, please contact the developer [hello@dify.ai](mailto:hello@dify.ai) or refer to the project repository for more information.
Last updated: December 2024
datasources/firecrawl_datasource/README.md
+277
View File
@@ -0,0 +1,277 @@
# Firecrawl Datasource Plugin
**Author**: langgenius
**Version**: 0.2.3
**Type**: datasource (website_crawl)
## Introduction
This plugin integrates with Firecrawl, a powerful web scraping and crawling API service that recursively searches through URLs and subdomains to gather structured content. Firecrawl provides clean, LLM-ready data extraction with advanced filtering options, making it ideal for building knowledge bases, monitoring websites, and extracting structured information for AI applications like Dify.
## Features
- **Recursive Web Crawling**: Automatically crawl websites and their subpages
- **Depth Control**: Configure maximum crawl depth relative to the starting URL
- **Smart Content Extraction**: Extract only main content, excluding headers, footers, and navigation
- **URL Pattern Filtering**: Include or exclude specific URL patterns
- **Page Limit Control**: Set maximum number of pages to crawl
- **Real-time Progress Tracking**: Monitor crawl status and completion
- **Clean Markdown Output**: Convert web content to structured, LLM-friendly format
- **Self-Hosted Support**: Use Firecrawl cloud or your own instance
## Setup
### Prerequisites
Before using this plugin, you need:
1. A Firecrawl API key (for cloud service) or self-hosted Firecrawl instance
2. Target URLs ready for crawling
3. Understanding of your crawling requirements (depth, limits, patterns)
### Configuration Steps
#### Option 1: Using Firecrawl Cloud Service
1. **Get a Firecrawl API Key**:
- Visit [Firecrawl](https://firecrawl.dev)
- Sign up for an account
- Navigate to your [account settings](https://www.firecrawl.dev/account)
- Copy your API key
2. **Configure the Plugin in Dify**:
- Navigate to the datasource plugins section in Dify
- Select Firecrawl
- **Base URL**: Leave empty or enter `https://api.firecrawl.dev`
- **API Key**: Enter your Firecrawl API key
- Click "Save" to store the configuration
#### Option 2: Using Self-Hosted Firecrawl
1. **Deploy Firecrawl**:
- Follow the [self-hosting guide](https://docs.firecrawl.dev/self-host)
- Note your instance URL
2. **Configure the Plugin in Dify**:
- Navigate to the datasource plugins section in Dify
- Select Firecrawl
- **Base URL**: Enter your self-hosted Firecrawl URL (e.g., `https://firecrawl.your-domain.com`)
- **API Key**: Enter any key (required by the plugin but can be arbitrary for self-hosted)
- Click "Save" to store the configuration
## Usage
### Basic Single Page Extraction
To extract content from a single page without crawling subpages:
**Parameters:**
- **Start URL** (required): `https://example.com/article`
- **Crawl Subpages**: `false`
- **Maximum pages to crawl**: `1`
- **Only Main Content**: `true`
### Full Website Crawling
To crawl an entire website with subpages:
**Parameters:**
- **Start URL** (required): `https://example.com`
- **Crawl Subpages**: `true`
- **Maximum crawl depth**: `2` (crawls up to 2 levels deep)
- **Maximum pages to crawl**: `50`
- **Only Main Content**: `false`
### Targeted Section Crawling
To crawl only specific sections of a website:
**Parameters:**
- **Start URL**: `https://example.com`
- **Crawl Subpages**: `true`
- **URL patterns to include**: `blog/*, docs/*`
- **URL patterns to exclude**: `archive/*, tag/*`
- **Maximum pages to crawl**: `30`
## Parameters Explained
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
| `url` | string | Yes | - | The base URL to start crawling from |
| `crawl_subpages` | boolean | No | true | Whether to crawl subpages |
| `exclude_paths` | string | No | - | Comma-separated patterns to exclude (e.g., `blog/*, about/*`) |
| `include_paths` | string | No | - | Comma-separated patterns to include (e.g., `docs/*, api/*`) |
| `max_depth` | number | No | 2 | Maximum depth to crawl (0 = only start URL) |
| `limit` | number | No | 10 | Maximum number of pages to crawl |
| `only_main_content` | boolean | No | false | Extract only main content, excluding navigation elements |
### Understanding Crawl Depth
- **Depth 0**: Only the entered URL
- **Depth 1**: The entered URL + all directly linked pages
- **Depth 2**: The entered URL + directly linked pages + pages linked from those
- Higher values follow the same pattern
### URL Pattern Examples
**Include Patterns:**
- `blog/*` - Include all blog pages
- `docs/api/*` - Include API documentation
- `products/*/specs` - Include all product specification pages
**Exclude Patterns:**
- `admin/*` - Exclude admin pages
- `*.pdf` - Exclude PDF files
- `tag/*,category/*` - Exclude tag and category pages
## Output Format
The plugin returns structured data for each crawled page:
```json
{
"source_url": "https://example.com/page",
"title": "Page Title",
"description": "Meta description of the page",
"content": "Clean markdown formatted content..."
}
```
## How It Works
1. **Job Creation**: The plugin creates a crawl job with Firecrawl API
2. **Asynchronous Crawling**: Firecrawl processes the website based on your parameters
3. **Status Monitoring**: The plugin polls for job status every 5 seconds
4. **Content Processing**: Completed pages are formatted and structured
5. **Result Delivery**: Clean, structured content is returned
## Use Cases
### 1. Documentation Indexing
Crawl technical documentation sites for AI-powered search:
```
URL: https://docs.example.com
Include paths: api/*, guides/*
Max depth: 3
Limit: 100
```
### 2. Blog Content Extraction
Extract all blog posts for content analysis:
```
URL: https://blog.example.com
Include paths: posts/*, articles/*
Exclude paths: tag/*, author/*
Only main content: true
```
### 3. Product Catalog Building
Gather product information from e-commerce sites:
```
URL: https://shop.example.com/products
Max depth: 2
Limit: 50
Only main content: true
```
### 4. Competitor Monitoring
Track competitor website changes:
```
URL: https://competitor.com
Include paths: products/*, pricing/*
Exclude paths: blog/*, news/*
Max depth: 2
```
## Best Practices
1. **Start Small**: Begin with lower limits and depths for testing
2. **Use Pattern Filtering**: Focus crawling with include/exclude patterns
3. **Respect Robots.txt**: Ensure target sites allow crawling
4. **Monitor Progress**: Check crawl status for large operations
5. **Extract Main Content**: Use `only_main_content` for cleaner data
6. **Set Appropriate Limits**: Balance comprehensiveness with efficiency
7. **Test Patterns**: Verify your URL patterns match intended pages
## Performance Considerations
- **Large Sites**: May take several minutes to crawl
- **Deep Crawling**: Exponentially increases pages (be cautious with depth > 3)
- **Rate Limiting**: Firecrawl handles rate limiting automatically
- **Concurrent Jobs**: Multiple crawl jobs can run simultaneously
## Troubleshooting
### Common Issues
1. **"API key is required" error**:
- Verify your API key is correctly entered
- Check if using the correct base URL
2. **"Failed to crawl" error**:
- Check if the target URL is accessible
- Verify your API key is valid
- Ensure you haven't exceeded rate limits
3. **Incomplete crawling**:
- Some sites may block automated crawling
- JavaScript-heavy sites might not render fully
- Check if robots.txt restricts access
4. **Slow crawling**:
- Large sites naturally take longer
- Consider reducing depth or page limits
- Use pattern filtering to focus crawling
5. **Missing pages**:
- Verify include/exclude patterns are correct
- Check if pages are within specified depth
- Ensure limit hasn't been reached
6. **Self-hosted connection issues**:
- Verify base URL is correct and accessible
- Check firewall/network settings
- Ensure SSL certificates are valid
## API Limits
### Firecrawl Cloud
- Rate limits based on your plan
- Check your [account dashboard](https://www.firecrawl.dev/account) for usage
### Self-Hosted
- No external rate limits
- Performance depends on your infrastructure
## Security Considerations
- API keys are transmitted securely via HTTPS
- Use environment variables for API key storage in production
- For sensitive data, consider self-hosting
- Review crawled content for any inadvertently captured sensitive information
## Privacy
Please refer to the [Privacy Policy](PRIVACY.md) for information on how your data is handled when using this plugin.
## Support
For issues or questions:
- **Plugin Support**: [hello@dify.ai](mailto:hello@dify.ai)
- **Firecrawl Documentation**: [https://docs.firecrawl.dev](https://docs.firecrawl.dev)
- **Firecrawl Support**: Visit [Firecrawl](https://firecrawl.dev)
## Additional Resources
- [Firecrawl API Reference](https://docs.firecrawl.dev/api-reference)
- [Self-Hosting Guide](https://docs.firecrawl.dev/self-host)
- [Firecrawl GitHub](https://github.com/mendableai/firecrawl)
## Updates and Changelog
**Version 0.2.2** (Current)
- Enhanced pattern filtering
- Improved error handling
- Better progress tracking
- Self-hosted instance support
Last updated: December 2024
datasources/firecrawl_datasource/_assets/icon.svg
+17
View File
File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB

datasources/firecrawl_datasource/datasources/crawl.py
+108
View File
@@ -0,0 +1,108 @@
import time
from collections.abc import Generator
from typing import Any, Mapping
from dify_plugin.entities.datasource import (
WebSiteInfo,
WebSiteInfoDetail,
WebsiteCrawlMessage,
)
from dify_plugin.interfaces.datasource.website import WebsiteCrawlDatasource
from dify_plugin.errors.tool import ToolProviderCredentialValidationError
from requests import HTTPError
from datasources.firecrawl_app import FirecrawlApp, get_array_params, get_json_params
class CrawlDatasource(WebsiteCrawlDatasource):
def _get_website_crawl(
self, datasource_parameters: Mapping[str, Any]
) -> Generator[WebsiteCrawlMessage, None, None]:
"""
the api doc:
https://docs.firecrawl.dev/api-reference/endpoint/crawl
"""
source_url = datasource_parameters.get("url")
if not source_url:
raise ValueError("Url is required")
if not self.runtime.credentials.get("firecrawl_api_key"):
raise ToolProviderCredentialValidationError("api key is required")
try:
app = FirecrawlApp(
api_key=self.runtime.credentials.get("firecrawl_api_key"),
base_url=self.runtime.credentials.get("base_url")
or "https://api.firecrawl.dev",
)
crawl_sub_pages = datasource_parameters.get("crawl_subpages", True)
scrapeOptions = {
"onlyMainContent": datasource_parameters.get("only_main_content", True)
}
scrapeOptions = {
k: v for k, v in scrapeOptions.items() if v not in (None, "")
}
payload = {
"excludePaths": get_array_params(datasource_parameters, "exclude_paths")
if crawl_sub_pages
else [],
"includePaths": get_array_params(datasource_parameters, "include_paths")
if crawl_sub_pages
else [],
"maxDepth": datasource_parameters.get("max_depth")
if crawl_sub_pages
else None,
"limit": 1
if not crawl_sub_pages
else datasource_parameters.get("limit", 5),
"scrapeOptions": scrapeOptions or None,
}
payload = {k: v for k, v in payload.items() if v not in (None, "")}
crawl_res = WebSiteInfo(web_info_list=[], status="", total=0, completed=0)
_crawl_result = app.crawl_url(
url=datasource_parameters["url"], wait=False, **payload
)
job_id = _crawl_result["id"]
crawl_res.status = "processing"
print(crawl_res)
yield self.create_crawl_message(crawl_res)
while True:
status = app.check_crawl_status(job_id=job_id)
if status["status"] == "completed":
self._process_completed_job(app, status, crawl_res)
crawl_res.status = "completed"
crawl_res.total = status["total"] or 0
crawl_res.completed = status["completed"] or 0
yield self.create_crawl_message(crawl_res)
break
elif status["status"] == "failed":
raise HTTPError(f"Job {crawl_res.job_id} failed: {status['error']}")
else:
crawl_res.status = "processing"
crawl_res.total = status["total"] or 0
crawl_res.completed = status["completed"] or 0
yield self.create_crawl_message(crawl_res)
time.sleep(5)
except Exception as e:
raise ValueError(f"An error occurred: {str(e)}")
@staticmethod
def _process_completed_job(app: FirecrawlApp, status: dict, crawl_res: WebSiteInfo):
_format_res = app.format_crawl_status_response(status["status"], status)
crawl_res.web_info_list = [
WebSiteInfoDetail(
source_url=item["source_url"],
content=item["content"] or "",
title=item["title"] or "",
description=item["description"] or "",
)
for item in _format_res["data"]
]
datasources/firecrawl_datasource/datasources/crawl.yaml
+116
View File
@@ -0,0 +1,116 @@
description:
en_US: Recursively search through a urls subdomains, and gather the content.
zh_Hans: 递归爬取一个网址的子域名，并收集内容。
identity:
author: langgenius
label:
en_US: Firecrawl
zh_Hans: Firecrawl
name: crawl
parameters:
- form: llm
description:
en_US: The base URL to start crawling from.
zh_Hans: 要爬取网站的起始URL。
label:
en_US: Start URL
zh_Hans: 起始URL
name: url
required: true
type: string
- form: form
description:
en_US: is crawl subpages
zh_Hans: 是否爬取子页面
label:
zh_Hans: 爬取子页面
en_US: Crawl Subpages
default: true
name: crawl_subpages
placeholder:
zh_Hans: 是否爬取子页面
en_US: is crawl subpages
type: boolean
- form: form
description:
en_US: 'Pages matching these patterns will be skipped. Example: blog/*, about/*'
zh_Hans: 匹配这些模式的页面将被跳过。示例：blog/*, about/*
label:
en_US: URL patterns to exclude
zh_Hans: 排除路径
name: exclude_paths
placeholder:
en_US: Use commas to separate multiple tags
zh_Hans: 多个标签时使用半角逗号分隔
type: string
- form: form
description:
en_US: 'Only pages matching these patterns will be crawled. Example: blog/*, about/*'
zh_Hans: 只有与这些模式匹配的页面才会被爬取。示例：blog/*, about/*
label:
en_US: URL patterns to include
zh_Hans: 仅包含路径
name: include_paths
placeholder:
en_US: Use commas to separate multiple tags
zh_Hans: 多个标签时使用半角逗号分隔
required: false
type: string
- form: form
description:
en_US: Maximum depth to crawl relative to the entered URL. A maxDepth of 0 scrapes
only the entered URL. A maxDepth of 1 scrapes the entered URL and all pages
one level deep. A maxDepth of 2 scrapes the entered URL and all pages up to
two levels deep. Higher values follow the same pattern.
zh_Hans: 相对于输入的URL，爬取的最大深度。maxDepth为0时，仅抓取输入的URL。maxDepth为1时，抓取输入的URL以及所有一级深层页面。maxDepth为2时，抓取输入的URL以及所有两级深层页面。更高值遵循相同模式。
label:
en_US: Maximum crawl depth
zh_Hans: 最大深度
min: 0
name: max_depth
type: number
default: 2
- form: form
description:
en_US: Specify the maximum number of pages to crawl. The crawler will stop after
reaching this limit.
zh_Hans: 指定要爬取的最大页面数。爬虫将在达到此限制后停止。
label:
en_US: Maximum pages to crawl
zh_Hans: 限制数量
min: 1
name: limit
required: false
type: number
default: 10
- form: form
description:
en_US: Only return the main content of the page excluding headers, navs, footers,
etc.
zh_Hans: 只返回页面的主要内容，不包括头部、导航栏、尾部等。
label:
en_US: only Main Content
zh_Hans: 仅提取主要内容(无标题、导航、页脚等)
name: only_main_content
type: boolean
default: false
output_schema:
type: object
properties:
source_url:
type: string
description: the source url of the website
content:
type: string
description: the content from the website
title:
type: string
description: the title of the website
"description":
type: string
description: the description of the website
provider_type: website_crawl
extra:
python:
source: datasources/crawl.py
datasources/firecrawl_datasource/datasources/firecrawl_app.py
+157
View File
@@ -0,0 +1,157 @@
import json
import logging
import time
from collections.abc import Mapping
from typing import Any
import requests
from requests.exceptions import HTTPError
logger = logging.getLogger(__name__)
class FirecrawlApp:
def __init__(self, api_key: str | None = None, base_url: str | None = None):
self.api_key = api_key
self.base_url = base_url or "https://api.firecrawl.dev"
if not self.api_key:
raise ValueError("API key is required")
def _prepare_headers(self, idempotency_key: str | None = None):
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
if idempotency_key:
headers["Idempotency-Key"] = idempotency_key
return headers
def _request(
self,
method: str,
url: str,
data: Mapping[str, Any] | None = None,
headers: Mapping[str, str] | None = None,
retries: int = 3,
backoff_factor: float = 0.3,
) -> Mapping[str, Any] | None:
if not headers:
headers = self._prepare_headers()
for i in range(retries):
try:
response = requests.request(method, url, json=data, headers=headers)
return response.json()
except requests.exceptions.RequestException:
if i < retries - 1:
time.sleep(backoff_factor * (2**i))
else:
raise
return None
def scrape_url(self, url: str, **kwargs):
endpoint = f"{self.base_url}/v1/scrape"
data = {"url": url, **kwargs}
logger.debug(f"Sent request to {endpoint=} body={data}")
response = self._request("POST", endpoint, data)
if response is None:
raise HTTPError("Failed to scrape URL after multiple retries")
return response
def map(self, url: str, **kwargs):
endpoint = f"{self.base_url}/v1/map"
data = {"url": url, **kwargs}
logger.debug(f"Sent request to {endpoint=} body={data}")
response = self._request("POST", endpoint, data)
if response is None:
raise HTTPError("Failed to perform map after multiple retries")
return response
def crawl_url(
self,
url: str,
wait: bool = True,
poll_interval: int = 2,
idempotency_key: str | None = None,
**kwargs,
):
endpoint = f"{self.base_url}/v1/crawl"
headers = self._prepare_headers(idempotency_key)
data = {"url": url, **kwargs}
logger.debug(f"Sent request to {endpoint=} body={data}")
response = self._request("POST", endpoint, data, headers)
if response is None:
raise HTTPError("Failed to initiate crawl after multiple retries")
elif response.get("success") == False:
raise HTTPError(f"Failed to crawl: {response.get('error')}")
job_id: str = response["id"]
if wait:
return self._monitor_job_status(job_id=job_id, poll_interval=poll_interval)
return response
def check_crawl_status(self, job_id: str):
endpoint = f"{self.base_url}/v1/crawl/{job_id}"
response = self._request("GET", endpoint)
if response is None:
raise HTTPError(
f"Failed to check status for job {job_id} after multiple retries"
)
return response
def cancel_crawl_job(self, job_id: str):
endpoint = f"{self.base_url}/v1/crawl/{job_id}"
response = self._request("DELETE", endpoint)
if response is None:
raise HTTPError(f"Failed to cancel job {job_id} after multiple retries")
return response
def _monitor_job_status(self, job_id: str, poll_interval: int):
while True:
status = self.check_crawl_status(job_id)
if status["status"] == "completed":
status = self.format_crawl_status_response(status["status"], status)
return status
elif status["status"] == "failed":
raise HTTPError(f"Job {job_id} failed: {status['error']}")
time.sleep(poll_interval)
def format_crawl_status_response(
self, status: str, crawl_status_response: dict[str, Any]
) -> dict[str, Any]:
data = crawl_status_response.get("data", [])
url_data_list = []
for item in data:
if isinstance(item, dict) and "metadata" in item and "markdown" in item:
url_data = self._extract_common_fields(item)
url_data_list.append(url_data)
return {
"status": status,
"total": crawl_status_response.get("total"),
"current": crawl_status_response.get("completed"),
"data": url_data_list,
}
def _extract_common_fields(self, item: dict[str, Any]) -> dict[str, Any]:
return {
"title": item.get("metadata", {}).get("title"),
"description": item.get("metadata", {}).get("description"),
"source_url": item.get("metadata", {}).get("sourceURL"),
"content": item.get("markdown"),
}
def get_array_params(tool_parameters: dict[str, Any], key):
param = tool_parameters.get(key)
if param:
return param.split(",")
def get_json_params(tool_parameters: dict[str, Any], key):
param = tool_parameters.get(key)
if param:
try:
# support both single quotes and double quotes
param = param.replace("'", '"')
param = json.loads(param)
except Exception:
raise ValueError(f"Invalid {key} format.")
return param
datasources/firecrawl_datasource/main.py
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import Plugin, DifyPluginEnv
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/firecrawl_datasource/manifest.yaml
+43
View File
@@ -0,0 +1,43 @@
version: 0.2.3
type: plugin
author: langgenius
name: firecrawl_datasource
label:
en_US: Firecrawl
ja_JP: Firecrawl
zh_Hans: Firecrawl
pt_BR: Firecrawl
description:
en_US: Firecrawl Datasource
ja_JP: Firecrawl Datasource
zh_Hans: Firecrawl Datasource
pt_BR: Firecrawl Datasource
icon: icon.svg
resource:
memory: 268435456
permission:
model:
enabled: true
llm: false
text_embedding: false
rerank: false
tts: false
speech2text: false
moderation: false
plugins:
datasources:
- provider/firecrawl_datasource.yaml
meta:
version: 0.0.1
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-05-14T16:03:40.268524+08:00
privacy: PRIVACY.md
verified: false
provider_type: website_crawl
datasources/firecrawl_datasource/provider/firecrawl_datasource.py
+36
View File
@@ -0,0 +1,36 @@
from typing import Any, Mapping
import requests
from dify_plugin.errors.tool import ToolProviderCredentialValidationError
from dify_plugin.interfaces.datasource import DatasourceProvider
class FirecrawlDatasourceProvider(DatasourceProvider):
def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
try:
api_key = credentials.get("firecrawl_api_key", "")
if not api_key:
raise ToolProviderCredentialValidationError("api key is required")
base_url = credentials.get("base_url") or "https://api.firecrawl.dev"
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}",
}
payload = {
"url": "https://example.com",
"includePaths": [],
"excludePaths": [],
"limit": 1,
"scrapeOptions": {"onlyMainContent": True},
}
response = requests.post(
f"{base_url}/v1/crawl", json=payload, headers=headers
)
if response.status_code == 200:
return True
else:
raise ToolProviderCredentialValidationError("api key is invalid")
except Exception as e:
raise ToolProviderCredentialValidationError(str(e))
datasources/firecrawl_datasource/provider/firecrawl_datasource.yaml
+41
View File
@@ -0,0 +1,41 @@
credentials_schema:
- name: base_url
label:
en_US: Firecrawl server's Base URL
zh_Hans: Firecrawl服务器的API URL
placeholder:
en_US: https://api.firecrawl.dev
required: false
type: text-input
- name: firecrawl_api_key
help:
en_US: Get your Firecrawl API key from your Firecrawl account settings.If you
are using a self-hosted version, you may enter any key at your convenience.
zh_Hans: 从您的 Firecrawl 账户设置中获取 Firecrawl API 密钥。如果是自托管版本，可以随意填写密钥。
label:
en_US: Firecrawl API Key
zh_Hans: Firecrawl API 密钥
placeholder:
en_US: Please input your Firecrawl API key
zh_Hans: 请输入您的 Firecrawl API 密钥，如果是自托管版本，可以随意填写密钥
required: true
type: secret-input
url: https://www.firecrawl.dev/account
identity:
author: langgenius
name: firecrawl
label:
en_US: Firecrawl
zh_Hans: Firecrawl
pt_BR: Firecrawl
description:
en_US: Firecrawl Datasource
zh_Hans: Firecrawl Datasource
pt_BR: Firecrawl Datasource
icon: icon.svg
provider_type: website_crawl
datasources:
- datasources/crawl.yaml
extra:
python:
source: provider/firecrawl_datasource.py
datasources/firecrawl_datasource/provider/linuxdo.yaml
+68
View File
@@ -0,0 +1,68 @@
identity:
author: "frederick"
name: "linuxdo"
label:
en_US: "LinuxDo Connect"
zh_Hans: "LinuxDo Connect"
pt_BR: "LinuxDo Connect"
description:
en_US: "Connect to LinuxDo forum with authentication, user verification, content search, personalized recommendations, and auto check-in"
zh_Hans: "连接LinuxDo论坛，支持身份验证、用户信息获取、内容搜索、个性化推荐和自动签到"
pt_BR: "Conectar ao fórum LinuxDo com autenticação, verificação de usuário, busca de conteúdo, recomendações personalizadas e check-in automático"
icon: "icon.svg"
credentials_schema:
- name: "client_id"
type: "secret-input"
required: true
placeholder:
en_US: "Please input your LinuxDo Client ID"
zh_Hans: "请输入你的 LinuxDo Client ID"
pt_BR: "Insira seu LinuxDo Client ID"
help:
en_US: "Client ID is used for Basic Authorization to LinuxDo Connect API. Get it from https://connect.linux.do"
zh_Hans: "Client ID 用于 LinuxDo Connect API 的基础认证。请从 https://connect.linux.do 获取"
pt_BR: "Client ID é usado para autorização básica da API LinuxDo Connect. Obtenha em https://connect.linux.do"
label:
zh_Hans: "Client ID"
en_US: "Client ID"
pt_BR: "Client ID"
- name: "client_secret"
type: "secret-input"
required: true
placeholder:
en_US: "Please input your LinuxDo Client Secret"
zh_Hans: "请输入你的 LinuxDo Client Secret"
pt_BR: "Insira seu LinuxDo Client Secret"
help:
en_US: "Client Secret is used for Basic Authorization to LinuxDo Connect API. Get it from https://connect.linux.do"
zh_Hans: "Client Secret 用于 LinuxDo Connect API 的基础认证。请从 https://connect.linux.do 获取"
pt_BR: "Client Secret é usado para autorização básica da API LinuxDo Connect. Obtenha em https://connect.linux.do"
label:
zh_Hans: "Client Secret"
en_US: "Client Secret"
pt_BR: "Client Secret"
- name: "api_key"
type: "secret-input"
required: true
placeholder:
en_US: "Please input your LinuxDo API Key"
zh_Hans: "请输入你的 LinuxDo API Key"
pt_BR: "Insira sua LinuxDo API Key"
help:
en_US: "API Key is used to identify your user account. Get it from https://connect.linux.do"
zh_Hans: "API Key 用于识别你的用户账户。请从 https://connect.linux.do 获取"
pt_BR: "API Key é usado para identificar sua conta de usuário. Obtenha em https://connect.linux.do"
label:
zh_Hans: "API Key"
en_US: "API Key"
pt_BR: "API Key"
tools:
- tools/linuxdo.yaml
- tools/content_search.yaml
- tools/recommendations.yaml
- tools/checkin.yaml
extra:
python:
source: provider/linuxdo.py
datasources/firecrawl_datasource/requirements.txt
+1
View File
@@ -0,0 +1 @@
dify_plugin==0.5.0b14
datasources/github/.difyignore
+160
View File
@@ -0,0 +1,160 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be added to the global gitignore or merged into this project gitignore. For a PyCharm
# project, it is common to ignore these directories.
.idea/
datasources/github/GUIDE.md
+116
View File
@@ -0,0 +1,116 @@
# GitHub Datasource Plugin 使用指南
## 概述
GitHub 数据源插件允许您将 GitHub 仓库、Issues、Pull Requests 作为 Dify 的数据源。
## 认证方式
### 方式一：Personal Access Token（推荐）
1. 访问 [GitHub Settings > Developer settings > Personal access tokens](https://github.com/settings/tokens)
2. 点击 "Generate new token (classic)"
3. 选择以下权限：
- `repo` - 访问私有和公共仓库
- `user:email` - 获取用户邮箱
- `read:user` - 读取用户信息
4. 生成并复制 token（格式：`ghp_xxxxxxxxxxxxxxxxxxxx`）
5. 在 Dify 中配置数据源时粘贴此 token
### 方式二：OAuth 应用
1. 访问 [GitHub Settings > Developer settings > OAuth Apps](https://github.com/settings/developers)
2. 创建新的 OAuth App
3. 设置回调 URL：`https://your-dify-domain.com/console/api/oauth/callback`
4. 获取 Client ID 和 Client Secret
5. 在 Dify 中配置 OAuth 凭证
## 支持的内容类型
### 1. 仓库文件
- 代码文件（Python, JavaScript, etc.）
- 文档文件（Markdown, README）
- 配置文件（JSON, YAML）
- 其他文本文件
### 2. Issues
- Issue 标题和描述
- Issue 评论
- 标签和状态信息
### 3. Pull Requests
- PR 标题和描述
- PR 评论
- 分支信息
### 4. Wiki 页面
- Wiki 页面内容（计划支持）
## 使用步骤
1. **配置认证**：选择 Personal Access Token 或 OAuth 方式
2. **浏览仓库**：系统会列出您有权限访问的仓库
3. **选择内容**：选择要导入的文件、Issues 或 PR
4. **自动处理**：系统自动处理 Markdown 和其他格式
## 功能特性
### 自动内容处理
- Markdown 文件自动转换为 HTML
- 代码文件保持原格式
- 自动提取文件元数据
### 限流处理
- 自动检测 API 限流
- 智能重试机制
- 错误提示和等待建议
### 安全性
- 支持私有仓库访问
- 安全的 token 存储
- OAuth 标准流程
## 权限说明
### Personal Access Token 权限
- `repo`: 访问所有仓库（包括私有）
- `user:email`: 获取用户邮箱
- `read:user`: 读取基本用户信息
### OAuth 权限范围
- 仓库访问权限
- 用户信息读取权限
- Issues 和 PR 访问权限
## 常见问题
### Q: Token 无效怎么办？
A: 检查 token 是否正确复制，确保包含所需权限，token 未过期。
### Q: 遇到限流怎么办？
A: 系统会自动处理限流，请等待提示的时间后重试。
### Q: 无法访问私有仓库？
A: 确保 Personal Access Token 包含 `repo` 权限。
### Q: 支持哪些文件格式？
A: 支持所有文本格式，特别优化了 Markdown、JSON、YAML、代码文件等。
## 技术支持
如遇问题，请检查：
1. 网络连接是否正常
2. GitHub 服务状态
3. Token 权限是否足够
4. API 限流状态
## 版本信息
当前版本：v0.3.0
- ✅ Personal Access Token 认证
- ✅ OAuth 认证
- ✅ 仓库文件访问
- ✅ Issues 和 PR 支持
- ✅ 限流处理
- ✅ 内容自动处理
datasources/github/PRIVACY.md
+160
View File
@@ -0,0 +1,160 @@
# GitHub Datasource Plugin Privacy Policy
## Data Collection and Usage
### Information Collected
This plugin requires the following information to provide services:
#### Required Authentication Information
- **Personal Access Token** or **OAuth Access Token**: Used to authenticate with GitHub API
- **User Login**: GitHub username for identification
#### Optional Configuration Information
- OAuth client credentials (stored in system configuration only)
### Purpose of Information Usage
The collected information is only used for:
- Authenticating with GitHub API
- Accessing authorized repositories, issues, and pull requests
- Retrieving file content and metadata
- Providing datasource service functionality
## Data Access Scope
### Content We Access
- Repository information and metadata
- File content from authorized repositories
- Issue and pull request content and comments
- User profile information (name, avatar, login)
- Repository activity and statistics
### Content We Do Not Access
- Private repositories without explicit authorization
- Other users' private data
- GitHub system administration data
- Billing or payment information
## Data Storage and Security
### Local Storage
- Authentication tokens are securely encrypted and stored in the Dify platform
- No repository content is permanently stored
- Only necessary metadata is temporarily cached
### Transmission Security
- All communications with GitHub use HTTPS encryption
- Follows GitHub's API security standards
- Respects GitHub's rate limiting and usage policies
### Access Control
- Only authorized Dify users can access configured datasources
- Token permissions are limited to explicitly granted scopes
- Follows the principle of least privilege
## Third-Party Services
### GitHub
- This plugin communicates directly with GitHub's REST API
- Follows GitHub's Terms of Service and Privacy Policy
- GitHub Privacy Policy: https://docs.github.com/en/site-policy/privacy-policies/github-privacy-statement
### Data Location
- Data remains within GitHub's infrastructure
- No data transfer to unauthorized third parties
- Follows GitHub's data residency policies
## User Rights
### Data Control
- You retain full control over all data in your GitHub repositories
- Can revoke plugin access permissions at any time
- Can delete or modify stored authentication information
### Access and Deletion
- Can view the scope of data accessed by the plugin
- Can disable or delete datasource configuration at any time
- Deleting configuration will clear all related authentication information
## Security Best Practices
### Token Security
- Use Personal Access Tokens with minimal required scopes
- Regularly rotate access tokens
- Monitor token usage in GitHub settings
- Revoke unused or suspicious tokens
### Permission Management
- Grant only necessary repository access permissions
- Use fine-grained personal access tokens when available
- Regularly review granted permissions
- Monitor access logs in GitHub audit trail
## Rate Limiting and Usage
### API Usage
- Respects GitHub API rate limits
- Implements automatic retry with backoff
- Does not abuse GitHub's infrastructure
- Follows GitHub's API usage guidelines
### Monitoring
- Tracks API usage to prevent rate limit violations
- Logs access patterns for debugging purposes
- Alerts users when rate limits are approached
## Compliance
### Data Protection Regulations
- Supports GDPR compliance for EU users
- Follows applicable data protection laws
- Supports data subject rights requests
### Industry Standards
- Follows industry security best practices
- Implements secure coding standards
- Regular security reviews and updates
## Incident Response
### Security Incidents
If security incidents are discovered, we will:
- Immediately investigate and assess impact
- Promptly notify affected users
- Take measures to prevent further data exposure
- Cooperate with GitHub for investigation if needed
### Token Compromise
If token compromise is detected:
- Immediately revoke affected tokens
- Notify users to regenerate tokens
- Review access logs for unauthorized activity
- Update security measures as needed
## Contact Information
### Privacy Issues
For privacy-related questions, please contact:
- Email: privacy@dify.ai
- Address: Dify Data Protection Officer
### Technical Support
For technical issues, please contact:
- Email: support@dify.ai
- Documentation: https://docs.dify.ai
## Policy Updates
### Update Notifications
- Privacy policy changes will be notified to users in advance
- Major changes require explicit user consent
- Update history is maintained in this document
### Version History
- **v1.0** (2025-01-27): Initial comprehensive privacy policy
- Complete GitHub API integration privacy protection
- Clear scope of data collection and usage
---
Last Updated: January 27, 2025
Version: 1.0
datasources/github/README.md
+246
View File
@@ -0,0 +1,246 @@
# GitHub Datasource Plugin
Access GitHub repositories, issues, pull requests, and wiki pages as a datasource for Dify with comprehensive authentication support.
## Features
- **Repository Access**: Browse and download files from public and private repositories
- **Issues & Pull Requests**: Access issue and PR content with comments
- **Multiple Authentication**: Support both Personal Access Token and OAuth
- **Rate Limit Handling**: Automatic rate limit detection and handling
- **Content Processing**: Automatic markdown processing and content extraction
- **Multi-Content Types**: Support for various file formats and GitHub content types
## Supported Content Types
- Repository files (Markdown, code, documentation)
- GitHub Issues with comments
- Pull Requests with comments
- Various file formats (JSON, YAML, Python, JavaScript, etc.)
- README files and repository metadata
## Setup and Installation
### Requirements
- Dify platform version >= 2.0.0
- Python 3.12+
- Valid GitHub account with appropriate permissions
### Installation
1. Install the plugin in your Dify instance
2. Configure authentication credentials (see Authentication section below)
3. Test the connection and start using GitHub as a datasource
## Authentication
### Option 1: Personal Access Token (Recommended for Development)
1. Go to GitHub Settings > Developer settings > Personal access tokens
2. Click "Generate new token (classic)"
3. Select the following scopes:
- `repo` - Full control of private repositories
- `user:email` - Access user email addresses
- `read:user` - Read user profile data
4. Copy the generated token
5. In Dify, configure the datasource with your token
**Example Configuration:**
```yaml
access_token: "ghp_xxxxxxxxxxxxxxxxxxxx"
```
### Option 2: OAuth (Recommended for Production)
1. Create a GitHub OAuth App:
- Go to GitHub Settings > Developer settings > OAuth Apps
- Click "New OAuth App"
- Fill in application details:
- Application name: "Dify GitHub Integration"
- Homepage URL: "https://your-dify-domain.com"
- Authorization callback URL: "https://your-dify-domain.com/console/api/oauth/callback"
2. Note the Client ID and Client Secret
3. Configure in Dify system settings
4. Users can then authorize through OAuth flow
**System Configuration Example:**
```yaml
client_id: "your_github_client_id"
client_secret: "your_github_client_secret"
```
## Usage Workflows
### 1. Repository Content Access
1. Add GitHub datasource to your Dify knowledge base
2. The plugin will automatically discover your accessible repositories
3. Select repositories, files, issues, or PRs to include
4. Content is automatically processed and indexed
### 2. Issue Tracking Integration
1. Configure GitHub datasource with appropriate permissions
2. Issues and PRs are automatically discovered and indexed
3. Search and query issue content using Dify's AI capabilities
4. Get insights from issue discussions and resolutions
### 3. Documentation Management
1. Connect repository with extensive documentation
2. README files and markdown documents are automatically processed
3. Create AI-powered documentation search and Q&A
4. Keep documentation knowledge base synchronized
## Configuration Examples
### Basic Datasource Configuration
```yaml
# datasources/github.yaml
name: github_datasource
type: online_document
provider: github_datasource
config:
repositories:
- "owner/repository-name"
include_issues: true
include_prs: true
max_repositories: 50
```
### Provider Configuration
```yaml
# provider/github.yaml
identity:
name: github_datasource
author: langgenius
label:
en_US: GitHub
provider_type: online_document
credentials_schema:
- name: access_token
type: secret-input
required: true
label:
en_US: Personal Access Token
oauth_schema:
client_schema:
- name: client_id
type: secret-input
label:
en_US: Client ID
- name: client_secret
type: secret-input
label:
en_US: Client Secret
```
## Rate Limits and Performance
### GitHub API Limits
- **Personal Access Token**: 5,000 requests per hour
- **OAuth**: 5,000 requests per hour per user
- **GitHub Apps**: Higher limits available
### Optimization Features
- Automatic rate limit detection and handling
- Intelligent request batching
- Content caching to reduce API calls
- Graceful degradation when limits are reached
## Troubleshooting
### Common Issues
#### "Invalid access token" Error
**Problem**: Authentication fails with token error
**Solution**:
1. Verify token is valid and not expired
2. Check token has required scopes (`repo`, `user:email`, `read:user`)
3. Ensure token hasn't been revoked
4. Generate a new token if needed
#### "Rate limit exceeded" Error
**Problem**: Too many API requests in short time
**Solution**:
1. Wait for rate limit reset (shown in error message)
2. Reduce the number of repositories being accessed
3. Consider using GitHub Apps for higher limits
4. Implement request batching in your usage
#### "Repository not found" Error
**Problem**: Cannot access specific repository
**Solution**:
1. Verify repository name is correct (case-sensitive)
2. Check if repository is private and token has access
3. Ensure token has `repo` scope for private repositories
4. Verify you have read access to the repository
### Debug Mode
Enable debug logging to troubleshoot issues:
```python
import logging
logging.getLogger('datasources.github').setLevel(logging.DEBUG)
```
## FAQ
### Q: Can I access private repositories?
**A**: Yes, with proper Personal Access Token with `repo` scope or OAuth authorization.
### Q: Are GitHub Enterprise repositories supported?
**A**: Currently supports GitHub.com only. Enterprise support may be added in future versions.
### Q: How often is content synchronized?
**A**: Content is fetched in real-time when accessed. No background synchronization is performed.
### Q: What happens if my token expires?
**A**: Personal Access Tokens don't expire unless manually revoked. OAuth tokens are handled automatically.
### Q: Can I limit which repositories are accessible?
**A**: Yes, you can configure specific repositories in the datasource configuration.
## Security Considerations
### Token Management
- Store tokens securely using Dify's encrypted credential storage
- Regularly rotate Personal Access Tokens
- Monitor token usage in GitHub settings
- Use minimal required scopes
### Data Privacy
- Plugin only accesses explicitly authorized content
- No data is stored outside of Dify platform
- All API communications are encrypted with HTTPS
- Follows GitHub's data handling policies
## Support and Community
### Documentation
- [Dify Documentation](https://docs.dify.ai)
- [GitHub API Documentation](https://docs.github.com/en/rest)
### Issues and Feature Requests
- Report issues through Dify support channels
- Feature requests can be submitted via Dify community
### Version History
- **v0.3.0**: Complete OAuth support, Issue/PR access
- **v0.2.0**: Enhanced authentication methods
- **v0.1.0**: Basic repository file access
## Version: 0.3.0
This plugin implements all features up to v0.3.0 including comprehensive OAuth support, advanced content access, and enhanced security measures.
datasources/github/_assets/icon.svg
+5
View File
@@ -0,0 +1,5 @@
<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="40" height="40" fill="white"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M20 8C13.3726 8 8 13.3726 8 20C8 25.3016 11.438 29.7998 16.2074 31.386C16.8074 31.4946 17.025 31.1306 17.025 30.8186C17.025 30.5386 17.0158 29.718 17.01 28.618C13.672 29.3506 12.967 26.9586 12.967 26.9586C12.421 25.582 11.634 25.2206 11.634 25.2206C10.545 24.4726 11.717 24.4886 11.717 24.4886C12.922 24.5746 13.555 25.7186 13.555 25.7186C14.625 27.5586 16.364 27.0186 17.047 26.7186C17.157 25.9426 17.467 25.4026 17.81 25.1106C15.145 24.8146 12.343 23.7586 12.343 19.2786C12.343 18.0066 12.812 16.9666 13.579 16.1506C13.455 15.8546 13.043 14.6146 13.695 12.9586C13.695 12.9586 14.704 12.6426 16.995 14.1906C17.954 13.9306 18.98 13.8006 20 13.7946C21.02 13.8006 22.047 13.9306 23.006 14.1906C25.296 12.6426 26.304 12.9586 26.304 12.9586C26.956 14.6146 26.544 15.8546 26.42 16.1506C27.188 16.9666 27.654 18.0066 27.654 19.2786C27.654 23.7706 24.848 24.8106 22.177 25.0986C22.607 25.4706 22.99 26.2026 22.99 27.3226C22.99 28.9306 22.975 30.2306 22.975 30.8186C22.975 31.1326 23.19 31.4986 23.797 31.3846C28.563 29.7966 32 25.3006 32 20C32 13.3726 26.6274 8 20 8Z" fill="#24292e"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

datasources/github/datasources/github.py
+349
View File
@@ -0,0 +1,349 @@
from collections.abc import Generator
import requests
import time
import base64
import markdown
from typing import Dict, List, Optional, Any
from urllib.parse import urlparse
from dify_plugin.entities.datasource import (
DatasourceGetPagesResponse,
DatasourceMessage,
GetOnlineDocumentPageContentRequest,
OnlineDocumentInfo,
)
from dify_plugin.interfaces.datasource.online_document import OnlineDocumentDatasource
class GitHubDataSource(OnlineDocumentDatasource):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.base_url = "https://api.github.com"
def _get_headers(self) -> Dict[str, str]:
"""Get API request headers"""
credentials = self.runtime.credentials
access_token = credentials.get("access_token")
return {
"Authorization": f"token {access_token}",
"Accept": "application/vnd.github.v3+json",
"User-Agent": "Dify-GitHub-Datasource"
}
def _handle_rate_limit(self, response: requests.Response) -> None:
"""Handle API rate limiting"""
if response.status_code == 403:
rate_limit_remaining = response.headers.get("X-RateLimit-Remaining", "0")
if rate_limit_remaining == "0":
reset_time = int(response.headers.get("X-RateLimit-Reset", "0"))
current_time = int(time.time())
sleep_time = max(reset_time - current_time + 1, 60)
raise ValueError(f"GitHub API rate limit exceeded. Please wait {sleep_time} seconds.")
elif response.status_code == 401:
raise ValueError("Invalid GitHub access token. Please check your credentials.")
elif response.status_code >= 400:
raise ValueError(f"GitHub API error: {response.status_code} - {response.text}")
def _make_request(self, url: str, params: Optional[Dict] = None) -> Dict:
"""Make API request and handle errors"""
headers = self._get_headers()
response = requests.get(url, headers=headers, params=params, timeout=30)
self._handle_rate_limit(response)
return response.json()
def _get_pages(self, datasource_parameters: dict[str, Any]) -> DatasourceGetPagesResponse:
"""Get GitHub page list (repositories, Issues, PRs)"""
access_token = self.runtime.credentials.get("access_token")
if not access_token:
raise ValueError("Access token not found in credentials")
# Get user information
user_info = self._make_request(f"{self.base_url}/user")
workspace_name = f"{user_info.get('name', user_info.get('login'))}'s GitHub"
workspace_icon = user_info.get('avatar_url', '')
workspace_id = str(user_info.get('id', ''))
pages = []
# Get user repositories
repos = self._get_repositories()
for repo in repos:
# Add repository as page
pages.append({
"page_id": f"repo:{repo['full_name']}",
"page_name": repo['name'],
"last_edited_time": repo.get("updated_at", ""),
"type": "repository",
"url": repo['html_url'],
"metadata": {
"description": repo.get("description", ""),
"language": repo.get("language", ""),
"stars": repo.get("stargazers_count", 0),
"updated_at": repo.get("updated_at", ""),
"private": repo.get("private", False)
}
})
# Add README file (if exists)
try:
readme_info = self._make_request(f"{self.base_url}/repos/{repo['full_name']}/readme")
pages.append({
"page_id": f"file:{repo['full_name']}:README.md",
"page_name": f"{repo['name']} - README",
"last_edited_time": repo.get("updated_at", ""),
"type": "file",
"url": readme_info.get('html_url', ''),
"metadata": {
"repository": repo['full_name'],
"file_path": "README.md",
"size": readme_info.get('size', 0)
}
})
except ValueError:
pass # README doesn't exist
# Add popular Issues
try:
issues = self._make_request(
f"{self.base_url}/repos/{repo['full_name']}/issues",
params={"state": "all", "per_page": 5, "sort": "updated"}
)
for issue in issues:
if "pull_request" not in issue: # Exclude PRs
pages.append({
"page_id": f"issue:{repo['full_name']}:{issue['number']}",
"page_name": f"Issue #{issue['number']}: {issue['title']}",
"last_edited_time": issue.get('updated_at', ''),
"type": "issue",
"url": issue['html_url'],
"metadata": {
"repository": repo['full_name'],
"issue_number": issue['number'],
"state": issue['state'],
"author": issue['user']['login'],
"created_at": issue['created_at']
}
})
except ValueError:
pass # Issues access failed
# Add popular PRs
try:
prs = self._make_request(
f"{self.base_url}/repos/{repo['full_name']}/pulls",
params={"state": "all", "per_page": 5, "sort": "updated"}
)
for pr in prs:
pages.append({
"page_id": f"pr:{repo['full_name']}:{pr['number']}",
"page_name": f"PR #{pr['number']}: {pr['title']}",
"last_edited_time": pr.get('updated_at', ''),
"type": "pull_request",
"url": pr['html_url'],
"metadata": {
"repository": repo['full_name'],
"pr_number": pr['number'],
"state": pr['state'],
"author": pr['user']['login'],
"base_branch": pr['base']['ref'],
"head_branch": pr['head']['ref']
}
})
except ValueError:
pass # PRs access failed
online_document_info = OnlineDocumentInfo(
workspace_name=workspace_name,
workspace_icon=workspace_icon,
workspace_id=workspace_id,
pages=pages,
total=len(pages),
)
return DatasourceGetPagesResponse(result=[online_document_info])
def _get_repositories(self, max_repos: int = 20) -> List[Dict]:
"""Get user repository list"""
params = {"per_page": max_repos, "sort": "updated", "direction": "desc"}
repos = self._make_request(f"{self.base_url}/user/repos", params)
return repos
def _get_content(self, page: GetOnlineDocumentPageContentRequest) -> Generator[DatasourceMessage, None, None]:
"""Get page content"""
access_token = self.runtime.credentials.get("access_token")
if not access_token:
raise ValueError("Access token not found in credentials")
page_id = page.page_id
if page_id.startswith("repo:"):
# Get repository information
yield from self._get_repository_content(page_id)
elif page_id.startswith("file:"):
# Get file content
yield from self._get_file_content(page_id)
elif page_id.startswith("issue:"):
# Get Issue content
yield from self._get_issue_content(page_id)
elif page_id.startswith("pr:"):
# Get PR content
yield from self._get_pr_content(page_id)
else:
raise ValueError(f"Unsupported page type: {page_id}")
def _get_repository_content(self, page_id: str) -> Generator[DatasourceMessage, None, None]:
"""Get repository information content"""
repo_name = page_id[5:] # Remove "repo:" prefix
repo_info = self._make_request(f"{self.base_url}/repos/{repo_name}")
content = f"# {repo_info['name']}\n\n"
content += f"**Repository:** {repo_info['full_name']}\n"
content += f"**Description:** {repo_info.get('description', 'No description')}\n"
content += f"**Language:** {repo_info.get('language', 'Not specified')}\n"
content += f"**Stars:** {repo_info.get('stargazers_count', 0)}\n"
content += f"**Forks:** {repo_info.get('forks_count', 0)}\n"
content += f"**Created:** {repo_info.get('created_at', '')}\n"
content += f"**Updated:** {repo_info.get('updated_at', '')}\n"
content += f"**URL:** {repo_info.get('html_url', '')}\n\n"
if repo_info.get('topics'):
topics = ", ".join(repo_info['topics'])
content += f"**Topics:** {topics}\n\n"
# Try to get README
try:
readme_info = self._make_request(f"{self.base_url}/repos/{repo_name}/readme")
if readme_info.get("encoding") == "base64":
readme_content = base64.b64decode(readme_info["content"]).decode("utf-8")
content += "## README\n\n" + readme_content
except ValueError:
content += "## README\n\nNo README file found."
yield self.create_variable_message("content", content)
yield self.create_variable_message("page_id", page_id)
yield self.create_variable_message("title", repo_info['name'])
yield self.create_variable_message("repository", repo_name)
yield self.create_variable_message("type", "repository")
def _get_file_content(self, page_id: str) -> Generator[DatasourceMessage, None, None]:
"""Get file content"""
# page_id format: "file:owner/repo:path"
parts = page_id.split(":", 2)
repo_name = parts[1]
file_path = parts[2]
file_info = self._make_request(f"{self.base_url}/repos/{repo_name}/contents/{file_path}")
if file_info.get("type") != "file":
raise ValueError("Can only get content for files, not directories")
# Get file content
if file_info.get("encoding") == "base64":
content = base64.b64decode(file_info["content"]).decode("utf-8")
else:
download_url = file_info.get("download_url")
if download_url:
response = requests.get(download_url, timeout=30)
response.raise_for_status()
content = response.text
else:
content = file_info.get("content", "")
# If it's a Markdown file, add title
file_name = file_info["name"]
if file_name.lower().endswith(('.md', '.markdown')):
content = f"# {file_name}\n\n{content}"
yield self.create_variable_message("content", content)
yield self.create_variable_message("page_id", page_id)
yield self.create_variable_message("title", file_name)
yield self.create_variable_message("repository", repo_name)
yield self.create_variable_message("file_path", file_path)
yield self.create_variable_message("type", "file")
def _get_issue_content(self, page_id: str) -> Generator[DatasourceMessage, None, None]:
"""Get Issue content"""
# page_id format: "issue:owner/repo:number"
parts = page_id.split(":", 2)
repo_name = parts[1]
issue_number = parts[2]
issue = self._make_request(f"{self.base_url}/repos/{repo_name}/issues/{issue_number}")
content = f"# Issue #{issue['number']}: {issue['title']}\n\n"
content += f"**Repository:** {repo_name}\n"
content += f"**Author:** {issue['user']['login']}\n"
content += f"**State:** {issue['state']}\n"
content += f"**Created:** {issue['created_at']}\n"
content += f"**Updated:** {issue['updated_at']}\n"
content += f"**URL:** {issue['html_url']}\n\n"
if issue.get('labels'):
labels = ", ".join([label['name'] for label in issue['labels']])
content += f"**Labels:** {labels}\n\n"
if issue.get('body'):
content += "## Description\n\n"
content += issue['body'] + "\n\n"
# Get comments
try:
comments = self._make_request(f"{self.base_url}/repos/{repo_name}/issues/{issue_number}/comments")
if comments:
content += "## Comments\n\n"
for comment in comments:
content += f"### {comment['user']['login']} - {comment['created_at']}\n\n"
content += comment['body'] + "\n\n"
except ValueError:
pass
yield self.create_variable_message("content", content)
yield self.create_variable_message("page_id", page_id)
yield self.create_variable_message("title", f"Issue #{issue['number']}: {issue['title']}")
yield self.create_variable_message("repository", repo_name)
yield self.create_variable_message("issue_number", issue_number)
yield self.create_variable_message("type", "issue")
def _get_pr_content(self, page_id: str) -> Generator[DatasourceMessage, None, None]:
"""Get PR content"""
# page_id format: "pr:owner/repo:number"
parts = page_id.split(":", 2)
repo_name = parts[1]
pr_number = parts[2]
pr = self._make_request(f"{self.base_url}/repos/{repo_name}/pulls/{pr_number}")
content = f"# Pull Request #{pr['number']}: {pr['title']}\n\n"
content += f"**Repository:** {repo_name}\n"
content += f"**Author:** {pr['user']['login']}\n"
content += f"**State:** {pr['state']}\n"
content += f"**Base Branch:** {pr['base']['ref']}\n"
content += f"**Head Branch:** {pr['head']['ref']}\n"
content += f"**Created:** {pr['created_at']}\n"
content += f"**Updated:** {pr['updated_at']}\n"
content += f"**URL:** {pr['html_url']}\n\n"
if pr.get('body'):
content += "## Description\n\n"
content += pr['body'] + "\n\n"
# Get comments
try:
comments = self._make_request(f"{self.base_url}/repos/{repo_name}/issues/{pr_number}/comments")
if comments:
content += "## Comments\n\n"
for comment in comments:
content += f"### {comment['user']['login']} - {comment['created_at']}\n\n"
content += comment['body'] + "\n\n"
except ValueError:
pass
yield self.create_variable_message("content", content)
yield self.create_variable_message("page_id", page_id)
yield self.create_variable_message("title", f"PR #{pr['number']}: {pr['title']}")
yield self.create_variable_message("repository", repo_name)
yield self.create_variable_message("pr_number", pr_number)
yield self.create_variable_message("type", "pull_request")
datasources/github/datasources/github.yaml
+29
View File
@@ -0,0 +1,29 @@
identity:
name: github
author: langgenius
label:
en_US: GitHub Repository
zh_Hans: GitHub 仓库
icon: icon.svg
description:
en_US: Access GitHub repositories, issues, pull requests, and wiki pages
zh_Hans: 访问 GitHub 仓库、问题、拉取请求和 Wiki 页面
parameters:
output_schema:
type: object
properties:
page_id:
type: string
description: page id
content:
type: string
description: page content
workspace_id:
type: string
description: workspace id
extra:
python:
source: datasources/github.py
datasources/github/main.py
+6
View File
@@ -0,0 +1,6 @@
from dify_plugin import DifyPluginEnv, Plugin
plugin = Plugin(DifyPluginEnv(MAX_REQUEST_TIMEOUT=120))
if __name__ == "__main__":
plugin.run()
datasources/github/manifest.yaml
+31
View File
@@ -0,0 +1,31 @@
version: 0.3.1
type: plugin
author: langgenius
name: github_datasource
label:
en_US: GitHub
zh_Hans: GitHub
description:
en_US: GitHub Repository Datasource - Access repositories, issues, PRs, and wiki pages
zh_Hans: GitHub 仓库数据源 - 访问仓库、问题、拉取请求和 Wiki 页面
icon: icon.svg
resource:
memory: 268435456
permission: {}
plugins:
datasources:
- provider/github.yaml
meta:
version: 0.3.0
arch:
- amd64
- arm64
runner:
language: python
version: "3.12"
entrypoint: main
minimum_dify_version: 1.7.2
created_at: 2025-01-27T10:00:00.000000+08:00
privacy: PRIVACY.md
verified: false
datasources/github/provider/github.py
+105
View File
@@ -0,0 +1,105 @@
from typing import Any, Mapping
import requests
import urllib.parse
from flask import Request
from dify_plugin.interfaces.datasource import DatasourceProvider, DatasourceOAuthCredentials
class GitHubDatasourceProvider(DatasourceProvider):
_AUTH_URL = "https://github.com/login/oauth/authorize"
_TOKEN_URL = "https://github.com/login/oauth/access_token"
_USERINFO_URL = "https://api.github.com/user"
def _validate_credentials(self, credentials: Mapping[str, Any]) -> None:
"""Validate credentials"""
access_token = credentials.get("access_token")
if not access_token:
raise ValueError("Access token is required")
# Validate token validity
headers = {
"Authorization": f"token {access_token}",
"Accept": "application/vnd.github.v3+json",
"User-Agent": "Dify-GitHub-Datasource"
}
try:
response = requests.get(self._USERINFO_URL, headers=headers, timeout=10)
if response.status_code == 401:
raise ValueError("Invalid access token")
elif response.status_code >= 400:
raise ValueError(f"GitHub API error: {response.status_code} {response.text}")
except requests.exceptions.RequestException as e:
raise ValueError(f"Failed to validate GitHub token: {str(e)}")
def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str:
"""Get OAuth authorization URL"""
scopes = [
"repo", # Access private and public repositories
"user:email", # Get user email
"read:user", # Read user information
]
params = {
"client_id": system_credentials["client_id"],
"redirect_uri": redirect_uri,
"scope": " ".join(scopes),
"response_type": "code",
}
return f"{self._AUTH_URL}?{urllib.parse.urlencode(params)}"
def _oauth_get_credentials(
self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request
) -> DatasourceOAuthCredentials:
"""Handle OAuth callback and get credentials"""
code = request.args.get("code")
if not code:
raise ValueError("No authorization code provided")
# Exchange access token
token_data = {
"client_id": system_credentials["client_id"],
"client_secret": system_credentials["client_secret"],
"code": code,
"redirect_uri": redirect_uri,
}
headers = {
"Accept": "application/json",
"User-Agent": "Dify-GitHub-Datasource"
}
token_response = requests.post(self._TOKEN_URL, data=token_data, headers=headers, timeout=15)
if token_response.status_code >= 400:
raise ValueError(f"GitHub token exchange error: {token_response.status_code} {token_response.text}")
token_json = token_response.json()
access_token = token_json.get("access_token")
if not access_token:
raise ValueError(f"Error in GitHub OAuth token exchange: {token_json}")
# Get user information
userinfo_headers = {
"Authorization": f"token {access_token}",
"Accept": "application/vnd.github.v3+json",
"User-Agent": "Dify-GitHub-Datasource"
}
userinfo_resp = requests.get(self._USERINFO_URL, headers=userinfo_headers, timeout=10)
if userinfo_resp.status_code >= 400:
raise ValueError(f"Failed to get GitHub user info: {userinfo_resp.status_code}")
user = userinfo_resp.json()
return DatasourceOAuthCredentials(
name=user.get("name") or user.get("login"),
avatar_url=user.get("avatar_url"),
credentials={
"access_token": access_token,
"user_login": user.get("login"),
},
)
def _refresh_access_token(self, credentials: Mapping[str, Any]) -> Mapping[str, Any]:
"""Refresh access token - GitHub doesn't support refresh token, return original credentials"""
# GitHub OAuth tokens don't expire, no need to refresh
# If token is invalid, re-authorization is required
return credentials

Some files were not shown because too many files have changed in this diff Show More