mirror of
https://github.com/langgenius/dify-plugin-sdks.git
synced 2026-08-26 18:56:41 -04:00
Yeuoly
8f5e27732c
- Introduced `OnlineDocumentDatasource` and `WebsiteCrawlDatasource` abstract classes for handling online document and website crawl functionalities. - Added corresponding methods for retrieving pages and content, enhancing the datasource capabilities. - Created new models for online document and website information to support the new interfaces.
44 lines
1.2 KiB
Python
44 lines
1.2 KiB
Python
from abc import ABC, abstractmethod
|
|
from collections.abc import Mapping
|
|
from typing import Any, final
|
|
|
|
from dify_plugin.core.runtime import Session
|
|
from dify_plugin.entities.datasource import DatasourceRuntime, GetWebsiteCrawlResponse
|
|
|
|
|
|
class WebsiteCrawlDatasource(ABC):
|
|
"""
|
|
Website Crawl Datasource abstract class
|
|
"""
|
|
|
|
runtime: DatasourceRuntime
|
|
session: Session
|
|
|
|
@final
|
|
def __init__(
|
|
self,
|
|
runtime: DatasourceRuntime,
|
|
session: Session,
|
|
):
|
|
"""
|
|
Initialize the datasource
|
|
|
|
NOTE:
|
|
- This method has been marked as final, DO NOT OVERRIDE IT.
|
|
"""
|
|
self.runtime = runtime
|
|
self.session = session
|
|
|
|
def get_website_crawl(self, datasource_parameters: Mapping[str, Any]) -> GetWebsiteCrawlResponse:
|
|
"""
|
|
Get the website crawl result
|
|
"""
|
|
return self._get_website_crawl(datasource_parameters)
|
|
|
|
@abstractmethod
|
|
def _get_website_crawl(self, datasource_parameters: Mapping[str, Any]) -> GetWebsiteCrawlResponse:
|
|
"""
|
|
Get the website crawl result
|
|
"""
|
|
raise NotImplementedError("This method should be implemented by a subclass")
|