baidu_search #232

New Issue

2026-02-22T17:22:52-05:00

yindo commented

2026-02-22 17:22:52 -05:00

Originally created by @cbangzan on GitHub (Aug 20, 2025).

Plugin Name

baidu_search

Function Description

import requests
from bs4 import BeautifulSoup
import urllib.parse
import time
import re
from urllib.robotparser import RobotFileParser
import json

class BaiduSearchCrawler:
def init(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
self.session = requests.Session()
self.session.headers.update(self.headers)

def check_robots_txt(self, url):
    """检查目标网站的robots.txt"""
    try:
        base_url = f"{urllib.parse.urlparse(url).scheme}://{urllib.parse.urlparse(url).netloc}"
        rp = RobotFileParser()
        rp.set_url(f"{base_url}/robots.txt")
        rp.read()
        return rp.can_fetch(self.headers['User-Agent'], url)
    except:
        return True  # 如果无法获取robots.txt，默认允许爬取

def get_baidu_results(self, keyword, num_results=5):
    """从百度搜索获取结果"""
    encoded_keyword = urllib.parse.quote(keyword)
    url = f"https://www.baidu.com/s?wd={encoded_keyword}"

    try:
        response = self.session.get(url, timeout=10)
        response.raise_for_status()

        soup = BeautifulSoup(response.text, 'html.parser')
        results = []

        # 查找搜索结果项
        for item in soup.find_all('div', class_='result'):
            if len(results) >= num_results:
                break

            # 获取标题和链接
            title_elem = item.find('h3')
            link_elem = item.find('a')

            if title_elem and link_elem:
                title = title_elem.get_text().strip()
                link = link_elem.get('href')

                # 处理百度跳转链接
                if link and link.startswith('http'):
                    # 尝试解析真实的URL（百度会使用重定向）
                    try:
                        resp = self.session.head(link, timeout=5, allow_redirects=True)
                        real_url = resp.url
                    except:
                        real_url = link

                    results.append({
                        'title': title,
                        'url': real_url
                    })

        return results
    except Exception as e:
        print(f"搜索过程中出错: {e}")
        return []

def get_webpage_content(self, url):
    """获取网页内容"""
    # 检查robots.txt
    if not self.check_robots_txt(url):
        return "无法抓取：此网站不允许爬虫访问"

    try:
        response = self.session.get(url, timeout=10)
        response.raise_for_status()

        # 检测编码
        if response.encoding.lower() == 'iso-8859-1':
            response.encoding = response.apparent_encoding or 'utf-8'

        soup = BeautifulSoup(response.text, 'html.parser')

        # 移除不需要的标签
        for script in soup(["script", "style", "nav", "footer", "aside"]):
            script.decompose()

        # 获取正文内容
        text = soup.get_text()

        # 清理文本
        lines = (line.strip() for line in text.splitlines())
        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
        text = ' '.join(chunk for chunk in chunks if chunk)

        return text[:2000] + "..." if len(text) > 2000 else text

    except Exception as e:
        return f"获取网页内容时出错: {e}"

def search_and_crawl(self, keyword):
    """主函数：搜索并抓取内容"""
    print(f"正在搜索关键词: {keyword}")

    # 获取百度搜索结果
    results = self.get_baidu_results(keyword)

    if not results:
        print("未找到搜索结果")
        return

    print(f"\n找到 {len(results)} 条结果:")

    # 处理每个结果
    for i, result in enumerate(results, 1):
        print(f"\n{i}. {result['title']}")
        print(f"   网址: {result['url']}")

        # 获取网页内容
        content = self.get_webpage_content(result['url'])
        print(f"   内容预览: {content[:200]}...")

        # 添加延迟，避免请求过于频繁
        time.sleep(2)

    return results

使用示例

if name == "main":
crawler = BaiduSearchCrawler()
keyword = input("请输入搜索关键词: ")
crawler.search_and_crawl(keyword)

Official Website URL

No response

Originally created by @cbangzan on GitHub (Aug 20, 2025). ### Plugin Name baidu_search ### Function Description import requests from bs4 import BeautifulSoup import urllib.parse import time import re from urllib.robotparser import RobotFileParser import json class BaiduSearchCrawler: def __init__(self): self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } self.session = requests.Session() self.session.headers.update(self.headers) def check_robots_txt(self, url): """检查目标网站的robots.txt""" try: base_url = f"{urllib.parse.urlparse(url).scheme}://{urllib.parse.urlparse(url).netloc}" rp = RobotFileParser() rp.set_url(f"{base_url}/robots.txt") rp.read() return rp.can_fetch(self.headers['User-Agent'], url) except: return True # 如果无法获取robots.txt，默认允许爬取 def get_baidu_results(self, keyword, num_results=5): """从百度搜索获取结果""" encoded_keyword = urllib.parse.quote(keyword) url = f"https://www.baidu.com/s?wd={encoded_keyword}" try: response = self.session.get(url, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') results = [] # 查找搜索结果项 for item in soup.find_all('div', class_='result'): if len(results) >= num_results: break # 获取标题和链接 title_elem = item.find('h3') link_elem = item.find('a') if title_elem and link_elem: title = title_elem.get_text().strip() link = link_elem.get('href') # 处理百度跳转链接 if link and link.startswith('http'): # 尝试解析真实的URL（百度会使用重定向） try: resp = self.session.head(link, timeout=5, allow_redirects=True) real_url = resp.url except: real_url = link results.append({ 'title': title, 'url': real_url }) return results except Exception as e: print(f"搜索过程中出错: {e}") return [] def get_webpage_content(self, url): """获取网页内容""" # 检查robots.txt if not self.check_robots_txt(url): return "无法抓取：此网站不允许爬虫访问" try: response = self.session.get(url, timeout=10) response.raise_for_status() # 检测编码 if response.encoding.lower() == 'iso-8859-1': response.encoding = response.apparent_encoding or 'utf-8' soup = BeautifulSoup(response.text, 'html.parser') # 移除不需要的标签 for script in soup(["script", "style", "nav", "footer", "aside"]): script.decompose() # 获取正文内容 text = soup.get_text() # 清理文本 lines = (line.strip() for line in text.splitlines()) chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) text = ' '.join(chunk for chunk in chunks if chunk) return text[:2000] + "..." if len(text) > 2000 else text except Exception as e: return f"获取网页内容时出错: {e}" def search_and_crawl(self, keyword): """主函数：搜索并抓取内容""" print(f"正在搜索关键词: {keyword}") # 获取百度搜索结果 results = self.get_baidu_results(keyword) if not results: print("未找到搜索结果") return print(f"\n找到 {len(results)} 条结果:") # 处理每个结果 for i, result in enumerate(results, 1): print(f"\n{i}. {result['title']}") print(f" 网址: {result['url']}") # 获取网页内容 content = self.get_webpage_content(result['url']) print(f" 内容预览: {content[:200]}...") # 添加延迟，避免请求过于频繁 time.sleep(2) return results # 使用示例 if __name__ == "__main__": crawler = BaiduSearchCrawler() keyword = input("请输入搜索关键词: ") crawler.search_and_crawl(keyword) ### Official Website URL _No response_

yindo closed this issue

2026-02-22 17:22:57 -05:00

yindo commented

2026-02-22 17:22:59 -05:00

@jingfelix commented on GitHub (Aug 31, 2025):

Hi, if you would like to submit new plugins to Dify Marketplace, please check https://docs.dify.ai/plugin-dev-en/0321-release-overview

@jingfelix commented on GitHub (Aug 31, 2025): Hi, if you would like to submit new plugins to Dify Marketplace, please check https://docs.dify.ai/plugin-dev-en/0321-release-overview

yindo referenced this issue

2026-02-22 17:37:24 -05:00

[PR #232] [MERGED] update quicksandzn/elasticsearch to 0.0.4 #731

Sign in to join this conversation.

1 Participants

Notifications

Due Date

No due date set.

Dependencies

No dependencies set.

Reference: langgenius/dify-plugins#232