ScrapeGraphAI-使用demo

使用ScrapeGraphAI和browser_use实现单页面和多页面爬虫的Python代码示例,包含deepseek集成和文件保存功能。

单页面爬虫-newsee前端代码库

  • 采用deepseek + 分步式的ai问答记录

多页面爬虫记录

python57 行
import asyncio
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from typing import List, Optional

from dotenv import load_dotenv
from langchain_openai import ChatOpenAI
from pydantic import SecretStr,BaseModel

from browser_use import Agent
from browser_use.controller.service import Controller

# dotenv
load_dotenv()

api_key = os.getenv('DEEPSEEK_API_KEY', '')
if not api_key:
    raise ValueError('DEEPSEEK_API_KEY is not set')

# Initialize the controller
controller = Controller()

class MdContent(BaseModel):
    title: str
    url: str
    content: str

@controller.action('save content to file', param_model=MdContent)
def save_content_to_file(params: MdContent):
    # 将内容追加保存到文件 'content.md'
    with open('content.md', 'a') as f:
        f.write(f'{params.title}:  \n {params.url} , {params.content}\n')

async def run_search():
    agent = Agent(
        task=(
            '1. Go to https://saastest.new-see.com/ns-doc/baseComponents/Basic/Button.html'
            # '2. 点击 \'组件文档\' 找到该按钮下拉菜单中的\'基础组件\' 点击'
            '2. Look up main content as content,"数据中台" as title, save the content to file.'
        ),
        llm=ChatOpenAI(
            base_url='https://api.deepseek.com/v1',
            model='deepseek-chat',
            api_key=SecretStr(api_key),
        ),
        controller=controller,
        use_vision=False,
    )

    history = await agent.run()
    print('📃📃:::History⭐️⭐️:', history)

if __name__ == '__main__':
    asyncio.run(run_search())