ScrapeGraphAI-使用demo
使用ScrapeGraphAI和browser_use实现单页面和多页面爬虫的Python代码示例,包含deepseek集成和文件保存功能。
单页面爬虫-newsee前端代码库
- 采用deepseek + 分步式的ai问答记录
多页面爬虫记录
import asyncio
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from typing import List, Optional
from dotenv import load_dotenv
from langchain_openai import ChatOpenAI
from pydantic import SecretStr,BaseModel
from browser_use import Agent
from browser_use.controller.service import Controller
# dotenv
load_dotenv()
api_key = os.getenv('DEEPSEEK_API_KEY', '')
if not api_key:
raise ValueError('DEEPSEEK_API_KEY is not set')
# Initialize the controller
controller = Controller()
class MdContent(BaseModel):
title: str
url: str
content: str
@controller.action('save content to file', param_model=MdContent)
def save_content_to_file(params: MdContent):
# 将内容追加保存到文件 'content.md'
with open('content.md', 'a') as f:
f.write(f'{params.title}: \n {params.url} , {params.content}\n')
async def run_search():
agent = Agent(
task=(
'1. Go to https://saastest.new-see.com/ns-doc/baseComponents/Basic/Button.html'
# '2. 点击 \'组件文档\' 找到该按钮下拉菜单中的\'基础组件\' 点击'
'2. Look up main content as content,"数据中台" as title, save the content to file.'
),
llm=ChatOpenAI(
base_url='https://api.deepseek.com/v1',
model='deepseek-chat',
api_key=SecretStr(api_key),
),
controller=controller,
use_vision=False,
)
history = await agent.run()
print('📃📃:::History⭐️⭐️:', history)
if __name__ == '__main__':
asyncio.run(run_search())