n8n部署
见上一篇内容
部署
https://blog.csdn.net/weixin_39348931/article/details/148586843?spm=1001.2014.3001.5501
创建工作流
复制以下内容
{"nodes": [{"parameters": {"options": {}},"type": "@n8n/n8n-nodes-langchain.chatTrigger","typeVersion": 1.1,"position": [0,0],"id": "f6a44802-0a91-47f5-b7fb-a9411aaf8be0","name": "When chat message received","webhookId": "ae262f47-33cc-4204-a83f-e87a3c4adbcf","notesInFlow": true,"notes": "请输入你想智能浏览的网址"},{"parameters": {"jsCode": "// 从输入中安全提取第一个URL\ntry {\n const input = $input.first();\n if (!input || !input.json || !input.json.chatInput) {\n throw new Error(\"Invalid input format\");\n }\n \n const message = input.json.chatInput.trim();\n if (!message) {\n throw new Error(\"Empty message received\");\n }\n \n // 支持各种URL格式\n const regex = /https?:\\/\\/[\\w-]+(\\.[\\w-]+)+(:\\d+)?([\\w/#?&%@+=-]*[\\w/#?&%@+=])?/g;\n const urls = message.match(regex);\n \n if (!urls || urls.length === 0) {\n throw new Error(\"No URL found in message\");\n }\n \n // 获取最佳匹配的URL\n const bestMatch = urls.sort((a, b) => b.length - a.length)[0];\n let finalUrl = bestMatch;\n \n // 清理常见的结尾标点\n const punctuations = ['.', ',', '!', '?', ')', ']', '}', ';', ':', '|', '\"', \"'\", '...'];\n while (punctuations.includes(finalUrl.slice(-1))) {\n finalUrl = finalUrl.slice(0, -1);\n }\n \n return {\n json: {\n extractedUrl: finalUrl,\n allFoundUrls: urls,\n originalMessage: message,\n success: true\n }\n };\n \n} catch (error) {\n return {\n json: {\n error: error.message,\n userInput: $input.first().json.chatInput || \"N/A\",\n success: false\n }\n };\n}"},"type": "n8n-nodes-base.code","typeVersion": 2,"position": [220,0],"id": "969af117-f012-465f-bcb7-a6a716c83b88","name": "Code"},{"parameters": {"url": "=http://10.253.4.55:8000/crawler","sendQuery": true,"queryParameters": {"parameters": [{"name": "url","value": "={{ $json.extractedUrl }}"}]},"options": {}},"type": "n8n-nodes-base.httpRequest","typeVersion": 4.2,"position": [440,0],"id": "6b5b8715-c562-4383-9ef5-42b627f33fa0","name": "HTTP Request"}],"connections": {"When chat message received": {"main": [[{"node": "Code","type": "main","index": 0}]]},"Code": {"main": [[{"node": "HTTP Request","type": "main","index": 0}]]}},"pinData": {},"meta": {"instanceId": "29034d9b8fafb4702d23fb5e38d157bfcc3104f9c0e0c682010b18fcca669c87"}}
粘贴到工作流面板
ctrlc+ctrlv
看到如下工作流
启动后端爬虫服务
import json
import subprocess
from test_cra import main
from fastapi import FastAPI, BackgroundTasks
import uvicorn
from fastapi.middleware.cors import CORSMiddleware
import pathlib
import os
import time
import config
app = FastAPI()
# 2、声明一个 源 列表；重点：要包含跨域的客户端 源
origins = ["*"]# 3、配置 CORSMiddleware
app.add_middleware(CORSMiddleware,allow_origins=origins, # 允许访问的源allow_credentials=True, # 支持 cookieallow_methods=["*"], # 允许使用的请求方法allow_headers=["*"], # 允许携带的 Headers
)# 调用url为http://localhost:5555/crawler?url=xxx
@app.get("/crawler")
async def clawer(url: str):# 先根据base_url判断是哪个平台，然后调用对应的爬虫base_url = "/".join(url.split("/")[0:3])print(base_url)platform = ""if "zhihu.com" in base_url:platform='zhihu'config.ZHIHU_SPECIFIED_ID_LIST = [url]dir_path = pathlib.Path(__file__).parent.absolute()# 构建文件路径date = time.strftime('%Y-%m-%d', time.localtime())file_path = dir_path / f"data/{platform}/json/detail_contents_{date}.json"if not os.path.exists(file_path):target = dir_path / f"data/{platform}/json"target.mkdir(parents=True, exist_ok=True)with open(file_path, "w", encoding="utf-8") as f:json.dump({}, f, ensure_ascii=False, indent=4)now_time = time.time()os.chdir(dir_path)print(os.getcwd())# 获取文件更新时间file_update_time = pathlib.Path(file_path).stat().st_mtime# 进入无限循环while True:subprocess.run([os.path.join(dir_path,"venv/Scripts/python.exe"), "main.py", "--platform", platform,"--type", "detail", "--lt", "cookie", "--save_data_option", "json", "--zhihu_url", url])file_update_time = pathlib.Path(file_path).stat().st_mtimeif file_update_time > now_time:breakwith open(file_path, "r", encoding="utf-8") as f:file_data = json.load(f)print(file_data)return {"message": file_data[url]}if __name__ == "__main__":# 使用字符串形式传递app，以便支持reload等功能uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=True)
