950 字
5 分钟
Celery分布式爬虫demo
Celery分布式爬虫demo
官网文档: Celery - Distributed Task Queue — Celery 5.5.3 documentation
中文文档: Celery - 分布式任务队列 — Celery 5.3.1 文档
Github: celery/celery: Distributed Task Queue (development branch)
以爬取 Scrape | Book 作为样例(有点大材小用了), 实际更适合视频下载, 以及CPU耗时等的任务, 可以充分发挥多设备的CPU或者下载带宽
创建项目
uv init celery-demo所有依赖如下, 安装依赖
"celery>=5.5.3", "flower>=2.0.1", "gevent>=25.5.1", "loguru>=0.7.3", "pymongo>=4.13.2", "redis>=6.2.0", "requests>=2.32.4",配置Celery Worker(消费者)
配置文件 celeryconfig.py
# 消息队列broker_url = "redis://root:123456@localhost:6379/0"# 结果存储# result_backend = 'mongodb://root:123456@localhost:27017/celery_db'
# 副本集群mongomongodb_backend_settings = { "database": "celery_db", "taskmeta_collection": "my_taskmeta", # 自定义集合名称 "host": [ "xxx:3717", "xxx:3717", ], "username": "root", "password": "123456", # 替换为您的实际密码 "options": { "replicaSet": "mgset-1150525521", "authSource": "admin", "readPreference": "secondaryPreferred", "connectTimeoutMS": 5000, "serverSelectionTimeoutMS": 10000, },}result_backend = "mongodb"include = ["tasks"]# 进程池重启只支持 prefork 和 threads 池类型, gevent 协程池不支持# worker_pool_restarts = Trueapp.py
from celery import Celeryimport celeryconfig
app = Celery('spa5-crawler')app.config_from_object(celeryconfig)
if __name__ == '__main__': # 启动Worker: celery -A app worker --loglevel=info --pool=gevent --concurrency=10 # 启动监控: celery -A app flower --basic-auth=admin:123456 pass现在即可通过命令启动一个Worker了, 但是队列中还没有任务, 所以worker会空闲
celery -A app worker --loglevel=info --pool=gevent --concurrency=10 # 启动Worker
celery -A app flower --basic-auth=admin:123456 # 启动监控 localhost:5555NOTEWindows不支持Celery默认的prefork进程模型
解决方案1: 使用
solo池(单进程,但支持并发)来避免多进程问题。celery -A tasks worker —loglevel=info —pool=solo
解决方案2: 使用事件驱动模型(适合I/O 密集任务)
pip install gevent
celery -A tasks worker —loglevel=info —pool=gevent —concurrency=10
concurrency 并发数
添加任务
tasks.py
import asyncioimport timeimport randomimport requestsfrom requests.exceptions import RequestExceptionfrom app import app# 定义任务@app.taskdef add(x, y): return x + y
@app.taskdef fetch(url): time.sleep(3) return url
@app.task( # bind=True, # 启用绑定模式,可以访问 self autoretry_for=(RequestException,), # 指定需要自动重试的异常 retry_backoff=True, # 启用指数退避 retry_backoff_max=60, # 最大退避时间(秒) retry_jitter=True, # 添加随机抖动避免同步重试 max_retries=3, # 最大重试次数)def get_books(offset=0):
headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0', }
params = { 'limit': '18', 'offset': offset, } # 增加请求时间, 便于观察 rtime = random.uniform(0.5, 3) time.sleep(rtime) response = requests.get('https://spa5.scrape.center/api/book/', params=params, headers=headers) response.raise_for_status() json_data = response.json() return json_data['results']提交任务(生产者)
main.py
from tasks import add, fetch, get_booksfrom celery import group, chainfrom loguru import logger
logger.add("spa5.log")
def test1(): # 异步执行任务 result = add.delay(4, 6) # 立即返回 AsyncResult 对象
# 获取结果(阻塞等待) print(result.get(timeout=10)) # 输出 10
def test2(): """顺序提交多个任务,逐个获取结果""" results = [] for i in range(5): # 每次调用都会返回一个AsyncResult对象 result = fetch.delay(f'https://example.com/api/{i}') results.append(result)
# 等待所有任务完成并获取结果 final_results = [r.get(timeout=10) for r in results] print("顺序执行结果:", final_results)
# 2. 使用group并行执行任务(推荐)def test3(): """并行执行多个任务,一次性获取所有结果""" # 创建任务组 task_group = group( get_books.s(i) for i in range(0, 9000, 18) )
# 执行任务组 group_result = task_group.apply_async()
# 获取所有结果(等待所有任务完成) # results = group_result.get(timeout=60) # logger.info("并行执行结果:", results)
# 3. 使用chain创建任务链def test4(): """任务链:前一个任务的结果作为下一个任务的输入""" # 创建任务链: (0+1) -> (结果+2) -> (结果+3)... task_chain = chain( add.s(0, 1), # 0 + 1 = 1 add.s(2), # 1 + 2 = 3 add.s(3), # 3 + 3 = 6 add.s(4) # 6 + 4 = 10 )
# 执行任务链 chain_result = task_chain.apply_async() final_result = chain_result.get(timeout=10) print("任务链结果:", final_result)
if __name__ == "__main__": test3()python main.py # 提交任务到队列给Worker执行最终目录如下
celery-demo ├─ uv.lock ├─ tasks.py ├─ spa5.log ├─ README.md ├─ pyproject.toml ├─ main.py ├─ celeryconfig.py └─ app.py监控flower
追踪每一个worker

追踪每一个任务


缓存每一次任务的执行状态以及返回结果等

Celery分布式爬虫demo
https://blog.tiamix.cn/posts/celery分布式爬虫demo/