scrapy 管理部署的爬虫项目的python类详解程序员

这个类可以比较方便的去管理你的爬虫服务

# 测试浏览器弹窗的验证: 
import requests 
from urllib import parse 
import logging 
 
logging.basicConfig(level=logging.INFO) 
 
 
class ScrapyManager(object): 
    def __init__(self, url, project_name, spider=None, username=None, pwd=None): 
        self.url = url 
        self.project_name = project_name 
        self.spider = spider 
        self.auth = (username, pwd) 
 
    def start_project(self): 
        """ 
        爬虫项目启动 
        :return: 
        """ 
        if not self.spider: 
            raise Exception('未提供爬虫名称!') 
        data = dict( 
            project=self.project_name, 
            spider=self.spider, 
        ) 
        start_url = parse.urljoin(self.url, 'schedule.json') 
        res = requests.post(url=start_url, data=data, auth=self.auth) 
        logging.info(res.text) 
 
    def del_project(self): 
        """ 
        项目删除 
        :return: 
        """ 
        data = dict( 
            project=self.project_name, 
            spider=self.spider, 
        ) 
        start_url = parse.urljoin(self.url, 'delproject.json') 
        res = requests.post(url=start_url, data=data, auth=self.auth) 
        logging.info(res.text) 
 
    def stop_job(self, job_id): 
        """ 
        停止任务 
        :param job_id: 任务id 
        :return: 
        """ 
        data = dict( 
            project=self.project_name, 
            job=job_id, 
        ) 
        start_url = parse.urljoin(self.url, 'cancel.json') 
        res = requests.post(url=start_url, data=data, auth=self.auth) 
        logging.info(res.text) 

还有部分api接口没有添加进来,可以参照官方的文档添加.

IT虾米网