# 导入Task基类
from ratel.task.task_basic import Task


class RequestTask(Task):
    def __init__(self):
        """
        request方式的任务
        """
        self.__url = None
        self.__result = None

    @property
    def get_url(self):
        return self.__url

    def set_url(self, url):
        self.__url = url

    @property
    def get_result(self):
        return self.__result

    def set_result(self, result):
        self.__result = result


class SeleniumTask(Task):
    def __init__(self):
        """
        selenium方式的任务
        """
        self.__url = None
        self.__result = None

    @property
    def get_url(self):
        return self.__url

    def set_url(self, url):
        self.__url = url

    @property
    def get_result(self):
        return self.__result

    def set_result(self, result):
        self.__result = result

# 导入Dispose基类
from ratel.task.task_basic import Dispose


class RequestDispose(Dispose):
    def __init__(self):
        self.__result = None

    @property
    def get_result(self):
        return self.__result

    def set_result(self, result):
        self.__result = result

    def __str__(self):
        return '<RequestDispose> result: %s' % self.__result


class SeleniumDispose(Dispose):
    def __init__(self):
        self.__result = None

    @property
    def get_result(self):
        return self.__result

    def set_result(self, result):
        self.__result = result

    def __str__(self):
        return '<SeleniumDispose> result: %s' % self.__result

from ratel.task.task_basic import Execute
from task.demo_task import RequestTask, SeleniumTask
from dispose.demo_dispose import RequestDispose, SeleniumDispose
from ratel.collect.request_basic import Url, request_url
from ratel.collect.webdriver_basic import Driver

import time
import threading


class RequestExecute(Execute):
    def __init__(self):
        Execute.__init__(self)

    def get_task(self):
        """
        模拟取任务,任务可使用队列
        :return:
        """
        while True:
            # 模拟创建一条任务
            task = RequestTask()
            task.set_url(Url(url='http://www.baidu.com', method='GET'))
            # 将任务放入至队列
            self.put_task(task)
            time.sleep(10)

    def do_task(self):
        """
        模拟执行任务
        :return:
        """
        # 使用线程取任务, 可不使用
        threading.Thread(target=self.get_task, name='get_task', daemon=True).start()
        # 使用线程上报任务, 可不使用
        threading.Thread(target=self.up_task, name='up_task', daemon=True).start()
        while True:
            # 判断队列是否为空
            if not self.task_empty:
                try:
                    # 从队列中取出任务
                    task = self.poll_task
                    # 通知队列任务以取出
                    self.task_done()
                    # 执行任务
                    resp = request_url(url=task.get_url)
                    # 处理任务结果
                    task.set_result(self.dispose(resp))
                    self.put_success(task)
                except Exception as e:
                    print(e)
            time.sleep(2)

    def dispose(self, result):
        """
        模拟处理任务
        :param result:
        :return:
        """
        # 将任务的处理结果放入上报队列
        request_dispose = RequestDispose()
        request_dispose.set_result(result)
        return request_dispose

    def up_task(self):
        """
        模拟上报任务
        :return:
        """
        while True:
            # 判断队列是否为空
            if not self.success_empty:
                # 取出任务
                task = self.poll_success
                self.success_done()
                # 模拟上报
                print(task.get_result)
            time.sleep(2)

    def end(self):
        """
        任务结束的处理,例如释放资源等
        :return:
        """
        pass


class SeleniumExecute(Execute):
    def __init__(self):
        Execute.__init__(self)

    def get_task(self):
        while True:
            task = SeleniumTask()
            task.set_url("http://www.baidu.com")
            self.put_task(task)
            time.sleep(10)

    def do_task(self):
        # 使用线程取任务, 可不使用
        threading.Thread(target=self.get_task, name='get_task', daemon=True).start()
        # 使用线程上报任务, 可不使用
        threading.Thread(target=self.up_task, name='up_task', daemon=True).start()
        while True:
            if not self.task_empty:
                try:
                    task = self.poll_task
                    self.task_done()
                    # 创建chromeDriver
                    driver_impl = Driver(driver_type='chrome', executable_path='/Users/xw/Documents/Tools/chromedriver')
                    driver = driver_impl.get_driver
                    # 打开浏览器并跳转到指定URL
                    driver.get(task.get_url)
                    # 等待页面加载
                    """
                        等待有3种方式
                        1).time.sleep(),最简单的方式但过于死板,不管页面是否加载完毕都要等待
                        2).driver.implicitly_wait(),较sleep稍好,但仍有局限性当超过等待时间页面未加载完毕会抛出异常
                        3).selenium.webdriver.support.wait模块中的WebDriverWait,可根据条件灵活的设置等待时间.
                        调用该类的until或until_not方法让程序每隔n秒去判断一下条件,看是否满足,如不满足,继续等待直至超时.
                        超时会抛出TimeoutException异常
                    """
                    time.sleep(5)
                    result = driver.page_source
                    driver.close()
                    task.set_result(self.dispose(result))
                    self.put_success(task)
                except Exception as e:
                    print(e)
            time.sleep(2)

    def dispose(self, result):
        selenium = SeleniumDispose()
        selenium.set_result(result)
        return selenium

    def up_task(self):
        """
        模拟上报任务
        :return:
        """
        while True:
            # 判断队列是否为空
            if not self.success_empty:
                # 取出任务
                task = self.poll_success
                self.success_done()
                # 模拟上报
                print(task.get_result)
            time.sleep(2)

    def end(self):
        pass

# 导入do_task模块
import ratel.main.do_task as do_task

if __name__ == '__main__':
    do_task.do_pool()

<!-- 可在任意目录下创建该文件 -->
<config>
    <options>
        <pool_size>10</pool_size>
    </options>
    <tasks>
        <task>
            <task_class>RequestExecute</task_class>
            <task_package>execute.demo_execute</task_package>
        </task>
    </tasks>
</config>