Python task_queue.TaskQueue类代码示例

    def __init__(self, options):
        """
        :param options: argparse parsed options object
        """
        global manager
        assert not manager, 'Only one instance of Manager should be created at a time!'
        self.options = options
        self.config_base = None
        self.config_name = None
        self.config_path = None
        self.db_filename = None
        self.engine = None
        self.lockfile = None
        self.database_uri = None
        self.db_upgraded = False
        self._has_lock = False
        self.is_daemon = False

        self.config = {}

        self.ipc_server = IPCServer(self, options.ipc_port)
        self.task_queue = TaskQueue()
        manager = self

        log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
        log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding())
        log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames)
        if codecs.lookup(sys.getfilesystemencoding()).name == 'ascii' and not os.path.supports_unicode_filenames:
            log.warning('Your locale declares ascii as the filesystem encoding. Any plugins reading filenames from '
                        'disk will not work properly for filenames containing non-ascii characters. Make sure your '
                        'locale env variables are set up correctly for the environment which is launching FlexGet.')

        self.initialize()

        # cannot be imported at module level because of circular references
        from flexget.utils.simple_persistence import SimplePersistence
        self.persist = SimplePersistence('manager')

        if db_schema.upgrade_required():
            log.info('Database upgrade is required. Attempting now.')
            # Make sure not to fire the lock-acquired event yet
            # TODO: Detect if any database upgrading is needed and acquire the lock only in one place
            with self.acquire_lock(event=False):
                fire_event('manager.upgrade', self)
                if manager.db_upgraded:
                    fire_event('manager.db_upgraded', self)
        fire_event('manager.startup', self)

    def __init__(self, options):
        """
        :param options: argparse parsed options object
        """
        global manager
        assert not manager, 'Only one instance of Manager should be created at a time!'
        self.options = options
        self.config_base = None
        self.config_name = None
        self.config_path = None
        self.db_filename = None
        self.engine = None
        self.lockfile = None
        self.database_uri = None
        self.db_upgraded = False
        self._has_lock = False
        self.is_daemon = False

        self.config = {}

        self.ipc_server = IPCServer(self, options.ipc_port)
        self.task_queue = TaskQueue()
        manager = self
        self.initialize()

        # cannot be imported at module level because of circular references
        from flexget.utils.simple_persistence import SimplePersistence
        self.persist = SimplePersistence('manager')

        log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
        log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding())
        log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames)

        if db_schema.upgrade_required():
            log.info('Database upgrade is required. Attempting now.')
            # Make sure not to fire the lock-acquired event yet
            # TODO: Detect if any database upgrading is needed and acquire the lock only in one place
            with self.acquire_lock(event=False):
                fire_event('manager.upgrade', self)
                if manager.db_upgraded:
                    fire_event('manager.db_upgraded', self)
        fire_event('manager.startup', self)

    def initialize(self):
        """
        Load plugins, database, and config. Also initializes (but does not start) the task queue and ipc server.
        This should only be called after obtaining a lock.
        """
        if self.initialized:
            raise RuntimeError('Cannot call initialize on an already initialized manager.')

        plugin.load_plugins(
            extra_plugins=[os.path.join(self.config_base, 'plugins')],
            extra_components=[os.path.join(self.config_base, 'components')],
        )

        # Reparse CLI options now that plugins are loaded
        if not self.args:
            self.args = ['--help']
        self.options = get_parser().parse_args(self.args)

        self.task_queue = TaskQueue()
        self.ipc_server = IPCServer(self, self.options.ipc_port)

        self.setup_yaml()
        self.init_sqlalchemy()
        fire_event('manager.initialize', self)
        try:
            self.load_config()
        except ValueError as e:
            log.critical('Failed to load config file: %s' % e.args[0])
            raise

        # cannot be imported at module level because of circular references
        from flexget.utils.simple_persistence import SimplePersistence

        self.persist = SimplePersistence('manager')

        if db_schema.upgrade_required():
            log.info('Database upgrade is required. Attempting now.')
            fire_event('manager.upgrade', self)
            if manager.db_upgraded:
                fire_event('manager.db_upgraded', self)
        fire_event('manager.startup', self)
        self.initialized = True

    def execute_command(self, options):
        """
        Handles the 'execute' CLI command.

        If there is already a task queue running in this process, adds the execution to the queue.
        If FlexGet is being invoked with this command, starts up a task queue and runs the execution.

        Fires events:

        * manager.execute.started
        * manager.execute.completed

        :param options: argparse options
        """
        fire_event('manager.execute.started', self, options)
        if self.task_queue.is_alive() or self.is_daemon:
            if not self.task_queue.is_alive():
                log.error(
                    'Task queue has died unexpectedly. Restarting it. Please open an issue on Github and include'
                    ' any previous error logs.'
                )
                self.task_queue = TaskQueue()
                self.task_queue.start()
            if len(self.task_queue):
                log.verbose('There is a task already running, execution queued.')
            finished_events = self.execute(
                options, output=logger.get_capture_stream(), loglevel=logger.get_capture_loglevel()
            )
            if not options.cron:
                # Wait until execution of all tasks has finished
                for _, _, event in finished_events:
                    event.wait()
        else:
            self.task_queue.start()
            self.ipc_server.start()
            self.execute(options)
            self.shutdown(finish_queue=True)
            self.task_queue.wait()
        fire_event('manager.execute.completed', self, options)

#.........这里部分代码省略.........
        else:
            log_file = os.path.expanduser(self.options.logfile)
            # If an absolute path is not specified, use the config directory.
            if not os.path.isabs(log_file):
                log_file = os.path.join(self.config_base, log_file)

            logger.start(log_file, self.options.loglevel.upper(), to_console=not self.options.cron)

        manager = self

        log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
        log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding())
        log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames)
        if codecs.lookup(sys.getfilesystemencoding()).name == 'ascii' and not os.path.supports_unicode_filenames:
            log.warning('Your locale declares ascii as the filesystem encoding. Any plugins reading filenames from '
                        'disk will not work properly for filenames containing non-ascii characters. Make sure your '
                        'locale env variables are set up correctly for the environment which is launching FlexGet.')

    def initialize(self):
        """
        Load plugins, database, and config. Also initializes (but does not start) the task queue and ipc server.
        This should only be called after obtaining a lock.
        """
        if self.initialized:
            raise RuntimeError('Cannot call initialize on an already initialized manager.')

        plugin.load_plugins(extra_dirs=[os.path.join(self.config_base, 'plugins')])

        # Reparse CLI options now that plugins are loaded
        if not self.args:
            self.args = ['--help']
        self.options = get_parser().parse_args(self.args)

        self.task_queue = TaskQueue()
        self.ipc_server = IPCServer(self, self.options.ipc_port)

        self.setup_yaml()
        self.init_sqlalchemy()
        fire_event('manager.initialize', self)
        try:
            self.load_config()
        except ValueError as e:
            log.critical('Failed to load config file: %s' % e.args[0])
            raise

        # cannot be imported at module level because of circular references
        from flexget.utils.simple_persistence import SimplePersistence
        self.persist = SimplePersistence('manager')

        if db_schema.upgrade_required():
            log.info('Database upgrade is required. Attempting now.')
            fire_event('manager.upgrade', self)
            if manager.db_upgraded:
                fire_event('manager.db_upgraded', self)
        fire_event('manager.startup', self)
        self.initialized = True

    @property
    def tasks(self):
        """A list of tasks in the config"""
        if not self.config:
            return []
        return list(self.config.get('tasks', {}).keys())

    @property
    def has_lock(self):

class Manager(object):

    """Manager class for FlexGet

    Fires events:

    * manager.initialize

      The first time the manager is initialized, before config is loaded

    * manager.before_config_load

      Before the config file is loaded from disk

    * manager.before_config_validate

      When updating the config, before the validator is run on it

    * manager.config_updated

      After a configuration file has been loaded or changed (and validated) this event is fired

    * manager.startup

      After manager has been initialized. This is when application becomes ready to use, however no database lock is
      present, so the database must not be modified on this event.

    * manager.lock_acquired

      The manager does not always require a lock on startup, if one is requested, this event will run when it has been
      acquired successfully

    * manager.upgrade

      If any plugins have declared a newer schema version than exists in the database, this event will be fired to
      allow plugins to upgrade their tables

    * manager.shutdown

      When the manager is exiting

    * manager.execute.completed

      If execution in current process was completed

    * manager.daemon.started
    * manager.daemon.completed
    * manager.db_cleanup
    """

    unit_test = False
    options = None

    def __init__(self, options):
        """
        :param options: argparse parsed options object
        """
        global manager
        assert not manager, 'Only one instance of Manager should be created at a time!'
        self.options = options
        self.config_base = None
        self.config_name = None
        self.config_path = None
        self.db_filename = None
        self.engine = None
        self.lockfile = None
        self.database_uri = None
        self.db_upgraded = False
        self._has_lock = False
        self.is_daemon = False

        self.config = {}

        self.ipc_server = IPCServer(self, options.ipc_port)
        self.task_queue = TaskQueue()
        manager = self
        self.initialize()

        # cannot be imported at module level because of circular references
        from flexget.utils.simple_persistence import SimplePersistence
        self.persist = SimplePersistence('manager')

        log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
        log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding())
        log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames)

        if db_schema.upgrade_required():
            log.info('Database upgrade is required. Attempting now.')
            # Make sure not to fire the lock-acquired event yet
            # TODO: Detect if any database upgrading is needed and acquire the lock only in one place
            with self.acquire_lock(event=False):
                fire_event('manager.upgrade', self)
                if manager.db_upgraded:
                    fire_event('manager.db_upgraded', self)
        fire_event('manager.startup', self)

    def __del__(self):
        global manager
        manager = None

#.........这里部分代码省略.........