Python Wizard.sources方法代码示例

# 需要导入模块: from wizard import Wizard [as 别名]
# 或者: from wizard.Wizard import sources [as 别名]
class Downloader(object):
    def __init__(self, endpoint, environment = environment.default, destination = None, limit = 0):
        super(Downloader, self).__init__()
        self.endpoint    = endpoint
        self.destination = destination
        self.success     = False
        self.limit       = limit
        self.environment = environment
        self.wizard      = Wizard(self.endpoint, environment = self.environment)

        self.init_callbacks()

    @classmethod
    def status_file_for(cls, endpoint):
        import os.path, tempfile
        tmpdir = tempfile.gettempdir()
        status = util.normalize_url(endpoint)
        return os.path.join(tmpdir, status)

    def file_name(self):
        original = self.consumer.file_name().encode()
        ext      = original.split('.')[-1]
        with_ext = '%s.%s' % (self.file_hint(), ext)

        return self.sanitize_file(with_ext)

    def sanitize_file(self, file_name):
        import re
        encoded  = file_name.encode()
        replaced = re.sub(r'[^a-zA-Z0-9. ]', '', encoded)

        return replaced

    def file_hint(self):
        return self.wizard.file_hint

    def status_file(self):    return Downloader.status_file_for(self.endpoint)
    def asset_url(self):      return self.consumer.asset_url().encode()
    def local_partfile(self): return self.local_file() + '.part'
    def local_file(self):
        import os.path
        return os.path.join(self.destination, self.file_name())

    def add_callback(self, group, cb): self.callbacks[group].append(cb)
    def on_start(self, cb):    self.add_callback('start',    cb)
    def on_success(self,  cb): self.add_callback('success',  cb)
    def on_error(self, cb):    self.add_callback('error',    cb)

    def run_start_callbacks(self):
        self.run_callbacks('_start')
        self.run_callbacks('start')

    def run_success_callbacks(self):
        self.run_callbacks('_success')
        self.run_callbacks('success')

    def run_error_callbacks(self):
        self.run_callbacks('_error')
        self.run_callbacks('error')

    def run_callbacks(self, group):
        for cb in self.callbacks[group]:
            cb(self)

    def init_callbacks(self):
        groups = ( 'start', 'success', 'error' )
        self.callbacks = {}

        for g in groups:
            self.callbacks[g]       = []
            self.callbacks['_' + g] = []

        def debug_dl(dl):
            lines = [ dl.pid, dl.consumer.url, dl.asset_url(), dl.local_file() ]

            for line in lines:
                self.environment.log(line)

        def rename_partfile(dl):
            import os
            os.rename( dl.local_partfile(), dl.local_file() )

        def cleanup_status_file(dl):
            dl.cleanup_status_file()

        #self.add_callback('_start',   debug_dl)
        self.add_callback('_success', rename_partfile)
        self.add_callback('_success', cleanup_status_file)

    def download(self):
        def perform_download(consumer):
            self.consumer = consumer
            self.success  = self.really_download()

        self.wizard.sources(perform_download)

        if self.success:
            self.run_success_callbacks()
        else:
            self.run_error_callbacks()
#.........这里部分代码省略.........

# 需要导入模块: from wizard import Wizard [as 别名]
# 或者: from wizard.Wizard import sources [as 别名]
class Downloader(object):
    def __init__(self, endpoint, destination = None, limit = 0, strategy = 'curl', avoid_small_files = False):
        self.endpoint    = endpoint
        self.destination = destination
        self.success     = False
        self.limit       = limit
        self.wizard      = Wizard(self.endpoint)
        self.strategy    = strategy
        self.avoid_small = avoid_small_files

        self.init_callbacks()

    @property
    def file_name(self):
        hinted = self.wizard.file_hint.encode()
        consumed = self.consumer.file_name.encode()
        ext = consumed.split('.')[-1]
        merged = '%s.%s' % (hinted, ext)

        return sanitize_file(merged)

    @property
    def status_file(self):
        return status_file_for(self.endpoint)

    @property
    def local_partfile(self):
        return self.local_file + '.part'

    @property
    def local_file(self):
        import os.path
        return os.path.join(self.destination, self.file_name)

    def add_callback(self, group, cb): self.callbacks[group].append(cb)
    def on_start(self, cb):    self.add_callback('start',    cb)
    def on_success(self,  cb): self.add_callback('success',  cb)
    def on_error(self, cb):    self.add_callback('error',    cb)

    def run_start_callbacks(self):
        log.debug('Running start callbacks')
        self.run_callbacks('_start')
        self.run_callbacks('start')

    def run_success_callbacks(self):
        log.debug('Running success callbacks')
        self.run_callbacks('_success')
        self.run_callbacks('success')

    def run_error_callbacks(self):
        log.debug('Running error callbacks')
        self.run_callbacks('_error')
        self.run_callbacks('error')

    def run_callbacks(self, group):
        for cb in self.callbacks[group]:
            cb(self)

    def init_callbacks(self):
        groups = ( 'start', 'success', 'error' )
        self.callbacks = {}

        for g in groups:
            self.callbacks[g]       = []
            self.callbacks['_' + g] = []

        def rename_partfile(dl):
            try:
                import os
                os.rename( dl.local_partfile, dl.local_file )
            except: pass

        def cleanup_status_file(dl):
            dl.cleanup_status_file()

        self.add_callback('_success', rename_partfile)
        self.add_callback('_success', cleanup_status_file)

    def download(self):
        def perform_download(consumer):
            self.consumer = consumer
            self.success  = self.really_download()

        self.wizard.sources(perform_download)

        if self.success:
            self.run_success_callbacks()
            log.info('Finished downloading %s' % self.wizard.file_hint)
        else:
            self.run_error_callbacks()

    def download_command(self):
        return globals()[self.strategy + '_strategy_command'](self)

    def really_download(self):
        from signal import SIGTERM
        import subprocess

        command  = self.download_command()
        piped    = subprocess.Popen(command)
#.........这里部分代码省略.........

# 需要导入模块: from wizard import Wizard [as 别名]
# 或者: from wizard.Wizard import sources [as 别名]
class Downloader(object):
    def __init__(self, endpoint, environment = environment.default, destination = None, limit = 0, strategy = 'curl'):
        super(Downloader, self).__init__()
        self.endpoint    = endpoint
        self.destination = destination
        self.success     = False
        self.limit       = limit
        self.environment = environment
        self.wizard      = Wizard(self.endpoint, environment = self.environment)
        self.strategy    = strategy

        self.init_callbacks()

    @classmethod
    def status_file_for(cls, endpoint):
        import os.path, tempfile
        tmpdir = tempfile.gettempdir()
        status = util.normalize_url(endpoint)
        return os.path.join(tmpdir, status)

    def file_name(self):
        original = self.consumer.file_name().encode()
        ext      = original.split('.')[-1]
        with_ext = '%s.%s' % (self.file_hint(), ext)

        return self.sanitize_file(with_ext)

    def sanitize_file(self, file_name):
        import re
        encoded  = file_name.encode()
        replaced = re.sub(r'[^a-zA-Z0-9. ]', '', encoded)

        return replaced

    def file_hint(self):
        return self.wizard.file_hint

    def status_file(self):    return Downloader.status_file_for(self.endpoint)
    def asset_url(self):      return self.consumer.asset_url().encode()
    def local_partfile(self): return self.local_file() + '.part'
    def local_file(self):
        import os.path
        return os.path.join(self.destination, self.file_name())

    def add_callback(self, group, cb): self.callbacks[group].append(cb)
    def on_start(self, cb):    self.add_callback('start',    cb)
    def on_success(self,  cb): self.add_callback('success',  cb)
    def on_error(self, cb):    self.add_callback('error',    cb)

    def run_start_callbacks(self):
        log.debug('Running start callbacks')
        self.run_callbacks('_start')
        self.run_callbacks('start')

    def run_success_callbacks(self):
        log.debug('Running success callbacks')
        self.run_callbacks('_success')
        self.run_callbacks('success')

    def run_error_callbacks(self):
        log.debug('Running error callbacks')
        self.run_callbacks('_error')
        self.run_callbacks('error')

    def run_callbacks(self, group):
        for cb in self.callbacks[group]:
            cb(self)

    def init_callbacks(self):
        groups = ( 'start', 'success', 'error' )
        self.callbacks = {}

        for g in groups:
            self.callbacks[g]       = []
            self.callbacks['_' + g] = []

        def rename_partfile(dl):
            import os
            os.rename( dl.local_partfile(), dl.local_file() )

        def cleanup_status_file(dl):
            dl.cleanup_status_file()

        self.add_callback('_success', rename_partfile)
        self.add_callback('_success', cleanup_status_file)

    def download(self):
        def perform_download(consumer):
            self.consumer = consumer
            self.success  = self.really_download()

        self.wizard.sources(perform_download)

        if self.success:
            self.run_success_callbacks()
            log.info('Finished downloading %s' % self.wizard.file_hint)
        else:
            self.run_error_callbacks()

    def download_command(self):
#.........这里部分代码省略.........