Python utils.run_command方法代码示例

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def compile_and_run_pipeline(
    client,
    experiment_id,
    pipeline_definition,
    input_params,
    output_file_dir,
    pipeline_name,
):
    pipeline_path = os.path.join(output_file_dir, pipeline_name)
    utils.run_command(
        f"dsl-compile --py {pipeline_definition} --output {pipeline_path}.yaml"
    )
    run = client.run_pipeline(
        experiment_id, pipeline_name, f"{pipeline_path}.yaml", input_params
    )
    return run.id 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def extract_forms(url, follow = "false", cookie_jar = None, filename = "forms.json"):
	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))

	if cookie_jar == None:
		try:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl form -o {} -a start_url="{}" -a follow={} -a proxy={}'.format(filename, url, follow, HTTP_PROXY)), EXTRACT_WAIT_TIME)
		except:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl form -o {} -a start_url="{}" -a follow={}'.format(filename, url, follow)), EXTRACT_WAIT_TIME)
	else:
		cookie_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename.replace('.json', '.txt'))
		cookie_jar.save(cookie_jar_path)
		out = utils.run_command('{} && {}'.format(
			utils.cd(os.path.dirname(os.path.abspath(__file__))),
			'scrapy crawl form_with_cookie -o {} -a start_url="{}" -a cookie_jar={}'.format(filename, url, cookie_jar_path)), EXTRACT_WAIT_TIME)

	with open(os.path.join(os.path.dirname(__file__), filename)) as json_forms:
		forms = json.load(json_forms)

	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))

	return forms 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def extract_urls(url, follow = "false", cookie_jar = None, filename = "urls.json"):
	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))

	if cookie_jar == None:
		try:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl url -o {} -a start_url="{}" -a follow={} -a proxy={}'.format(filename, url, follow, HTTP_PROXY)), EXTRACT_WAIT_TIME)
		except:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl url -o {} -a start_url="{}" -a follow={}'.format(filename, url, follow)), EXTRACT_WAIT_TIME)
	else:
		cookie_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename.replace('.json', '.txt'))
		cookie_jar.save(cookie_jar_path)
		out = utils.run_command('{} && {}'.format(
			utils.cd(os.path.dirname(os.path.abspath(__file__))),
			'scrapy crawl url_with_cookie -o {} -a start_url="{}" -a cookie_jar={}'.format(filename, url, cookie_jar_path)), EXTRACT_WAIT_TIME)

	with open(os.path.join(os.path.dirname(__file__), filename)) as json_urls:
		urls = json.load(json_urls)

	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))
	return urls 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def install_requirements(self, path):
        if path:
            git_clone_error_cnt = 0
            while True:
                out = self.run_command(path, 'bundle install --no-cache --clean')
                git_clone_error = re.search('Retrying git clone (.*) due to error', out[1])
                if git_clone_error:
                    if git_clone_error_cnt >= 3:
                        break
                    command = 'git clone {}'.format(git_clone_error.group(1)).replace('git://github.com/', 'https://github.com/')
                    LOG.info('Fix Git Fetching Error : {}'.format(command))
                    utils.run_command(command)
                    git_clone_error_cnt += 1
                else:
                    return out[1]
        return ''
    ## DEF 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def get_runtime(self, version = None):
        if self.runtime != None:
            return self.runtime
        latest_successful_attempt = self.get_latest_successful_attempt()
        if latest_successful_attempt != None:
            return {
                'executable': latest_successful_attempt.runtime.executable,
                'version': latest_successful_attempt.runtime.version
            }
        else:
            if version != None:
                return {
                    'executable': 'ruby',
                    'version': version
                }
            else:
                out = utils.run_command('ruby -v')[1].split(' ')
                return {
                    'executable': 'ruby',
                    'version': out[1]
                }
    ## DEF 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def checkout_from(cls, remote_repo_url, repo_path):
        """
        Checkout a repository from a remote URL into a local path.
        """
        LOG.info("Checking out repository from '%s' into '%s'" %
                 (remote_repo_url, repo_path))

        command = 'svn checkout '

        proxy = CONF.get('http_proxy')

        if proxy:
            url = urlparse.urlparse(proxy)
            host = url.scheme + '://' + url.hostname
            port = url.port
            options = ("servers:global:http-proxy-host='%s'" % host,
                       "servers:global:http-proxy-port='%s'" % port)

            proxy_conf = ['--config-option ' + option for option in options]

            command += ' '.join(proxy_conf) + ' '

        command += '%(remote_repo_url)s %(local_target_path)s' % \
                   {'remote_repo_url': remote_repo_url,
                    'local_target_path': repo_path}
        try:
            utils.run_command(command)
            return SvnRepository(remote_repo_url, repo_path)
        except:
            message = "Failed to clone repository"
            LOG.exception(message)
            raise exception.RepositoryError(message=message) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def checkout(self, revision):
        """
        Check out a revision.
        """
        LOG.info("%(name)s: Updating svn repository"
                 % dict(name=self.name))
        try:
            utils.run_command("svn update", cwd=self.working_copy_dir)
        except:
            LOG.debug("%(name)s: Failed to update svn repository"
                      % dict(name=self.name))
            pass
        else:
            LOG.info("%(name)s: Updated svn repository" % dict(name=self.name))

        LOG.info("%(name)s: Checking out revision %(revision)s"
                 % dict(name=self.name, revision=revision))
        try:
            utils.run_command("svn checkout %(repo_url)s@%(revision)s ." %
                dict(repo_url=self.url, revision=revision),
                cwd=self.working_copy_dir)
        except:
            message = ("Could not find revision %s at %s repository"
                       % (revision, self.name))
            LOG.exception(message)
            raise exception.RepositoryError(message=message) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def analyze_item(config, scan_path):
    if not os.path.exists(config['PLEX_DATABASE_PATH']):
        logger.warning("Could not analyze of '%s' because Plex database could not be found.", scan_path)
        return
    # get files metadata_item_id
    metadata_item_ids = get_file_metadata_ids(config, scan_path)
    if metadata_item_ids is None or not len(metadata_item_ids):
        logger.warning("Aborting analysis of '%s' because could not find any 'metadata_item_id' for it.", scan_path)
        return
    metadata_item_id = ','.join(str(x) for x in metadata_item_ids)

    # build Plex analyze command
    analyze_type = 'analyze-deeply' if config['PLEX_ANALYZE_TYPE'].lower() == 'deep' else 'analyze'
    if os.name == 'nt':
        final_cmd = '"%s" --%s --item %s' % (config['PLEX_SCANNER'], analyze_type, metadata_item_id)
    else:
        cmd = 'export LD_LIBRARY_PATH=' + config['PLEX_LD_LIBRARY_PATH'] + ';'
        if not config['USE_DOCKER']:
            cmd += 'export PLEX_MEDIA_SERVER_APPLICATION_SUPPORT_DIR=' + config['PLEX_SUPPORT_DIR'] + ';'
        cmd += config['PLEX_SCANNER'] + ' --' + analyze_type + ' --item ' + metadata_item_id

        if config['USE_DOCKER']:
            final_cmd = 'docker exec -u %s -i %s bash -c %s' % \
                        (cmd_quote(config['PLEX_USER']), cmd_quote(config['DOCKER_NAME']), cmd_quote(cmd))
        elif config['USE_SUDO']:
            final_cmd = 'sudo -u %s bash -c %s' % (config['PLEX_USER'], cmd_quote(cmd))
        else:
            final_cmd = cmd

    # begin analysis
    logger.debug("Starting %s analysis of 'metadata_item': %s",
                 'deep' if config['PLEX_ANALYZE_TYPE'].lower() == 'deep' else 'basic', metadata_item_id)
    logger.debug(final_cmd)
    utils.run_command(final_cmd.encode("utf-8"))
    logger.info("Finished %s analysis of 'metadata_item': %s",
                'deep' if config['PLEX_ANALYZE_TYPE'].lower() == 'deep' else 'basic', metadata_item_id) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def clusterize(self, parts):
        # create a 8 digits random identifier for cluster jobs:
        identifier = ''.join(random.choice(string.ascii_uppercase) for x in range(10))

        for part in parts:
            command = self.command % {'binary': self.binary, 'part': part}

            # create sh file
            shell_script = part + '.sh'
            open(shell_script, 'w').write(QSUB_SCRIPT % {'log': part + '.log',
                                                         'identifier': identifier,
                                                         'command': command})

            # submit script to cluster
            utils.run_command('qsub %s' % shell_script)


        while True:
            qstat_info = self.get_qstat_info(identifier)
            total_processes = sum(qstat_info.values())
            if total_processes == 0:
                break

            self.progress.update('Qstat Info :: Total Jobs: %s, %s' % (pp(total_processes),
                       ', '.join(['%s: %s' % (x, pp(qstat_info[x])) for x in qstat_info])))

            time.sleep(5)

        return True 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def print_workflow_logs(workflow_name):
    output = utils.run_command(
        f"argo logs {workflow_name} -n {utils.get_kfp_namespace()}"
    )
    print(f"workflow logs:\n", output.decode()) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def sync_server(self, path):
        LOG.info('Syncing server ...')
        command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py syncdb --noinput'.format(
            utils.to_env(self.base_path), utils.cd(path))
        output = utils.run_command(command)
        if 'Unknown command' in output[2]:
            command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py migrate --noinput'.format(
            utils.to_env(self.base_path), utils.cd(path))
        return utils.run_command(command)
    ## DEF 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def create_superuser(self, path):
        LOG.info('Creating superuser ...')
        command = '{} && {} && unset DJANGO_SETTINGS_MODULE && {}'.format(
            utils.to_env(self.base_path),
            utils.cd(path),
            """
            echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@test.com', 'admin')" | python manage.py shell
            """)
        return utils.run_command(command)
    ## DEF 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def load_fixtures(self, path):
        LOG.info('Loading fixtures ...')
        for file in os.listdir(os.path.join(path, 'fixtures')):
            LOG.info('Loading fixtures: {}'.format(file))
            command = '{} && {} && unset DJANGO_SETTINGS_MODULE && {}'.format(
                utils.to_env(self.base_path),
                utils.cd(path),
                "python manage.py loaddata {}".format(os.path.join(path, 'fixtures', file)))
            utils.run_command(command)
    ## DEF 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def check_server(self):
        LOG.info("Checking server ...")
        url = self.get_main_url()
        LOG.info("Main Url : {}".format(url))
        command = 'wget --spider {}'.format(url)
        out = utils.run_command(command)
        LOG.info(out)
        if not "200 OK" in out[2]:
            return ATTEMPT_STATUS_RUNNING_ERROR
        else:
            return ATTEMPT_STATUS_SUCCESS
    ## DEF 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import run_command [as 别名]
def run_command(self, path, command):
        commands = '{} && {} && {}'.format(
            utils.cd(path),
            utils.use_ruby_version(self.runtime['version']),
            command)
        return utils.run_command(commands)