Python nb_popen.NonBlockingPopen类代码示例

def win_cmd(command, **kwargs):
    '''
    Wrapper for commands to be run against Windows boxes
    '''
    try:
        proc = NonBlockingPopen(
            command,
            shell=True,
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stream_stds=kwargs.get('display_ssh_output', True),
        )
        log.debug(
            'Executing command(PID {0}): {1!r}'.format(
                proc.pid, command
            )
        )
        proc.poll_and_read_until_finish()
        proc.communicate()
        return proc.returncode
    except Exception as err:
        log.error(
            'Failed to execute command {0!r}: {1}\n'.format(
                command, err
            ),
            exc_info=True
        )
    # Signal an error
    return 1

def sync_minion(options):
    if 'salt_minion_synced' not in options:
        cmd = []
        if options.peer:
            cmd.extend(['salt-call', 'publish.runner'])
        else:
            cmd.append('salt')

        cmd.extend([
            build_minion_target(options),
            'saltutil.sync_all'
        ])
        print('Running CMD: {0!r}'.format(' '.join(cmd)))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            ' '.join(cmd),
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish()
        proc.communicate()

        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
            sys.exit(proc.returncode)

        setattr(options, 'salt_minion_synced', 'yes')

def delete_vm(options):
    '''
    Stop a VM
    '''
    cmd = []
    if options.peer:
        cmd.extend(['salt-call', 'publish.runner'])
    else:
        cmd.append('salt-run')
    if options.lxc:
        cmd.append('lxc.purge')
    else:
        cmd.append('cloud.destroy')

    cmd.append(options.vm_name)
    print('Running CMD: {0!r}'.format(' '.join(cmd)))
    sys.stdout.flush()

    proc = NonBlockingPopen(
        ' '.join(cmd),
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stream_stds=True
    )
    proc.poll_and_read_until_finish()
    proc.communicate()

def download_unittest_reports(options):
    '''
    Download the generated unit test reports from minion
    '''
    print('Downloading remote unittest reports...')
    sys.stdout.flush()

    xml_reports_path = os.path.join(options.workspace, 'xml-test-reports')
    if os.path.isdir(xml_reports_path):
        shutil.rmtree(xml_reports_path)

    if options.scp:
        cmds = (
            ' '.join(build_scp_command(options,
                                       '-r',
                                       '[email protected]{0}:/tmp/xml-unitests-output/*'.format(
                                           get_minion_external_address(options)
                                       ),
                                       os.path.join(options.workspace, 'xml-test-reports'))),
        )
    else:
        os.makedirs(xml_reports_path)
        cmds = (
            '{0} {1} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/',
            '{0} {1} cp.push /tmp/xml-test-reports.tar.gz',
            'mv -f /var/cache/salt/master/minions/{2}/files/tmp/xml-test-reports.tar.gz {3} && '
            'tar zxvf {3}/xml-test-reports.tar.gz -C {3}/xml-test-reports && '
            'rm -f {3}/xml-test-reports.tar.gz'
        )

    for cmd in cmds:
        cmd = cmd.format(
            'salt-call publish.publish' if options.lxc else 'salt',
            build_minion_target(options),
            options.vm_name,
            options.workspace
        )
        print('Running CMD: {0!r}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish()
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def download_remote_logs(options):
    print('Downloading remote logs...')
    sys.stdout.flush()

    workspace = options.workspace
    vm_name = options.download_remote_logs

    for fname in ('salt-runtests.log', 'minion.log'):
        if os.path.isfile(os.path.join(workspace, fname)):
            os.unlink(os.path.join(workspace, fname))

    if not options.remote_log_path:
        options.remote_log_path = [
            '/tmp/salt-runtests.log',
            '/var/log/salt/minion'
        ]

    cmds = []

    for remote_log in options.remote_log_path:
        cmds.extend([
            'salt {{0}} archive.gzip {0}'.format(remote_log),
            'salt {{0}} cp.push {0}.gz'.format(remote_log),
            'gunzip /var/cache/salt/master/minions/{{1}}/files{0}.gz'.format(remote_log),
            'mv /var/cache/salt/master/minions/{{1}}/files{0} {{2}}/{1}'.format(
                remote_log,
                '{0}{1}'.format(
                    os.path.basename(remote_log),
                    '' if remote_log.endswith('.log') else '.log'
                )
            )
        ])

    for cmd in cmds:
        cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish(interval=0.5)
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def download_coverage_report(options):
    '''
    Download the generated coverage report from minion
    '''
    print('Downloading remote coverage report...')
    sys.stdout.flush()

    if os.path.isfile(os.path.join(options.workspace, 'coverage.xml')):
        os.unlink(os.path.join(options.workspace, 'coverage.xml'))

    if options.scp:
        cmds = (
            ' '.join(build_scp_command(options,
                                       '[email protected]{0}:/tmp/coverage.xml'.format(
                                           get_minion_external_address(options)
                                       ),
                                       os.path.join(options.workspace, 'coverage.xml'))),
        )
    else:
        cmds = (
            '{0} {1} archive.gzip /tmp/coverage.xml',
            '{0} {1} cp.push /tmp/coverage.xml.gz',
            'gunzip /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml.gz',
            'mv /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml {3}'
        )

    for cmd in cmds:
        cmd = cmd.format(
            'salt-call publish.publish' if options.lxc else 'salt',
            build_minion_target(options),
            options.vm_name,
            options.workspace
        )
        print('Running CMD: {0!r}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish()
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def prepare_ssh_access(options):
    '''
    Generate a temporary SSH key, valid for one hour, and set it as an
    authorized key in the minion's root user account on the remote system.
    '''
    print('Generating temporary SSH Key')
    ssh_key_path = os.path.join(options.workspace, 'jenkins_ssh_key_test')
    subprocess.call(
        'ssh-keygen -t ecdsa -b 521 -C "$(whoami)@$(hostname)-$(date --rfc-3339=seconds)" '
        '-f {0} -N \'\' -V -10m:+1h'.format(ssh_key_path),
        shell=True,
    )
    cmd = []
    if options.peer:
        cmd.extend(['salt-call', 'publish.publish'])
    else:
        cmd.append('salt')
    pub_key_contents = open('{0}.pub'.format(ssh_key_path)).read().strip()
    enc, key, comment = pub_key_contents.split(' ', 2)
    cmd.extend([
        build_minion_target(options),
        'ssh.set_auth_key',
        'root',
        '{0!r}'.format(key),
        'enc={0}'.format(enc),
        'comment={0!r}'.format(comment)
    ])

    cmd = ' '.join(cmd)
    print('Running CMD: {0!r}'.format(cmd))
    sys.stdout.flush()

    proc = NonBlockingPopen(
        cmd,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stream_stds=True
    )
    proc.poll_and_read_until_finish()
    proc.communicate()
    if proc.returncode != 0:
        print(
            '\nFailed to execute command. Exit code: {0}'.format(
                proc.returncode
            )
        )

def cleanup(clean, vm_name):
    if not clean:
        return

    cmd = 'salt-cloud -d {0} -y'.format(vm_name)
    print('Running CMD: {0}'.format(cmd))
    sys.stdout.flush()

    proc = NonBlockingPopen(
        cmd,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stream_stds=True
    )
    proc.poll_and_read_until_finish()
    proc.communicate()

def delete_vm(options):
    '''
    Stop a VM
    '''
    cmd = 'salt-cloud -d {0} -y'.format(options.delete_vm)
    print('Running CMD: {0}'.format(cmd))
    sys.stdout.flush()

    proc = NonBlockingPopen(
        cmd,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stream_stds=True
    )
    proc.poll_and_read_until_finish(interval=0.5)
    proc.communicate()

def download_packages(options):
    print('Downloading packages...')
    sys.stdout.flush()

    workspace = options.workspace
    vm_name = options.download_packages

    for fglob in ('salt-*.rpm',
                  'salt-*.deb',
                  'salt-*.pkg.xz',
                  'salt-buildpackage.log'):
        for fname in glob.glob(os.path.join(workspace, fglob)):
            if os.path.isfile(fname):
                os.unlink(fname)

    cmds = [
        ('salt {{0}} archive.tar czf {0}.tar.gz sources=\'*.*\' cwd={0}'
         .format(options.package_artifact_dir)),
        'salt {{0}} cp.push {0}.tar.gz'.format(options.package_artifact_dir),
        ('tar -C {{2}} -xzf /var/cache/salt/master/minions/{{1}}/files{0}.tar.gz'
         .format(options.package_artifact_dir)),
    ]

    for cmd in cmds:
        cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish(interval=0.5)
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def download_remote_logs(options):
    print('Downloading remote logs...')
    sys.stdout.flush()

    workspace = options.workspace
    vm_name = options.download_remote_logs

    for fname in ('salt-runtests.log', 'minion.log'):
        if os.path.isfile(os.path.join(workspace, fname)):
            os.unlink(os.path.join(workspace, fname))

    cmds = (
        'salt {0} archive.gzip /tmp/salt-runtests.log',
        'salt {0} archive.gzip /var/log/salt/minion',
        'salt {0} cp.push /tmp/salt-runtests.log.gz',
        'salt {0} cp.push /var/log/salt/minion.gz',
        'gunzip /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log.gz',
        'gunzip /var/cache/salt/master/minions/{0}/files/var/log/salt/minion.gz',
        'mv /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log {1}/salt-runtests.log',
        'mv /var/cache/salt/master/minions/{0}/files/var/log/salt/minion {1}/minion.log'
    )

    for cmd in cmds:
        cmd = cmd.format(vm_name, workspace)
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish()
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def download_unittest_reports(options):
    print('Downloading remote unittest reports...')
    sys.stdout.flush()

    workspace = options.workspace
    xml_reports_path = os.path.join(workspace, 'xml-test-reports')
    if os.path.isdir(xml_reports_path):
        shutil.rmtree(xml_reports_path)

    os.makedirs(xml_reports_path)

    cmds = (
        'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unittests-output/',
        'salt {0} cp.push /tmp/xml-test-reports.tar.gz',
        'mv -f /var/cache/salt/master/minions/{1}/files/tmp/xml-test-reports.tar.gz {2} && '
        'tar zxvf {2}/xml-test-reports.tar.gz -C {2}/xml-test-reports && '
        'rm -f {2}/xml-test-reports.tar.gz'
    )

    vm_name = options.download_unittest_reports
    for cmd in cmds:
        cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish(interval=0.5)
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def download_unittest_reports(options):
    print('Downloading remote unittest reports...')
    sys.stdout.flush()

    if os.path.isdir('xml-test-reports'):
        shutil.rmtree('xml-test-reports')

    os.makedirs('xml-test-reports')

    cmds = (
        'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/',
        'salt {0} cp.push /tmp/xml-test-reports.tar.gz',
        'mv -f /var/cache/salt/master/minions/{0}/files/tmp/xml-test-reports.tar.gz {1}',
        'tar zxvf {1}/xml-test-reports.tar.gz -C {1}/xml-test-reports',
        'rm -f {1}/xml-test-reports.tar.gz'
    )

    vm_name = options.download_unittest_reports
    workspace = options.workspace
    for cmd in cmds:
        cmd = cmd.format(vm_name, workspace)
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish()
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )

def download_coverage_report(options):
    print('Downloading remote coverage report...')
    sys.stdout.flush()

    workspace = options.workspace
    vm_name = options.download_coverage_report

    if os.path.isfile(os.path.join(workspace, 'coverage.xml')):
        os.unlink(os.path.join(workspace, 'coverage.xml'))

    cmds = (
        'salt {0} archive.gzip /tmp/coverage.xml',
        'salt {0} cp.push /tmp/coverage.xml.gz',
        'gunzip /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml.gz',
        'mv /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml {2}'
    )

    for cmd in cmds:
        cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()

        proc = NonBlockingPopen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stream_stds=True
        )
        proc.poll_and_read_until_finish(interval=0.5)
        proc.communicate()
        if proc.returncode != 0:
            print(
                '\nFailed to execute command. Exit code: {0}'.format(
                    proc.returncode
                )
            )
        time.sleep(0.25)

def run(opts):
    '''
    RUN!
    '''
    vm_name = os.environ.get(
        'JENKINS_SALTCLOUD_VM_NAME',
        generate_vm_name(opts)
    )

    if opts.download_remote_reports:
        if opts.test_without_coverage is False:
            opts.download_coverage_report = vm_name
        opts.download_unittest_reports = vm_name
        opts.download_packages = vm_name

    if opts.bootstrap_salt_commit is not None:
        if opts.bootstrap_salt_url is None:
            opts.bootstrap_salt_url = 'https://github.com/saltstack/salt.git'
        cmd = (
            'salt-cloud -l debug'
            ' --script-args "-D -g {bootstrap_salt_url} -n git {1}"'
            ' -p {provider}_{platform} {0}'.format(
                vm_name,
                os.environ.get(
                    'SALT_MINION_BOOTSTRAP_RELEASE',
                    opts.bootstrap_salt_commit
                ),
                **opts.__dict__
            )
        )
    else:
        cmd = (
            'salt-cloud -l debug'
            ' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'.format(
                vm_name,
                os.environ.get(
                    'SALT_MINION_BOOTSTRAP_RELEASE',
                    opts.bootstrap_salt_commit
                ),
                **opts.__dict__
            )
        )
    print('Running CMD: {0}'.format(cmd))
    sys.stdout.flush()

    proc = NonBlockingPopen(
        cmd,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        stream_stds=True
    )
    proc.poll_and_read_until_finish(interval=0.5)
    proc.communicate()

    retcode = proc.returncode
    if retcode != 0:
        print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
        sys.stdout.flush()
        if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
            delete_vm(opts)
        sys.exit(retcode)

    print('VM Bootstrapped. Exit code: {0}'.format(retcode))
    sys.stdout.flush()

    print('Sleeping for 5 seconds to allow the minion to breathe a little')
    sys.stdout.flush()
    time.sleep(5)

    if opts.bootstrap_salt_commit is not None:
        # Let's find out if the installed version matches the passed in pillar
        # information
        print('Grabbing bootstrapped minion version information ... ')
        cmd = 'salt -t 100 {0} --out json test.version'.format(build_minion_target(opts, vm_name))
        print('Running CMD: {0}'.format(cmd))
        sys.stdout.flush()
        proc = subprocess.Popen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        stdout, _ = proc.communicate()

        retcode = proc.returncode
        if retcode != 0:
            print('Failed to get the bootstrapped minion version. Exit code: {0}'.format(retcode))
            sys.stdout.flush()
            if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
                delete_vm(opts)
            sys.exit(retcode)

        if not stdout.strip():
            print('Failed to get the bootstrapped minion version(no output). Exit code: {0}'.format(retcode))
            sys.stdout.flush()
            if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
                delete_vm(opts)
            sys.exit(retcode)

#.........这里部分代码省略.........