Python pkgpanda.Install类代码示例

def test_recovery_move_new(tmpdir):
    # From the "move_new" state correctly.
    shutil.copytree("resources/install_recovery_move", str(tmpdir.join("install")), symlinks=True)
    install = Install(str(tmpdir.join("install")), "resources/systemd", True, False, True)
    action, _ = install.recover_swap_active()
    assert action

    # TODO(cmaloney): expect_fs
    expect_fs(
        str(tmpdir.join("install")),
        {
            ".gitignore": None,
            "active": ["mesos"],
            "active.buildinfo.full.json": None,
            "bin": ["mesos", "mesos-dir"],
            "dcos.target.wants": [".gitignore"],
            "environment": None,
            "environment.export": None,
            "etc": [".gitignore"],
            "include": [".gitignore"],
            "lib": ["libmesos.so"]
        })

#.........这里部分代码省略.........
        raise BuildError("result folder must not exist. It will be made when the package is "
                         "built. {}".format(result_dir))

    # 'mkpanda add' all implicit dependencies since we actually need to build.
    for dep in auto_deps:
        print("Auto-adding dependency: {}".format(dep))
        # NOTE: Not using the name pkg_id because that overrides the outer one.
        id_obj = PackageId(dep)
        add_package_file(repository, package_store.get_package_path(id_obj))
        package = repository.load(dep)
        active_packages.append(package)

    # Checkout all the sources int their respective 'src/' folders.
    try:
        src_dir = cache_abs('src')
        if os.path.exists(src_dir):
            raise ValidationError(
                "'src' directory already exists, did you have a previous build? " +
                "Currently all builds must be from scratch. Support should be " +
                "added for re-using a src directory when possible. src={}".format(src_dir))
        os.mkdir(src_dir)
        for src_name, fetcher in sorted(fetchers.items()):
            root = cache_abs('src/' + src_name)
            os.mkdir(root)

            fetcher.checkout_to(root)
    except ValidationError as ex:
        raise BuildError("Validation error when fetching sources for package: {}".format(ex))

    # Activate the packages so that we have a proper path, environment
    # variables.
    # TODO(cmaloney): RAII type thing for temproary directory so if we
    # don't get all the way through things will be cleaned up?
    install = Install(
        root=install_dir,
        config_dir=None,
        rooted_systemd=True,
        manage_systemd=False,
        block_systemd=True,
        fake_path=True,
        manage_users=False,
        manage_state_dir=False)
    install.activate(active_packages)
    # Rewrite all the symlinks inside the active path because we will
    # be mounting the folder into a docker container, and the absolute
    # paths to the packages will change.
    # TODO(cmaloney): This isn't very clean, it would be much nicer to
    # just run pkgpanda inside the package.
    rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/")

    print("Building package in docker")

    # TODO(cmaloney): Run as a specific non-root user, make it possible
    # for non-root to cleanup afterwards.
    # Run the build, prepping the environment as necessary.
    mkdir(cache_abs("result"))

    # Copy the build info to the resulting tarball
    write_json(cache_abs("src/buildinfo.full.json"), final_buildinfo)
    write_json(cache_abs("result/buildinfo.full.json"), final_buildinfo)

    write_json(cache_abs("result/pkginfo.json"), pkginfo)

    # Make the folder for the package we are building. If docker does it, it
    # gets auto-created with root permissions and we can't actually delete it.
    os.makedirs(os.path.join(install_dir, "packages", str(pkg_id)))

def make_bootstrap_tarball(package_store, packages, variant):
    # Convert filenames to package ids
    pkg_ids = list()
    for pkg_path in packages:
        # Get the package id from the given package path
        filename = os.path.basename(pkg_path)
        if not filename.endswith(".tar.xz"):
            raise BuildError("Packages must be packaged / end with a .tar.xz. Got {}".format(filename))
        pkg_id = filename[:-len(".tar.xz")]
        pkg_ids.append(pkg_id)

    bootstrap_cache_dir = package_store.get_bootstrap_cache_dir()

    # Filename is output_name.<sha-1>.{active.json|.bootstrap.tar.xz}
    bootstrap_id = hash_checkout(pkg_ids)
    latest_name = "{}/{}bootstrap.latest".format(bootstrap_cache_dir, pkgpanda.util.variant_prefix(variant))

    output_name = bootstrap_cache_dir + '/' + bootstrap_id + '.'

    # bootstrap tarball = <sha1 of packages in tarball>.bootstrap.tar.xz
    bootstrap_name = "{}bootstrap.tar.xz".format(output_name)
    active_name = "{}active.json".format(output_name)

    def mark_latest():
        # Ensure latest is always written
        write_string(latest_name, bootstrap_id)

        print("bootstrap: {}".format(bootstrap_name))
        print("active: {}".format(active_name))
        print("latest: {}".format(latest_name))
        return bootstrap_id

    if (os.path.exists(bootstrap_name)):
        print("Bootstrap already up to date, not recreating")
        return mark_latest()

    check_call(['mkdir', '-p', bootstrap_cache_dir])

    # Try downloading.
    if package_store.try_fetch_bootstrap_and_active(bootstrap_id):
        print("Bootstrap already up to date, Not recreating. Downloaded from repository-url.")
        return mark_latest()

    print("Unable to download from cache. Building.")

    print("Creating bootstrap tarball for variant {}".format(variant))

    work_dir = tempfile.mkdtemp(prefix='mkpanda_bootstrap_tmp')

    def make_abs(path):
        return os.path.join(work_dir, path)

    pkgpanda_root = make_abs("opt/mesosphere")
    repository = Repository(os.path.join(pkgpanda_root, "packages"))

    # Fetch all the packages to the root
    for pkg_path in packages:
        filename = os.path.basename(pkg_path)
        pkg_id = filename[:-len(".tar.xz")]

        def local_fetcher(id, target):
            shutil.unpack_archive(pkg_path, target, "gztar")
        repository.add(local_fetcher, pkg_id, False)

    # Activate the packages inside the repository.
    # Do generate dcos.target.wants inside the root so that we don't
    # try messing with /etc/systemd/system.
    install = Install(
        root=pkgpanda_root,
        config_dir=None,
        rooted_systemd=True,
        manage_systemd=False,
        block_systemd=True,
        fake_path=True,
        skip_systemd_dirs=True,
        manage_users=False,
        manage_state_dir=False)
    install.activate(repository.load_packages(pkg_ids))

    # Mark the tarball as a bootstrap tarball/filesystem so that
    # dcos-setup.service will fire.
    make_file(make_abs("opt/mesosphere/bootstrap"))

    # Write out an active.json for the bootstrap tarball
    write_json(active_name, pkg_ids)

    # Rewrite all the symlinks to point to /opt/mesosphere
    rewrite_symlinks(work_dir, work_dir, "/")

    make_tar(bootstrap_name, pkgpanda_root)

    shutil.rmtree(work_dir)

    # Update latest last so that we don't ever use partially-built things.
    write_string(latest_name, bootstrap_id)

    print("Built bootstrap")
    return mark_latest()

def main():
    arguments = docopt(__doc__, version="Pkpganda Package Manager")
    umask(0o022)

    # NOTE: Changing root or repository will likely break actually running packages.
    install = Install(
        os.path.abspath(arguments['--root']),
        os.path.abspath(arguments['--config-dir']),
        arguments['--rooted-systemd'],
        not arguments['--no-systemd'], not arguments['--no-block-systemd'])
    repository = Repository(os.path.abspath(arguments['--repository']))

    try:
        if arguments['setup']:
            actions.setup(install, repository)
            sys.exit(0)

        if arguments['list']:
            print_repo_list(repository.list())
            sys.exit(0)

        if arguments['active']:
            for pkg in sorted(install.get_active()):
                print(pkg)
            sys.exit(0)

        if arguments['add']:
            actions.add_package_file(repository, arguments['<package-tarball>'])
            sys.exit(0)

        if arguments['fetch']:
            for package_id in arguments['<id>']:
                actions.fetch_package(
                    repository,
                    arguments['--repository-url'],
                    package_id,
                    os.getcwd())
            sys.exit(0)

        if arguments['activate']:
            actions.activate_packages(
                install,
                repository,
                arguments['<id>'],
                not arguments['--no-systemd'],
                not arguments['--no-block-systemd'])
            sys.exit(0)

        if arguments['swap']:
            actions.swap_active_package(
                install,
                repository,
                arguments['<package-id>'],
                not arguments['--no-systemd'],
                not arguments['--no-block-systemd'])
            sys.exit(0)

        if arguments['remove']:
            for package_id in arguments['<id>']:
                actions.remove_package(install, repository, package_id)
            sys.exit(0)

        if arguments['uninstall']:
            uninstall(install, repository)
            sys.exit(0)

        if arguments['check']:
            checks = find_checks(install, repository)
            if arguments['--list']:
                list_checks(checks)
                sys.exit(0)
            # Run all checks
            sys.exit(run_checks(checks, install, repository))
    except Exception as ex:
        print("ERROR: {0}".format(ex))
        sys.exit(1)
    except ValidationError as ex:
        print("Validation Error: {0}".format(ex))
        sys.exit(1)
    except PackageError as ex:
        print("Package Error: {0}".format(ex))
        sys.exit(1)

    print("unknown command")
    sys.exit(1)

def main():
    arguments = docopt(__doc__, version="Pkpganda Package Manager")
    umask(0o022)

    # NOTE: Changing root or repository will likely break actually running packages.
    install = Install(
        os.path.abspath(arguments['--root']),
        os.path.abspath(arguments['--config-dir']),
        arguments['--rooted-systemd'],
        not arguments['--no-systemd'], not arguments['--no-block-systemd'])
    repository = Repository(os.path.abspath(arguments['--repository']))

    if arguments['setup']:
        try:
            setup(install, repository)
        except ValidationError as ex:
            print("Validation Error: {0}".format(ex))
            sys.exit(1)
        sys.exit(0)

    if arguments['list']:
        print_repo_list(repository.list())
        sys.exit(0)

    if arguments['active']:
        for pkg in sorted(install.get_active()):
            print(pkg)
        sys.exit(0)

    if arguments['add']:
        add_to_repository(repository, arguments['<package-tarball>'])
        sys.exit(0)

    if arguments['fetch']:
        def fetcher(id, target):
            return requests_fetcher(arguments['--repository-url'], id, target, os.getcwd())

        for pkg_id in arguments['<id>']:
            # TODO(cmaloney): Make this not use escape sequences when not at a
            # `real` terminal.
            sys.stdout.write("\rFetching: {0}".format(pkg_id))
            sys.stdout.flush()
            try:
                repository.add(fetcher, pkg_id)
            except FetchError as ex:
                print("\nUnable to fetch package {0}: {1}".format(pkg_id, ex))
                sys.exit(1)
            sys.stdout.write("\rFetched: {0}\n".format(pkg_id))
            sys.stdout.flush()

        sys.exit(0)

    if arguments['activate']:
        do_activate(install, repository, arguments['<id>'], arguments['--no-systemd'], arguments['--no-block-systemd'])
        sys.exit(0)

    if arguments['swap']:
        active = install.get_active()
        # TODO(cmaloney): I guarantee there is a better way to write this and
        # I've written the same logic before...
        packages_by_name = dict()
        for id_str in active:
            pkg_id = PackageId(id_str)
            packages_by_name[pkg_id.name] = pkg_id

        new_id = PackageId(arguments['<package-id>'])
        if new_id.name not in packages_by_name:
            print("ERROR: No package with name {} currently active to swap with.".format(new_id.name))

        packages_by_name[new_id.name] = new_id
        new_active = list(map(str, packages_by_name.values()))
        # Activate with the new package name
        do_activate(install, repository, new_active, arguments['--no-systemd'], arguments['--no-block-systemd'])
        sys.exit(0)

    if arguments['remove']:
        # Make sure none of the packages are active
        active_packages = install.get_active()
        active = active_packages.intersection(set(arguments['<id>']))
        if len(active) > 0:
            print("Refusing to remove active packages {0}".format(" ".join(sorted(list(active)))))
            sys.exit(1)

        for pkg_id in arguments['<id>']:
            sys.stdout.write("\rRemoving: {0}".format(pkg_id))
            sys.stdout.flush()
            try:
                # Validate package id, that package is installed.
                PackageId(pkg_id)
                repository.remove(pkg_id)
            except ValidationError:
                print("\nInvalid package id {0}".format(pkg_id))
                sys.exit(1)
            except OSError as ex:
                print("\nError removing package {0}".format(pkg_id))
                print(ex)
                sys.exit(1)
            sys.stdout.write("\rRemoved: {0}\n".format(pkg_id))
            sys.stdout.flush()
        sys.exit(0)
#.........这里部分代码省略.........

#.........这里部分代码省略.........
    for dep in auto_deps:
        print("Auto-adding dependency: {}".format(dep))
        # NOTE: Not using the name pkg_id because that overrides the outer one.
        id_obj = PackageId(dep)
        add_to_repository(repository, pkg_abs('../{0}/{1}.tar.xz'.format(id_obj.name, dep)))
        package = repository.load(dep)
        active_packages.append(package)

    # Checkout all the sources int their respective 'src/' folders.
    try:
        src_dir = pkg_abs('src')
        if os.path.exists(src_dir):
            raise ValidationError(
                "'src' directory already exists, did you have a previous build? " +
                "Currently all builds must be from scratch. Support should be " +
                "added for re-using a src directory when possible. src={}".format(src_dir))
        os.mkdir(src_dir)
        for src_name, fetcher in sorted(fetchers.items()):
            root = pkg_abs('src/' + src_name)
            os.mkdir(root)

            fetcher.checkout_to(root)
    except ValidationError as ex:
        raise BuildError("Validation error when fetching sources for package: {}".format(ex))

    # Copy over environment settings
    if 'environment' in buildinfo:
        pkginfo['environment'] = buildinfo['environment']

    # Activate the packages so that we have a proper path, environment
    # variables.
    # TODO(cmaloney): RAII type thing for temproary directory so if we
    # don't get all the way through things will be cleaned up?
    install = Install(install_dir, None, True, False, True, True)
    install.activate(active_packages)
    # Rewrite all the symlinks inside the active path because we will
    # be mounting the folder into a docker container, and the absolute
    # paths to the packages will change.
    # TODO(cmaloney): This isn't very clean, it would be much nicer to
    # just run pkgpanda inside the package.
    rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/")

    print("Building package in docker")

    # TODO(cmaloney): Run as a specific non-root user, make it possible
    # for non-root to cleanup afterwards.
    # Run the build, prepping the environment as necessary.
    mkdir(pkg_abs("result"))

    # Copy the build info to the resulting tarball
    write_json(pkg_abs("src/buildinfo.full.json"), buildinfo)
    write_json(pkg_abs("result/buildinfo.full.json"), buildinfo)

    write_json(pkg_abs("result/pkginfo.json"), pkginfo)

    # Make the folder for the package we are building. If docker does it, it
    # gets auto-created with root permissions and we can't actually delete it.
    os.makedirs(os.path.join(install_dir, "packages", str(pkg_id)))

    # TOOD(cmaloney): Disallow writing to well known files and directories?
    # Source we checked out
    cmd.volumes.update({
        # TODO(cmaloney): src should be read only...
        pkg_abs("src"): "/pkg/src:rw",
        # The build script
        pkg_abs(buildinfo.get('build_script', 'build')): "/pkg/build:ro",

def make_bootstrap_tarball(packages_dir, packages, variant, repository_url):
    # Convert filenames to package ids
    pkg_ids = list()
    for pkg_path in packages:
        # Get the package id from the given package path
        filename = os.path.basename(pkg_path)
        if not filename.endswith(".tar.xz"):
            raise BuildError("Packages must be packaged / end with a .tar.xz. Got {}".format(filename))
        pkg_id = filename[:-len(".tar.xz")]
        pkg_ids.append(pkg_id)

    # Filename is output_name.<sha-1>.{active.json|.bootstrap.tar.xz}
    bootstrap_id = hash_checkout(pkg_ids)
    latest_name = "{}/{}bootstrap.latest".format(packages_dir, pkgpanda.util.variant_prefix(variant))

    output_name = packages_dir + '/' + bootstrap_id + '.'

    # bootstrap tarball = <sha1 of packages in tarball>.bootstrap.tar.xz
    bootstrap_name = "{}bootstrap.tar.xz".format(output_name)
    active_name = "{}active.json".format(output_name)

    def mark_latest():
        # Ensure latest is always written
        write_string(latest_name, bootstrap_id)

        print("bootstrap: {}".format(bootstrap_name))
        print("active: {}".format(active_name))
        print("latest: {}".format(latest_name))
        return bootstrap_name

    if (os.path.exists(bootstrap_name)):
        print("Bootstrap already up to date, not recreating")
        return mark_latest()

    # Try downloading.
    if repository_url:
        tmp_bootstrap = bootstrap_name + '.tmp'
        tmp_active = active_name + '.tmp'
        try:
            repository_url = repository_url.rstrip('/')
            bootstrap_url = repository_url + '/bootstrap/{}.bootstrap.tar.xz'.format(bootstrap_id)
            active_url = repository_url + '/bootstrap/{}.active.json'.format(bootstrap_id)
            print("Attempting to download", bootstrap_name, "from", bootstrap_url)
            # Normalize to no trailing slash for repository_url
            download(tmp_bootstrap, bootstrap_url, packages_dir)
            print("Attempting to download", active_name, "from", active_url)
            download(tmp_active, active_url, packages_dir)

            # Move into place
            os.rename(tmp_bootstrap, bootstrap_name)
            os.rename(tmp_active, active_name)

            print("Bootstrap already up to date, Not recreating. Downloaded from repository-url.")
            return mark_latest()
        except FetchError:
            try:
                os.remove(tmp_bootstrap)
            except:
                pass
            try:
                os.remove(tmp_active)
            except:
                pass

            # Fall out and do the build since the command errored.
            print("Unable to download from cache. Building.")

    print("Creating bootstrap tarball for variant {}".format(variant))

    work_dir = tempfile.mkdtemp(prefix='mkpanda_bootstrap_tmp')

    def make_abs(path):
        return os.path.join(work_dir, path)

    pkgpanda_root = make_abs("opt/mesosphere")
    repository = Repository(os.path.join(pkgpanda_root, "packages"))

    # Fetch all the packages to the root
    for pkg_path in packages:
        filename = os.path.basename(pkg_path)
        pkg_id = filename[:-len(".tar.xz")]

        def local_fetcher(id, target):
            shutil.unpack_archive(pkg_path, target, "gztar")
        repository.add(local_fetcher, pkg_id, False)

    # Activate the packages inside the repository.
    # Do generate dcos.target.wants inside the root so that we don't
    # try messing with /etc/systemd/system.
    install = Install(pkgpanda_root, None, True, False, True, True, True)
    install.activate(repository.load_packages(pkg_ids))

    # Mark the tarball as a bootstrap tarball/filesystem so that
    # dcos-setup.service will fire.
    make_file(make_abs("opt/mesosphere/bootstrap"))

    # Write out an active.json for the bootstrap tarball
    write_json(active_name, pkg_ids)

    # Rewrite all the symlinks to point to /opt/mesosphere
#.........这里部分代码省略.........

def main():
    arguments = docopt(
        __doc__.format(
            default_config_dir=constants.config_dir,
            default_root=constants.install_root,
            default_repository=constants.repository_base,
        ),
    )
    umask(0o022)

    # NOTE: Changing root or repository will likely break actually running packages.
    install = Install(
        os.path.abspath(arguments['--root']),
        os.path.abspath(arguments['--config-dir']),
        arguments['--rooted-systemd'],
        not arguments['--no-systemd'],
        not arguments['--no-block-systemd'],
        manage_users=True,
        add_users=not os.path.exists('/etc/mesosphere/manual_host_users'),
        manage_state_dir=True)

    repository = Repository(os.path.abspath(arguments['--repository']))

    try:
        if arguments['setup']:
            actions.setup(install, repository)
            sys.exit(0)

        if arguments['list']:
            print_repo_list(repository.list())
            sys.exit(0)

        if arguments['active']:
            for pkg in sorted(install.get_active()):
                print(pkg)
            sys.exit(0)

        if arguments['add']:
            actions.add_package_file(repository, arguments['<package-tarball>'])
            sys.exit(0)

        if arguments['fetch']:
            for package_id in arguments['<id>']:
                actions.fetch_package(
                    repository,
                    arguments['--repository-url'],
                    package_id,
                    os.getcwd())
            sys.exit(0)

        if arguments['activate']:
            actions.activate_packages(
                install,
                repository,
                arguments['<id>'],
                not arguments['--no-systemd'],
                not arguments['--no-block-systemd'])
            sys.exit(0)

        if arguments['swap']:
            actions.swap_active_package(
                install,
                repository,
                arguments['<package-id>'],
                not arguments['--no-systemd'],
                not arguments['--no-block-systemd'])
            sys.exit(0)

        if arguments['remove']:
            for package_id in arguments['<id>']:
                try:
                    actions.remove_package(install, repository, package_id)
                except PackageNotFound:
                    pass
            sys.exit(0)

        if arguments['uninstall']:
            uninstall(install, repository)
            sys.exit(0)

        if arguments['check']:
            checks = find_checks(install, repository)
            if arguments['--list']:
                list_checks(checks)
                sys.exit(0)
            # Run all checks
            sys.exit(run_checks(checks, install, repository))
    except ValidationError as ex:
        print("Validation Error: {0}".format(ex))
        sys.exit(1)
    except PackageError as ex:
        print("Package Error: {0}".format(ex))
        sys.exit(1)
    except Exception as ex:
        print("ERROR: {0}".format(ex))
        sys.exit(1)

    print("unknown command")
    sys.exit(1)