Python deploy.DeploymentConfig类代码示例

def deploy():
    maintenance("begin")
    supervisor_stop()
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd("deploy/codalab"):
        run("git pull")
        run("pip install -r requirements/dev_azure_nix.txt")
        run("python manage.py syncdb --migrate")
        run("python manage.py collectstatic --noinput")

        # Generate config
        run("python manage.py config_gen")
        run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
        sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
        sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")
        # run('python scripts/initialize.py')  # maybe not needed

        # Setup new relic
        cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
        run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())

    # Setup bundle service for worksheets
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd("deploy/bundles"):
        run("git pull")
        run("alembic upgrade head")

    supervisor()
    maintenance("end")

def _deploy():
    # Update website
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_tag)
        run('./dev_setup.sh')

    # Update bundle service
    with cd(env.deploy_codalab_cli_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_cli_tag)
        run('./setup.sh')
        run('venv/bin/pip install MySQL-Python')
        run('venv/bin/alembic upgrade head')

    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    # Update the website configuration
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
        # Generate configuration files (bundle_server_config, nginx, etc.)
        run('python manage.py config_gen')
        # Migrate database
        run('python manage.py syncdb --migrate')
        # Create static pages
        run('python manage.py collectstatic --noinput')
        # For sending email, have the right domain name.
        run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
        # Create a superuser for OAuth
        run('python manage.py create_codalab_user %s' % cfg.getDatabaseAdminPassword())
        # Allow bundle service to connect to website for OAuth
        run('mkdir -p ~/.codalab && python manage.py set_oauth_key ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
        # Put nginx and supervisor configuration files in place
        sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
        sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
        # Setup new relic
        run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")

def install_config():
    '''
    Install configuration files (do multiple times).
    '''
    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
            run('python manage.py config_gen')
            run('mkdir -p ~/.codalab && python scripts/set-oauth-key.py ./config/generated/bundle_server_config.json > ~/.codalab/config.json')
            sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
            sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
            # Setup new relic
            run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")

def put_mysql_dump_to_new_database():
    '''Puts dubmped database to new location'''
    require('configuration')
    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    db_host = "localhost"
    db_database = configuration.getDatabaseName()
    db_user = configuration.getDatabaseUser()
    db_password = configuration.getDatabasePassword()

    backup_directory = os.path.dirname(os.path.realpath(__file__))

    put(local_path='{}/competitiondump.sql.gz'.format(backup_directory),
        remote_path='/home/azureuser/db_dump.sql.gz',
        use_sudo=True)

def config(label):
    """
    Reads deployment parameters for the given setup.
    label: Label identifying the desired setup (e.g., prod, test, etc.)
    """
    env.cfg_label = label
    print "Deployment label is:", env.cfg_label
    print "Loading configuration from:", env.cfg_path
    configuration = DeploymentConfig(label, env.cfg_path)
    print "Configuring logger..."
    logging.config.dictConfig(configuration.getLoggerDictConfig())
    logger.info("Loaded configuration from file: %s", configuration.getFilename())
    env.roledefs = {'web': configuration.getWebHostnames()}

    # Credentials
    env.user = configuration.getVirtualMachineLogonUsername()

    # Repository
    env.git_codalab_tag = configuration.getGitTag()
    env.deploy_codalab_dir = 'codalab-competitions'  # Directory for codalab competitions

    env.django_settings_module = 'codalab.settings'
    env.django_configuration = configuration.getDjangoConfiguration()  # Prod or Test
    env.config_http_port = '80'
    env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())
    print "Deployment configuration is for:", env.config_server_name

    env.configuration = True
    env.SHELL_ENV = {}

def deploy_compute_worker(label):
    '''
    Deploy/update compute workers.
    For monitoring make sure the azure instance has the port 8000 forwarded

    :param label: Either test or prod
    '''
    env.deploy_codalab_dir = 'codalab-competitions'
    # Create .codalabconfig within home directory
    env.label = label
    cfg = DeploymentConfig(env.label, env.cfg_path if hasattr(env, 'cfg_path') else '.codalabconfig')
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.get_compute_workers_file_content())
    settings_file = os.path.join('~', '.codalabconfig')
    put(buf, settings_file)
    env.git_codalab_tag = cfg.getGitTag()
    env.logs_password = cfg.get_compute_worker_logs_password()

    # Initial setup
    with cd(env.deploy_codalab_dir):
        run('git checkout %s' % env.git_codalab_tag)
        run('git pull')
        run('source /home/azureuser/codalab-competitions/venv/bin/activate && pip install -r /home/azureuser/codalab-competitions/codalab/requirements/dev_azure.txt')
        # run('./dev_setup.sh')

    # run("source /home/azureuser/codalab-competitions/venv/bin/activate && pip install bottle==0.12.8")

    current_directory = os.path.dirname(os.path.realpath(__file__))

    put(
        local_path='{}/configs/upstart/codalab-compute-worker.conf'.format(current_directory),
        remote_path='/etc/init/codalab-compute-worker.conf',
        use_sudo=True
    )
    put(
        local_path='{}/configs/upstart/codalab-monitor.conf'.format(current_directory),
        remote_path='/etc/init/codalab-monitor.conf',
        use_sudo=True
    )
    # run("echo %s > /home/azureuser/codalab-competitions/codalab/codalabtools/compute/password.txt" % env.logs_password)

    with settings(warn_only=True):
        sudo("stop codalab-compute-worker")
        sudo("stop codalab-monitor")
        sudo("start codalab-compute-worker")
        sudo("start codalab-monitor")

def get_database_dump():
    """Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz"""
    require("configuration")
    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    db_host = "localhost"
    db_name = configuration.getDatabaseName()
    db_user = configuration.getDatabaseUser()
    db_password = configuration.getDatabasePassword()

    dump_file_name = "launchdump-%s.sql.gz" % datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")

    run(
        "mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s"
        % (db_host, db_user, db_password, db_name, dump_file_name)
    )

    backup_dir = os.environ.get("CODALAB_MYSQL_BACKUP_DIR", "")
    get("/tmp/%s" % dump_file_name, backup_dir)

def install_mysql(choice="all"):
    """
    Installs a local instance of MySQL of the web instance. This will only work
    if the number of web instances is one.

    choice: Indicates which assets to create/install:
        'mysql'      -> just install MySQL; don't create the databases
        'site_db'    -> just create the site database
        'bundles_db' -> just create the bundle service database
        'all' or ''  -> install all three
    """
    require("configuration")
    if len(env.roledefs["web"]) != 1:
        raise Exception("Task install_mysql requires exactly one web instance.")

    if choice == "mysql":
        choices = {"mysql"}
    elif choice == "site_db":
        choices = {"site_db"}
    elif choice == "bundles_db":
        choices = {"bundles_db"}
    elif choice == "all":
        choices = {"mysql", "site_db", "bundles_db"}
    else:
        raise ValueError("Invalid choice: %s. Valid choices are: 'build', 'web' or 'all'." % (choice))

    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    dba_password = configuration.getDatabaseAdminPassword()

    if "mysql" in choices:
        sudo("DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server")
        sudo("mysqladmin -u root password {0}".format(dba_password))

    if "site_db" in choices:
        db_name = configuration.getDatabaseName()
        db_user = configuration.getDatabaseUser()
        db_password = configuration.getDatabasePassword()
        cmds = [
            "create database {0};".format(db_name),
            "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
            "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user),
        ]
        run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))

    if "bundles_db" in choices:
        db_name = configuration.getBundleServiceDatabaseName()
        db_user = configuration.getBundleServiceDatabaseUser()
        db_password = configuration.getBundleServiceDatabasePassword()
        cmds = [
            "create database {0};".format(db_name),
            "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
            "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user),
        ]
        run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))

def get_database_dump():
    '''Saves backups to $CODALAB_MYSQL_BACKUP_DIR/launchdump-year-month-day-hour-min-second.sql.gz'''
    require('configuration')
    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    db_host = "localhost"
    db_name = configuration.getDatabaseName()
    db_user = configuration.getDatabaseUser()
    db_password = configuration.getDatabasePassword()

    dump_file_name = 'competitiondump.sql.gz'

    run('mysqldump --host=%s --user=%s --password=%s %s --port=3306 | gzip > /tmp/%s' % (
        db_host,
        db_user,
        db_password,
        db_name,
        dump_file_name)
        )
    backup_directory = os.path.dirname(os.path.realpath(__file__))

    get('%s' % dump_file_name, backup_directory)

def install_ssl_certificates():
    """
    Installs SSL certificates on the web instance.
    """
    require("configuration")
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")

def install_mysql():
    """
    Installs a local instance of MySQL of the web instance. This will only work
    if the number of web instances is one.
    """
    require('configuration')
    if len(env.roledefs['web']) <> 1:
        raise(Exception("Task install_mysql requires exactly one web instance."))

    configuration = DeploymentConfig(env.cfg_label, env.cfg_path)
    dba_password = configuration.getDatabaseAdminPassword()
    db_name = configuration.getDatabaseName()
    db_user = configuration.getDatabaseUser()
    db_password = configuration.getDatabasePassword()

    sudo('DEBIAN_FRONTEND=noninteractive apt-get -q -y install mysql-server')
    sudo('mysqladmin -u root password {0}'.format(dba_password))

    cmds = ["create database {0};".format(db_name),
            "create user '{0}'@'localhost' IDENTIFIED BY '{1}';".format(db_user, db_password),
            "GRANT ALL PRIVILEGES ON {0}.* TO '{1}'@'localhost' WITH GRANT OPTION;".format(db_name, db_user) ]
    run('mysql --user=root --password={0} --execute="{1}"'.format(dba_password, " ".join(cmds)))

def _deploy():
    # Update competition website
    # Pull branch and run requirements file, for info about requirments look into dev_setp.sh
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir):
        run('git pull')
        run('git checkout %s' % env.git_codalab_tag)
        run('./dev_setup.sh')

    # Create local.py
    cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
    dep = Deployment(cfg)
    buf = StringIO()
    buf.write(dep.getSettingsFileContent())
    # local.py is generated here. For more info about content look into deploy/__.init__.py
    settings_file = os.path.join(env.deploy_codalab_dir, 'codalab', 'codalab', 'settings', 'local.py')
    put(buf, settings_file)

    # Update the website configuration
    env_prefix, env_shell = setup_env()
    with env_prefix, env_shell, cd(env.deploy_codalab_dir), cd('codalab'):
        # Generate configuration files (bundle_server_config, nginx, etc.)
        # For more info look into https://github.com/greyside/django-config-gen
        run('python manage.py config_gen')
        # Migrate database
        run('python manage.py syncdb --migrate')
        # Create static pages
        run('python manage.py collectstatic --noinput')
        # For sending email, have the right domain name.
        run('python manage.py set_site %s' % cfg.getSslRewriteHosts()[0])
        # Put nginx and supervisor configuration files in place, ln creates symbolic links
        sudo('ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf')
        sudo('ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf')
        # Setup new relic
        run('newrelic-admin generate-config %s newrelic.ini' % cfg.getNewRelicKey())

    # Install SSL certficates (/etc/ssl/certs/)
    require('configuration')
    if (len(cfg.getSslCertificateInstalledPath()) > 0) and (len(cfg.getSslCertificateKeyInstalledPath()) > 0):
        put(cfg.getSslCertificatePath(), cfg.getSslCertificateInstalledPath(), use_sudo=True)
        put(cfg.getSslCertificateKeyPath(), cfg.getSslCertificateKeyInstalledPath(), use_sudo=True)
    else:
        logger.info("Skipping certificate installation because both files are not specified.")

def deploy_web():
    """
    Installs the output of the build on the web instances.
    """
    require("configuration")
    if exists(env.deploy_dir):
        run("rm -rf %s" % env.deploy_dir)
    run("tar -xvzf %s" % env.build_archive)
    run("mv %s deploy" % env.git_tag)

    run("source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv venv")
    env.SHELL_ENV = dict(
        DJANGO_SETTINGS_MODULE=env.django_settings_module,
        DJANGO_CONFIGURATION=env.django_configuration,
        CONFIG_HTTP_PORT=env.config_http_port,
        CONFIG_SERVER_NAME=env.config_server_name,
    )
    print env.SHELL_ENV
    with cd(env.deploy_dir):
        with prefix("source /usr/local/bin/virtualenvwrapper.sh && workon venv"), shell_env(**env.SHELL_ENV):
            requirements_path = "/".join(["codalab", "requirements", "dev_azure_nix.txt"])
            pip_cmd = "pip install -r {0}".format(requirements_path)
            run(pip_cmd)
            # additional requirements for bundle service
            run("pip install SQLAlchemy simplejson")
            with cd("codalab"):
                run("python manage.py config_gen")
                run("mkdir -p ~/.codalab && cp ./config/generated/bundle_server_config.json ~/.codalab/config.json")
                run("python manage.py syncdb --migrate")
                run("python scripts/initialize.py")
                run("python manage.py collectstatic --noinput")
                sudo("ln -sf `pwd`/config/generated/nginx.conf /etc/nginx/sites-enabled/codalab.conf")
                sudo("ln -sf `pwd`/config/generated/supervisor.conf /etc/supervisor/conf.d/codalab.conf")

                # Setup new relic
                cfg = DeploymentConfig(env.cfg_label, env.cfg_path)
                run("newrelic-admin generate-config %s newrelic.ini" % cfg.getNewRelicKey())

def config(label=None):
    """
    Reads deployment parameters for the given setup.

    label: Label identifying the desired setup.
    """
    env.cfg_label = label
    print "Deployment label is: ", env.cfg_label
    filename = ".codalabconfig"
    if "cfg_path" not in env:
        env.cfg_path = os.path.join(os.getcwd(), filename)
        if os.path.exists(env.cfg_path) == False:
            env.cfg_path = os.path.join(os.path.expanduser("~"), filename)
    print "Loading configuration from: ", env.cfg_path
    configuration = DeploymentConfig(label, env.cfg_path)
    print "Configuring logger..."
    logging.config.dictConfig(configuration.getLoggerDictConfig())
    logger.info("Loaded configuration from file: %s", configuration.getFilename())

    env.user = configuration.getVirtualMachineLogonUsername()
    env.password = configuration.getVirtualMachineLogonPassword()
    env.key_filename = configuration.getServiceCertificateKeyFilename()
    env.roledefs = {"build": [configuration.getBuildHostname()]}

    if label is not None:
        env.roledefs.update({"web": configuration.getWebHostnames()})
        # Information about main CodaLab repo
        env.git_user = configuration.getGitUser()
        env.git_repo = configuration.getGitRepo()
        env.git_tag = configuration.getGitTag()
        env.git_repo_url = "https://github.com/{0}/{1}.git".format(env.git_user, env.git_repo)
        # Information about Bundles repo
        env.git_bundles_user = configuration.getBundleServiceGitUser()
        env.git_bundles_repo = configuration.getBundleServiceGitRepo()
        env.git_bundles_tag = configuration.getBundleServiceGitTag()
        if len(configuration.getBundleServiceUrl()) > 0:
            env.git_bundles_repo_url = "https://github.com/{0}/{1}.git".format(
                env.git_bundles_user, env.git_bundles_repo
            )
        else:
            env.git_bundles_repo_url = ""
        env.deploy_dir = "deploy"
        env.build_archive = "{0}.tar.gz".format(env.git_tag)
        env.django_settings_module = "codalab.settings"
        env.django_configuration = configuration.getDjangoConfiguration()
        env.config_http_port = "80"
        env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())

    env.configuration = True

def config(label=None):
    """
    Reads deployment parameters for the given setup.

    label: Label identifying the desired setup.
    """
    env.cfg_label = label
    print "Deployment label is: ", env.cfg_label
    if 'cfg_path' not in env:
        env.cfg_path = os.path.join(os.environ['HOME'], '.codalabconfig')
    print "Loading configuration from: ", env.cfg_path
    configuration = DeploymentConfig(label, env.cfg_path)
    print "Configuring logger..."
    logging.config.dictConfig(configuration.getLoggerDictConfig())
    logger.info("Loaded configuration from file: %s", configuration.getFilename())

    env.user = configuration.getVirtualMachineLogonUsername()
    env.password = configuration.getVirtualMachineLogonPassword()
    env.key_filename = configuration.getServiceCertificateKeyFilename()
    env.roledefs = { 'build' : [ configuration.getBuildHostname() ] }

    if label is not None:
        env.roledefs.update({ 'web' : configuration.getWebHostnames() })
        env.git_user = configuration.getGitUser()
        env.git_repo = configuration.getGitRepo()
        env.git_tag = configuration.getGitTag()
        env.git_repo_url = 'https://github.com/{0}/{1}.git'.format(env.git_user, env.git_repo)
        env.deploy_dir = 'deploy'
        env.build_archive = '{0}.tar.gz'.format(env.git_tag)
        env.django_settings_module = 'codalab.settings'
        env.django_configuration = configuration.getDjangoConfiguration()
        env.config_http_port = '80'
        env.config_server_name = "{0}.cloudapp.net".format(configuration.getServiceName())

    env.configuration = True