Python models.Task类代码示例

def test_build_image(client, test_db):
    # 反正本地也跑不过 -_-!
    return

    app, version, group, pod, host = create_local_test_data()

    rv = client.post(
        '/api/deploy/build/group/pod/blueberry',
        data=json.dumps({
            'base':
            'containerops.cn/tonicbupt/ubuntu:python-2014.11.28',
            'version':
            version.sha
        }),
        content_type='application/json')
    assert rv.status_consts == 200
    r = json.loads(rv.data)
    assert r[u'r'] == 0
    task_id = r[u'task']
    assert task_id
    task = Task.get(task_id)
    assert task.host_id == host.id
    assert task.app_id == app.id
    assert task.version_id == version.id
    assert task.type == consts.TASK_BUILD
    assert task.props == {
        'base': 'containerops.cn/tonicbupt/ubuntu:python-2014.11.28'
    }

def _create_task(type_, version, host, ncontainer,
    cores, nshare, networks, spec_ips, entrypoint, env, image=''):
    network_ids = [n.id for n in networks]
    task_props = {
        'ncontainer': ncontainer,
        'entrypoint': entrypoint,
        'env': env,
        'full_cores': [c.label for c in cores.get('full', [])],
        'part_cores': [c.label for c in cores.get('part', [])],
        'nshare': nshare,
        'networks': network_ids,
        'image': image,
    }
    task = Task.create(type_, version, host, task_props)
    if not task:
        return None

    try:
        create_containers_with_macvlan.apply_async(
            args=(task.id, ncontainer, nshare, cores, network_ids, spec_ips),
            task_id='task:%d' % task.id
        )
    except Exception as e:
        logger.exception(e)
        host.release_cores(cores)

    return task

def rm_containers():
    cids = request.get_json()['cids']

    if not all(len(cid) >= 7 for cid in cids):
        abort(400, 'must given at least 7 chars for container_id')

    version_dict = {}
    for cid in cids:
        container = Container.get_by_container_id(cid)
        if not container:
            continue
        version_dict.setdefault((container.version, container.host), []).append(container)

    ts, watch_keys = [], []
    for (version, host), containers in version_dict.iteritems():
        cids = [c.id for c in containers]
        task = Task.create(consts.TASK_REMOVE, version, host, {'container_ids': cids})

        all_host_cids = [c.id for c in Container.get_multi_by_host(host) if c and c.version_id == version.id]
        need_to_delete_image = set(cids) == set(all_host_cids)

        remove_containers.apply_async(
            args=(task.id, cids, need_to_delete_image),
            task_id='task:%d' % task.id
        )
        ts.append(task.id)
        watch_keys.append(task.result_key)
    return {'r': 0, 'msg': 'ok', 'tasks': ts, 'watch_keys': watch_keys}

def build_image_v2():
    # form post
    appname = request.form.get('appname', default='')
    version = request.form.get('version', default='')
    base = request.form.get('base', default='')
    if not base:
        abort(400, 'base image must be set')
    _, version = _get_app_and_version(appname, version)

    if ':' not in base:
        base = base + ':latest'

    host = Host.get_random_public_host()
    if not host:
        abort(406, 'no host is available')

    # if no artifacts.zip is set
    # ignore and just do the cloning and building
    file_path = None
    if 'artifacts.zip' in request.files:
        f = request.files['artifacts.zip']
        file_path = os.path.join(tempfile.mkdtemp(), secure_filename(f.filename))
        f.save(file_path)

    task = Task.create(TASK_BUILD, version, host, {'base': base})
    build_docker_image.apply_async(
        args=(task.id, base, file_path),
        task_id='task:%d' % task.id
    )
    return {'task': task.id, 'watch_key': task.result_key}

def remove_containers(task_id, cids, rmi=False):
    current_flask.logger.info('Task<id=%s>: Started', task_id)
    task = Task.get(task_id)
    if not task:
        current_flask.logger.error('Task (id=%s) not found, quit', task_id)
        return

    notifier = TaskNotifier(task)
    containers = Container.get_multi(cids)
    container_ids = [c.container_id for c in containers]
    host = task.host
    try:
        flags = {'eru:agent:%s:container:flag' % cid: 1 for cid in container_ids}
        rds.mset(**flags)
        for c in containers:
            remove_container_backends(c)
            current_flask.logger.info('Task<id=%s>: Container (cid=%s) backends removed',
                    task_id, c.container_id[:7])
        appnames = {c.appname for c in containers}
        publish_to_service_discovery(*appnames)

        dockerjob.remove_host_containers(containers, host)
        current_flask.logger.info('Task<id=%s>: Containers (cids=%s) removed', task_id, cids)
        if rmi:
            dockerjob.remove_image(task.version, host)
    except Exception, e:
        task.finish_with_result(consts.TASK_FAILED)
        notifier.pub_fail()
        current_flask.logger.error('Task<id=%s>: Exception (e=%s)', task_id, e)

def build_docker_image(task_id, base, file_path):
    task = Task.get(task_id)
    if not task:
        _log.error('Task (id=%s) not found, quit', task_id)
        return

    _log.info('Task<id=%s>: Start on host %s', task_id, task.host.ip)
    notifier = TaskNotifier(task)

    app = task.app
    host = task.host
    version = task.version

    try:
        repo, tag = base.split(':', 1)
        repo = repo if repo.startswith('eru/') else 'eru/' + repo.strip('/')
        _log.info('Task<id=%s>: Pull base image (base=%s)', task_id, base)
        notifier.store_and_broadcast(dockerjob.pull_image(host, repo, tag))

        _log.info('Task<id=%s>: Build image (base=%s)', task_id, base)
        notifier.store_and_broadcast(dockerjob.build_image(host, version, base, file_path))

        _log.info('Task<id=%s>: Push image (base=%s)', task_id, base)
        last_line = notifier.store_and_broadcast(dockerjob.push_image(host, version))
        dockerjob.remove_image(version, host)
    except Exception, e:
        task.finish(consts.TASK_FAILED)
        task.reason = str(e.message)
        notifier.pub_fail()
        _log.error('Task<id=%s>, exception', task_id)
        _log.exception(e)

def task_log(task_id):
    ws = request.environ['wsgi.websocket']

    task = Task.get(task_id)
    if not task:
        ws.close()
        logger.info('Task %s not found, close websocket' % task_id)
        return 'websocket closed'

    notifier = TaskNotifier(task)
    try:
        pub = rds.pubsub()
        pub.subscribe(task.publish_key)

        for line in notifier.get_store_logs():
            ws.send(line)

        if task.finished:
            return ''

        for line in pub.listen():
            if line['data'] == code.PUB_END_MESSAGE:
                break
            if line['type'] != 'message':
                continue
            ws.send(line['data'])
    except geventwebsocket.WebSocketError, e:
        logger.exception(e)

def test_create_container(client, test_db):
    # 反正本地也跑不过 -_-!
    return

    app, version, group, pod, host = create_local_test_data()
    rv = client.post(
        '/api/deploy/public/group/pod/blueberry',
        data=json.dumps({
            'ncontainer': 1,
            'version': version.sha,
            'entrypoint': 'web',
            'env': 'prod'
        }),
        content_type='application/json')
    assert rv.status_consts == 200
    r = json.loads(rv.data)
    assert len(r['tasks']) == 1
    task_id = r['tasks'][0]
    assert task_id
    task = Task.get(task_id)
    assert task.host_id == host.id
    assert task.app_id == app.id
    assert task.version_id == version.id
    assert task.type == consts.TASK_CREATE
    props = task.props
    assert props['ncontainer'] == 1
    assert props['entrypoint'] == 'web'
    assert props['cores'] == []

def _create_task(type_, version, host, ncontainer, cores, nshare, networks, spec_ips, entrypoint, env, image=""):
    network_ids = [n.id for n in networks]
    task_props = {
        "ncontainer": ncontainer,
        "entrypoint": entrypoint,
        "env": env,
        "full_cores": [c.label for c in cores.get("full", [])],
        "part_cores": [c.label for c in cores.get("part", [])],
        "nshare": nshare,
        "networks": network_ids,
        "image": image,
    }
    task = Task.create(type_, version, host, task_props)
    if not task:
        return None

    try:
        create_containers_with_macvlan.apply_async(
            args=(task.id, ncontainer, nshare, cores, network_ids, spec_ips), task_id="task:%d" % task.id
        )
    except Exception as e:
        logger.exception(e)
        host.release_cores(cores)

    return task

def remove_containers(task_id, cids, rmi=False):
    task = Task.get(task_id)
    if not task:
        _log.error('Task (id=%s) not found, quit', task_id)
        return

    _log.info('Task<id=%s>: Start on host %s', task_id, task.host.ip)
    notifier = TaskNotifier(task)

    containers = Container.get_multi(cids)
    if not containers:
        _log.error('Task (id=%s) no container found, quit')
        return

    host = containers[0].host

    for c in containers:
        c.in_removal = 1

    container_ids = [c.container_id for c in containers if c]
    try:
        set_flag_for_agent(container_ids)
        for c in containers:
            remove_container_backends(c)
            _log.info('Task<id=%s>: Container (cid=%s) backends removed', task_id, c.short_id)

        appnames = {c.appname for c in containers}
        publish_to_service_discovery(*appnames)

        time.sleep(3)

        dockerjob.remove_host_containers(containers, host)
        _log.info('Task<id=%s>: Containers (cids=%s) removed', task_id, cids)

        if rmi:
            try:
                dockerjob.remove_image(task.version, host)
            except Exception as e:
                _log.error('Task<id=%s>, fail to remove image', task_id, e)
    except Exception as e:
        task.finish(consts.TASK_FAILED)
        task.reason = str(e.message)
        notifier.pub_fail()
        _log.error('Task<id=%s> exception', task_id)
        _log.exception(e)
    else:
        for c in containers:
            c.delete()
        task.finish(consts.TASK_SUCCESS)
        task.reason = 'ok'
        notifier.pub_success()
        remove_container_for_agent(host, container_ids)
        remove_flag_for_agent(container_ids)
        _log.info('Task<id=%s>: Done', task_id)

def build_image(group_name, pod_name, appname):
    data = request.get_json()
    group, pod, application, version = validate_instance(group_name, pod_name, appname, data["version"])
    # TODO
    # 这个group可以用这个pod不?
    # 这个group可以build这个version不?
    base = data["base"]
    host = pod.get_random_host()
    task = Task.create(consts.TASK_BUILD, version, host, {"base": base})
    build_docker_image.apply_async(args=(task.id, base), task_id="task:%d" % task.id)
    return {"r": 0, "msg": "ok", "task": task.id, "watch_key": task.result_key}

def build_image(group_name, pod_name, appname):
    data = request.get_json()
    group, pod, application, version = validate_instance(group_name,
            pod_name, appname, data['version'])
    # TODO
    # 这个group可以用这个pod不?
    # 这个group可以build这个version不?
    base = data['base']
    host = pod.get_random_host()
    task = Task.create(consts.TASK_BUILD, version, host, {'base': base})
    build_docker_image.apply_async(
        args=(task.id, base),
        task_id='task:%d' % task.id
    )
    return {'r': 0, 'msg': 'ok', 'task': task.id, 'watch_key': task.result_key}

def offline_version(group_name, pod_name, appname):
    data = request.get_json()
    group, pod, application, version = validate_instance(group_name, pod_name, appname, data["version"])
    d = {}
    ts, keys = [], []
    for container in version.containers.all():
        d.setdefault(container.host, []).append(container)
    for host, containers in d.iteritems():
        cids = [c.id for c in containers]
        task_props = {"container_ids": cids}
        task = Task.create(consts.TASK_REMOVE, version, host, task_props)
        remove_containers.apply_async(args=(task.id, cids, True), task_id="task:%d" % task.id)
        ts.append(task.id)
        keys.append(task.result_key)
    return {"r": 0, "msg": "ok", "tasks": ts, "watch_keys": keys}

def build_image(group_name, pod_name, appname):
    data = request.get_json()
    group, pod, _, version = validate_instance(group_name, pod_name, appname, data['version'])

    base = data['base']
    if ':' not in base:
        base = base + ':latest'

    host = Host.get_random_public_host() or pod.get_random_host()
    task = Task.create(consts.TASK_BUILD, version, host, {'base': base})
    build_docker_image.apply_async(
        args=(task.id, base),
        task_id='task:%d' % task.id
    )
    return {'r': 0, 'msg': 'ok', 'task': task.id, 'watch_key': task.result_key}

def remove_containers(task_id, cids, rmi=False):
    task = Task.get(task_id)
    if not task:
        current_flask.logger.error('Task (id=%s) not found, quit', task_id)
        return

    current_flask.logger.info('Task<id=%s>: Start on host %s' % (task_id, task.host.ip))
    notifier = TaskNotifier(task)
    containers = Container.get_multi(cids)
    container_ids = [c.container_id for c in containers if c]
    host = task.host
    version = task.version
    try:
        # flag, don't report these
        flags = {'eru:agent:%s:container:flag' % cid: 1 for cid in container_ids}
        rds.mset(**flags)
        for c in containers:
            remove_container_backends(c)
            current_flask.logger.info('Task<id=%s>: Container (cid=%s) backends removed',
                    task_id, c.container_id[:7])
        appnames = {c.appname for c in containers}
        publish_to_service_discovery(*appnames)

        time.sleep(3)

        dockerjob.remove_host_containers(containers, host)
        current_flask.logger.info('Task<id=%s>: Containers (cids=%s) removed', task_id, cids)
        if rmi:
            try:
                dockerjob.remove_image(task.version, host)
            except Exception as e:
                current_flask.logger.error('Task<id=%s>: Exception (e=%s), fail to remove image', task_id, e)
    except Exception as e:
        task.finish_with_result(consts.TASK_FAILED)
        notifier.pub_fail()
        current_flask.logger.error('Task<id=%s>: Exception (e=%s)', task_id, e)
    else:
        for c in containers:
            c.delete()
        task.finish_with_result(consts.TASK_SUCCESS)
        notifier.pub_success()
        if container_ids:
            rds.hdel('eru:agent:%s:containers:meta' % host.name, *container_ids)
        rds.delete(*flags.keys())
        current_flask.logger.info('Task<id=%s>: Done', task_id)

    if not version.containers.count():
        falcon_remove_alarms(version)