Python models.Host类代码示例

def _get_host(id_or_name):
    if id_or_name.isdigit():
        host = Host.get(id_or_name)
    else:
        host = Host.get_by_name(id_or_name)
    if not host:
        abort(404, 'Host %s not found' % id_or_name)
    return host

def test_get_max_container_count_single_host(test_db):
    pod = Pod.create('pod', 'pod', 10, -1)
    Host.create(pod, random_ipv4(), random_string(), random_uuid(), 64, 4096)

    assert get_max_container_count(pod, ncore=1, nshare=0) == 64
    assert get_max_container_count(pod, ncore=2, nshare=0) == 32
    assert get_max_container_count(pod, ncore=3, nshare=0) == 21
    assert get_max_container_count(pod, ncore=4, nshare=0) == 16
    assert get_max_container_count(pod, ncore=5, nshare=0) == 12
    assert get_max_container_count(pod, ncore=1, nshare=5) == 42
    assert get_max_container_count(pod, ncore=2, nshare=5) == 25

def test_container_release_cores(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
    v = a.add_version(random_sha1())
    p = Pod.create('pod', 'pod', 10, -1)
    host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200, 0)

    for core in host.cores:
        assert core.host_id == host.id
        assert core.remain == 10

    containers = []
    
    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
        used_cores = {'full': fcores, 'part': pcores}
        host.occupy_cores(used_cores, 5)
        c = Container.create(random_sha1(), host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=5)
        containers.append(c)

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
        for core in fcores:
            assert core.remain == 0
        for core in pcores:
            assert core.remain == 5

    for c in containers:
        c.delete()

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
        for core in fcores:
            assert core.remain == 10
        for core in pcores:
            assert core.remain == 10

def create_local_test_data(private=False):
    appyaml = {
        'appname': 'blueberry',
        'entrypoints': {
            'web': {
                'cmd': 'python app.py',
                'ports': ['5000/tcp'],
            },
            'daemon': {
                'cmd': 'python daemon.py',
            },
            'service': {
                'cmd': 'python service.py'
            },
        },
        'build': 'pip install -r ./requirements.txt',
    }
    app = App.get_or_create('blueberry', 'http://git.hunantv.com/tonic/blueberry.git', 'token')
    version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306')
    appconfig = version.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    group = Group.create('group', 'group')
    pod = Pod.create('pod', 'pod')
    pod.assigned_to_group(group)

    c = docker.Client(**kwargs_from_env(assert_hostname=False))
    r = c.info()
    host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'], r['NCPU'], r['MemTotal'])

    if private:
        host.assigned_to_group(group)

    return app, version, group, pod, host

def build_image_v2():
    # form post
    appname = request.form.get('appname', default='')
    version = request.form.get('version', default='')
    base = request.form.get('base', default='')
    if not base:
        abort(400, 'base image must be set')
    _, version = _get_app_and_version(appname, version)

    if ':' not in base:
        base = base + ':latest'

    host = Host.get_random_public_host()
    if not host:
        abort(406, 'no host is available')

    # if no artifacts.zip is set
    # ignore and just do the cloning and building
    file_path = None
    if 'artifacts.zip' in request.files:
        f = request.files['artifacts.zip']
        file_path = os.path.join(tempfile.mkdtemp(), secure_filename(f.filename))
        f.save(file_path)

    task = Task.create(TASK_BUILD, version, host, {'base': base})
    build_docker_image.apply_async(
        args=(task.id, base, file_path),
        task_id='task:%d' % task.id
    )
    return {'task': task.id, 'watch_key': task.result_key}

def _create_data(core_share, max_share_core, host_count):
    group = Group.create('group', 'group')
    pod = Pod.create('pod', 'pod', core_share, max_share_core)
    for _ in range(host_count):
        host = Host.create(pod, random_ipv4(), random_string(), random_uuid(), 16, 4096)
        host.assigned_to_group(group)
    return group, pod

def create_host():
    """为了文件, 只好不用json了"""
    addr = request.form.get('addr', type=str, default='')
    pod_name = request.form.get('pod_name', type=str, default='')
    if not (addr and pod_name):
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'need addr and pod_name')

    pod = Pod.get_by_name(pod_name)
    if not pod:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'No pod found')

    # 存证书, 没有就算了
    try:
        ca, cert, key = request.files['ca'], request.files['cert'], request.files['key']
        save_docker_certs(addr.split(':', 1)[0], ca.read(), cert.read(), key.read())
    finally:
        ca.close()
        cert.close()
        key.close()

    try:
        client = get_docker_client(addr, force_flush=True)
        info = client.info()
    except Exception:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Docker daemon error on host %s' % addr)

    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'], info['MemTotal']):
        raise EruAbortException(consts.HTTP_BAD_REQUEST)
    return consts.HTTP_CREATED, {'r':0, 'msg': consts.OK}

def create_host():
    """为了文件, 只好不用json了"""
    addr = request.form.get('addr', default='')
    pod_name = request.form.get('pod_name', default='')
    if not (addr and pod_name):
        abort(400, 'Need addr and pod_name')

    pod = Pod.get_by_name(pod_name)
    if not pod:
        abort(400, 'No pod found')

    # 存证书, 没有就算了
    if all(k in request.files for k in ['ca', 'cert', 'key']):
        try:
            ca, cert, key = request.files['ca'], request.files['cert'], request.files['key']
            save_docker_certs(addr.split(':', 1)[0], ca.read(), cert.read(), key.read())
        finally:
            ca.close()
            cert.close()
            key.close()

    try:
        client = get_docker_client(addr, force_flush=True)
        info = client.info()
    except Exception as e:
        abort(400, 'Docker daemon error on host %s, error: %s' % (addr, e.message))

    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'], info['MemTotal']):
        abort(400, 'Host create error.')
    return 201, {'r':0, 'msg': consts.OK}

def create_host():
    """为了文件, 只好不用json了"""
    addr = request.form.get('addr', default='')
    ip = addr.split(':', 1)[0]
    podname = request.form.get('podname', default='')
    is_public = request.form.get('is_public', default=False, type=bool)
    if not (addr and podname):
        abort(400, 'Bad addr or podname: addr="{}", podname="{}"'.format(addr, podname))

    pod = Pod.get_by_name(podname)
    if not pod:
        abort(400, 'Pod {} not found'.format(podname))

    # 存证书, 没有就算了
    certs = ['ca', 'cert', 'key']
    if all(k in request.files for k in certs):
        certs_contents = tuple(request.files[f].read() for f in certs)
        save_docker_certs(ip, *certs_contents)

    try:
        client = get_docker_client(addr, force_flush=True)
        info = client.info()
    except Exception as e:
        abort(400, 'Docker daemon error on host %s, error: %s' % (addr, e.message))

    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'],
                       info['MemTotal'], is_public=is_public):
        abort(400, 'Error while creating host')

    return 201, DEFAULT_RETURN_VALUE

def create_private():
    data = request.get_json()
    pod, _, version = _get_instances(**data)

    ncore, nshare = pod.get_core_allocation(float(data['ncore']))
    ports = data.get('ports', [])
    args = data.get('args', [])
    strategy = data.get('strategy', 'average')

    callback_url = data.get('callback_url', '')
    if callback_url and not is_strict_url(callback_url):
        abort(400, 'callback_url must start with http:// or https://')

    ncontainer = int(data['ncontainer'])
    if not ncontainer:
        abort(400, 'ncontainer must be > 0')

    networks = [ipam.get_pool(n) for n in data.get('networks', [])]
    spec_ips = data.get('spec_ips', [])
    appconfig = version.appconfig

    entrypoint = data['entrypoint']
    if entrypoint not in appconfig.entrypoints:
        abort(400, 'Entrypoint %s not in app.yaml' % entrypoint)

    hostname = data.get('hostname', '')
    host = hostname and Host.get_by_name(hostname) or None

    task_ids, watch_keys = [], []
    host_cores = _get_strategy(strategy)(pod, ncontainer, ncore, nshare, host)
    if not host_cores:
        abort(400, 'Not enough core resources')

    for (host, container_count), cores in host_cores.iteritems():
        t = _create_task(
            version,
            host,
            container_count,
            cores,
            nshare,
            networks,
            ports,
            args,
            spec_ips,
            entrypoint,
            data['env'],
            image=data.get('image', ''),
            callback_url=callback_url,
        )
        if not t:
            continue

        host.occupy_cores(cores, nshare)
        task_ids.append(t.id)
        watch_keys.append(t.result_key)

    return {'tasks': task_ids, 'watch_keys': watch_keys}

def create_private(group_name, pod_name, appname):
    """ncore: 需要的核心数, 可以是小数, 例如1.5个"""
    data = request.get_json()

    if data.get('raw', ''):
        vstr = consts.RAW_VERSION_PLACEHOLDER
    else:
        vstr = data['version']

    group, pod, application, version = validate_instance(group_name,
            pod_name, appname, vstr)

    # TODO check if group has this pod

    core_require = int(float(data['ncore']) * pod.core_share) # 是说一个容器要几个核...
    ncore = core_require / pod.core_share
    nshare = core_require % pod.core_share

    ncontainer = int(data['ncontainer'])
    networks = Network.get_multi(data.get('networks', []))
    spec_ips = data.get('spec_ips', [])
    appconfig = version.appconfig

    # 指定的host, 如果没有则按照编排分配host
    hostname = data.get('hostname', '')
    host = hostname and Host.get_by_name(hostname) or None
    if host and not (host.group_id == group.id and host.pod_id == pod.id):
        current_app.logger.error('Host must belong to pod/group (hostname=%s, pod=%s, group=%s)',
                host, pod_name, group_name)
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Host must belong to this pod and group')

    if not data['entrypoint'] in appconfig.entrypoints:
        current_app.logger.error('Entrypoint not in app.yaml (entry=%s, name=%s, version=%s)',
                data['entrypoint'], appname, version.short_sha)
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Entrypoint %s not in app.yaml' % data['entrypoint'])

    ts, keys = [], []
    with rds.lock('%s:%s' % (group_name, pod_name)):
        host_cores = group.get_free_cores(pod, ncontainer, ncore, nshare, spec_host=host)
        if not host_cores:
            current_app.logger.error('Not enough cores (name=%s, version=%s, ncore=%s)',
                    appname, version.short_sha, data['ncore'])
            raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Not enough core resources')

        for (host, container_count), cores in host_cores.iteritems():
            t = _create_task(consts.TASK_CREATE, version, host, container_count,
                    cores, nshare, networks, spec_ips, data['entrypoint'], data['env'],
                    image=data.get('image', ''))
            if not t:
                continue

            host.occupy_cores(cores, nshare)
            ts.append(t.id)
            keys.append(t.result_key)

    return {'r': 0, 'msg': 'ok', 'tasks': ts, 'watch_keys': keys}

def get_host_resource(host_id):
    host = Host.get(host_id)
    if not host:
        raise EruAbortException(code.HTTP_NOT_FOUND, 'Host %s not found' % host_id)
    core_count = len(host.cores.all())
    free_cores = host.get_free_cores()
    return {
        'core_count': core_count,
        'free_cores': [c.label for c in free_cores],
        'memory': host.mem,
    }

def test_container_transform(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
    assert a is not None

    v = a.add_version(random_sha1())
    v2 = a.add_version(random_sha1())
    assert v is not None
    assert v.app.id == a.id
    assert v.name == a.name
    assert len(v.containers.all()) == 0
    assert len(v.tasks.all()) == 0

    g = Group.create('group', 'group')
    p = Pod.create('pod', 'pod')
    assert p.assigned_to_group(g)
    hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'),
        random_uuid(), 4, 4096) for i in range(6)]

    for host in hosts[:3]:
        host.assigned_to_group(g)

    assert g.get_max_containers(p, 3, 0) == 3
    host_cores = g.get_free_cores(p, 3, 3, 0)
    assert len(host_cores) == 3

    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]}
            c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
            assert c is not None
            containers.append(c)
        host.occupy_cores(cores, 0)

    for host in g.private_hosts.all():
        assert len(host.get_free_cores()[0]) == 1
        assert len(host.containers.all()) == 1
        assert host.count == 1

    assert len(containers) == 3
    assert len(v.containers.all()) == 3

    cids = [c.container_id for c in containers]
    for c in containers:
        host = c.host
        cid = c.container_id
        c.transform(v2, random_sha1(), random_string())
        assert c.container_id != cid

    new_cids = [c.container_id for c in containers]
    assert new_cids != cids

def create_host():
    data = request.get_json()
    addr = data['addr']

    pod = Pod.get_by_name(data['pod_name'])
    if not pod:
        raise EruAbortException(code.HTTP_BAD_REQUEST)

    client = get_docker_client(addr)
    info = client.info()
    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'], info['MemTotal']):
        raise EruAbortException(code.HTTP_BAD_REQUEST)
    return {'r':0, 'msg': code.OK}

def get_host_resource(host_id):
    host = Host.get(host_id)
    if not host:
        abort(404, 'Host %s not found' % host_id)

    core_count = len(host.cores)
    fe, fs = host.get_free_cores()
    return {
        'core_count': core_count,
        'free_excluded_cores': [c.label for c in fe],
        'free_shared_cores': [c.label for c in fs],
        'memory': host.mem,
    }