Python cloud.Compute类代码示例

    def chk_and_launch_docker(self, user_id):
        if self.redirect_to_logged_in_instance(user_id):
            return

        nhops = int(self.get_argument('h', 0))
        numhopmax = JBoxCfg.get('numhopmax', 0)
        max_hop = nhops > numhopmax
        launched = self.try_launch_container(user_id, max_hop=max_hop)

        if launched:
            self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
            self.rendertpl("loading.tpl",
                           user_id=user_id,
                           cfg=JBoxCfg.nv,
                           js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
            return

        self.unset_affinity()
        self.log_debug("at hop %d for user %s", nhops, user_id)
        if max_hop:
            if JBoxCfg.get('cloud_host.scale_down'):
                msg = "JuliaBox is experiencing a sudden surge. Please try in a few minutes while we increase our capacity."
            else:
                msg = "JuliaBox servers are fully loaded. Please try after sometime."
            self.log_error("Server maxed out. Can't launch container at hop %d for user %s", nhops, user_id)
            self.rendertpl("index.tpl", cfg=JBoxCfg.nv, state=self.state(error=msg, success=''))
        else:
            redirect_instance = Compute.get_redirect_instance_id()
            if redirect_instance is not None:
                redirect_ip = Compute.get_instance_local_ip(redirect_instance)
                self.set_redirect_instance_id(redirect_ip)
            self.redirect('/?h=' + str(nhops + 1))

    def post_auth_launch_container(self, user_id):
        for plugin in JBPluginHandler.jbox_get_plugins(JBPluginHandler.JBP_HANDLER_POST_AUTH):
            self.log_info("Passing user %r to post auth plugin %r", user_id, plugin)
            pass_allowed = plugin.process_user_id(self, user_id)
            if not pass_allowed:
                self.log_info('Login restricted for user %r by plugin %r', user_id, plugin)
                return

        jbuser = JBoxUserV2(user_id, create=True)

        if not JBPluginHandler.is_user_activated(jbuser):
            self.redirect('/?pending_activation=' + user_id)
            return

        self.set_authenticated(user_id)
        if jbuser.is_new:
            jbuser.save()

        if self.redirect_to_logged_in_instance(user_id):
            return

        # check if the current instance is appropriate for launching this
        if self.try_launch_container(user_id, max_hop=False):
            self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
        else:
            # redirect to an appropriate instance
            redirect_instance = Compute.get_redirect_instance_id()
            if redirect_instance is not None:
                redirect_ip = Compute.get_instance_local_ip(redirect_instance)
                self.set_redirect_instance_id(redirect_ip)
        self.redirect('/')

def is_cluster_leader():
    self_id = Compute.get_instance_id()
    cluster = Compute.get_install_id()
    instances = Compute.get_all_instances()
    leader = JBoxDynConfig.get_cluster_leader(cluster)
    img_recentness = Compute.get_image_recentness()
    JBoxDB.log_debug("cluster: %s. instances: %s. leader: %s. image recentness: %d",
                     cluster, repr(instances), repr(leader), img_recentness)

    # if none set, or set instance is dead elect self as leader, but wait till next cycle to prevent conflicts
    if (leader is None) or (leader not in instances) and (img_recentness >= 0):
        JBoxDB.log_info("setting self (%s) as cluster leader", self_id)
        try:
            JBoxDynConfig.set_cluster_leader(cluster, self_id)
        except:
            JBoxDB.log_info("error setting self (%s) as cluster leader, will retry", self_id)
        return False

    is_leader = (leader == self_id)

    # if running an older ami, step down from cluster leader
    if (img_recentness < 0) and is_leader:
        JBoxDB.log_info("unmarking self (%s) as cluster leader", self_id)
        JBoxDynConfig.unset_cluster_leader(cluster)
        return False

    return is_leader

 def redirect_to_logged_in_instance(self, user_id):
     loggedin_instance = self.find_logged_in_instance(user_id)
     if loggedin_instance is not None \
             and loggedin_instance != Compute.get_instance_id() \
             and loggedin_instance != 'localhost':
         # redirect to the instance that has the user's session
         self.log_info("Already logged in to %s. Redirecting", loggedin_instance)
         redirect_ip = Compute.get_instance_local_ip(loggedin_instance)
         self.set_redirect_instance_id(redirect_ip)
         self.redirect('/')
         return True
     self.log_info("Logged in %s", "nowhere" if loggedin_instance is None else "here already")
     return False

    def handle_get_metadata(self, is_admin, courses_offered):
        mode = self.get_argument('mode', None)
        if (mode is None) or (mode != "metadata"):
            return False

        self.log_debug("handling answers")
        params = self.get_argument('params', None)
        params = json.loads(params)
        course_id = params['course']
        problemset_id = params['problemset']
        question_ids = params['questions'] if 'questions' in params else None
        send_answers = True

        if (not is_admin) and (course_id not in courses_offered):
            send_answers = False

        err = None
        course = JBoxDynConfig.get_course(Compute.get_install_id(), course_id)
        self.log_debug("got course %r", course)
        if problemset_id not in course['problemsets']:
            err = "Problem set %s not found!" % (problemset_id,)
        if question_ids is None:
            question_ids = course['questions'][problemset_id]

        if err is None:
            report = JBoxCourseHomework.get_problemset_metadata(course_id, problemset_id, question_ids, send_answers)
            code = 0
        else:
            report = err
            code = -1

        response = {'code': code, 'data': report}
        self.write(response)
        return True

 def find_logged_in_instance(user_id):
     sessname = unique_sessname(user_id)
     try:
         sess_props = JBoxSessionProps(Compute.get_install_id(), sessname)
         return sess_props.get_instance_id()
     except JBoxDBItemNotFound:
         return None

 def do_cluster_housekeeping():
     JBoxEBSHousekeep.log_debug("starting cluster housekeeping")
     detached_disks = JBoxDiskState.get_detached_disks()
     time_now = datetime.datetime.now(pytz.utc)
     for disk_key in detached_disks:
         disk_info = JBoxDiskState(disk_key=disk_key)
         user_id = disk_info.get_user_id()
         sess_props = JBoxSessionProps(Compute.get_install_id(), unique_sessname(user_id))
         incomplete_snapshots = []
         modified = False
         for snap_id in disk_info.get_snapshot_ids():
             if not EBSVol.is_snapshot_complete(snap_id):
                 incomplete_snapshots.append(snap_id)
                 continue
             JBoxEBSHousekeep.log_debug("updating latest snapshot of user %s to %s", user_id, snap_id)
             old_snap_id = sess_props.get_snapshot_id()
             sess_props.set_snapshot_id(snap_id)
             modified = True
             if old_snap_id is not None:
                 EBSVol.delete_snapshot(old_snap_id)
         if modified:
             sess_props.save()
             disk_info.set_snapshot_ids(incomplete_snapshots)
             disk_info.save()
         if len(incomplete_snapshots) == 0:
             if (time_now - disk_info.get_detach_time()).total_seconds() > 24 * 60 * 60:
                 vol_id = disk_info.get_volume_id()
                 JBoxEBSHousekeep.log_debug("volume %s for user %s unused for too long", vol_id, user_id)
                 disk_info.delete()
                 EBSVol.detach_volume(vol_id, delete=True)
         else:
             JBoxEBSHousekeep.log_debug("ongoing snapshots of user %s: %r", user_id, incomplete_snapshots)
     JBoxEBSHousekeep.log_debug("finished cluster housekeeping")

    def update_user_home_image(fetch=True):
        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is None:
            VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
            return

        home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
        pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)

        bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())

        new_home_img_path = os.path.join(home_img_dir, new_home_img)
        new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
        updated = False
        for img_path in (new_home_img_path, new_pkg_img_path):
            if not os.path.exists(img_path):
                if fetch:
                    VolMgr.log_debug("fetching new image to %s", img_path)
                    k = plugin.pull(bucket, img_path)
                    if k is not None:
                        VolMgr.log_debug("fetched new image")

        if os.path.exists(new_home_img_path):
            VolMgr.log_debug("set new home image to %s", new_home_img_path)
            JBoxVol.USER_HOME_IMG = new_home_img_path
            updated = True

        if os.path.exists(new_pkg_img_path):
            VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
            JBoxVol.PKG_IMG = new_pkg_img_path
            updated = True

        return updated

    def get(self):
        self.log_debug("APIInfo handler got GET request")
        key = self.get_argument("key", None)
        sign = self.get_argument("sign", None)

        if key is None or sign is None:
            self.send_error()
            return
        sign2 = signstr(key, JBoxCfg.get('sesskey'))

        if sign != sign2:
            self.log_info("signature mismatch. key:%r sign:%r expected:%r", key, sign, sign2)
            self.send_error()
            return

        api_status = JBoxInstanceProps.get_instance_status(Compute.get_install_id())
        self.log_info("cluster api status: %r", api_status)

        # filter out instances that should not accept more load
        filtered_api_status = {k: v for (k, v) in api_status.iteritems() if v['accept']}
        preferred_instances = filtered_api_status.keys()

        # flip the dict
        per_api_instances = dict()
        for (inst, status) in filtered_api_status.iteritems():
            api_names = status['api_status'].keys()
            for api_name in api_names:
                v = per_api_instances.get(api_name, [])
                v.append(inst)

        per_api_instances[" preferred "] = preferred_instances
        self.log_info("per api instances: %r", per_api_instances)
        self.write(per_api_instances)
        return

    def handle_if_instance_info(self, is_allowed):
        stats = self.get_argument('instance_info', None)
        if stats is None:
            return False

        if not is_allowed:
            AdminHandler.log_error("Show instance info not allowed for user")
            response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
        else:
            try:
                if stats == 'load':
                    result = {}
                    # get cluster loads
                    average_load = Compute.get_cluster_average_stats('Load')
                    if None != average_load:
                        result['Average Load'] = average_load

                    machine_loads = Compute.get_cluster_stats('Load')
                    if None != machine_loads:
                        for n, v in machine_loads.iteritems():
                            result['Instance ' + n] = v
                elif stats == 'sessions':
                    result = dict()
                    instances = Compute.get_all_instances()

                    for idx in range(0, len(instances)):
                        try:
                            inst = instances[idx]
                            result[inst] = JBoxAsyncJob.sync_session_status(inst)['data']
                        except:
                            JBoxHandler.log_error("Error receiving sessions list from %r", inst)
                elif stats == 'apis':
                    result = APIContainer.get_cluster_api_status()
                else:
                    raise Exception("unknown command %s" % (stats,))

                response = {'code': 0, 'data': result}
            except:
                AdminHandler.log_error("exception while getting stats")
                AdminHandler._get_logger().exception("exception while getting stats")
                response = {'code': -1, 'data': 'error getting stats'}

        self.write(response)
        return True

 def setup_instance_config(self, profiles=('default',)):
     for profile in profiles:
         profile_path = '.ipython/profile_' + profile
         profile_path = os.path.join(self.disk_path, profile_path)
         if not os.path.exists(profile_path):
             continue
         nbconfig = os.path.join(profile_path, 'ipython_notebook_config.py')
         wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
                     Compute.get_alias_hostname() + "'\n"
         JBoxVol.replace_in_file(nbconfig, "c.NotebookApp.websocket_url", wsock_cfg)

    def get(self):
        args = self.get_argument('m', default=None)

        if args is not None:
            self.log_debug("setting cookies")
            self.unpack(args)
            self.set_status(status_code=204)
            self.finish()
        else:
            args = self.pack()
            url = "//" + Compute.get_alias_hostname() + "/jboxcors/?m=" + args
            self.log_debug("redirecting to " + url)
            self.redirect(url)

 def get_cluster_api_status():
     result = dict()
     for inst in Compute.get_all_instances():
         try:
             api_status = JBoxAsyncJob.sync_api_status(inst)
             if api_status['code'] == 0:
                 result[inst] = api_status['data']
             else:
                 APIContainer.log_error("error fetching api status from %s", inst)
         except:
             APIContainer.log_error("exception fetching api status from %s", inst)
     APIContainer.log_debug("api status: %r", result)
     return result

    def get_active_sessions():
        instances = Compute.get_all_instances()

        active_sessions = set()
        for inst in instances:
            try:
                sessions = JBoxAsyncJob.sync_session_status(inst)['data']
                if len(sessions) > 0:
                    for sess_id in sessions.keys():
                        active_sessions.add(sess_id)
            except:
                SessContainer.log_error("Error receiving sessions list from %r", inst)

        return active_sessions

    def post_auth_launch_container(self, user_id):
        jbuser = JBoxUserV2(user_id, create=True)
        if not JBPluginHandler.is_user_activated(jbuser):
            self.redirect('/?pending_activation=' + user_id)
            return

        self.set_authenticated(user_id)
        if jbuser.is_new:
            jbuser.save()

        if self.redirect_to_logged_in_instance(user_id):
            return

        # check if the current instance is appropriate for launching this
        if self.try_launch_container(user_id, max_hop=False):
            self.set_container_initialized(Compute.get_instance_local_ip(), user_id)
        else:
            # redirect to an appropriate instance
            redirect_instance = Compute.get_redirect_instance_id()
            if redirect_instance is not None:
                redirect_ip = Compute.get_instance_local_ip(redirect_instance)
                self.set_redirect_instance_id(redirect_ip)
        self.redirect('/')