Python aws.CloudHost类代码示例

    def auto_activate():
        try:
            num_mails_24h, rate = CloudHost.get_email_rates()
            rate_per_second = min(JBoxd.MAX_ACTIVATIONS_PER_SEC, rate)
            num_mails = min(JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN, num_mails_24h)

            JBoxd.log_info("Will activate max %d users at %d per second. AWS limits: %d mails at %d per second",
                           num_mails, rate_per_second,
                           num_mails_24h, rate)
            user_ids = JBoxUserV2.get_pending_activations(num_mails)
            JBoxd.log_info("Got %d user_ids to be activated", len(user_ids))

            for user_id in user_ids:
                JBoxd.log_info("Activating %s", user_id)

                # send email by SES
                CloudHost.send_email(user_id, JBoxd.ACTIVATION_SENDER, JBoxd.ACTIVATION_SUBJECT, JBoxd.ACTIVATION_BODY)

                # set user as activated
                user = JBoxUserV2(user_id)
                user.set_activation_state(JBoxUserV2.ACTIVATION_CODE_AUTO, JBoxUserV2.ACTIVATION_GRANTED)
                user.save()

                rate_per_second -= 1
                if rate_per_second <= 0:
                    time.sleep(1)
                    rate_per_second = min(JBoxd.MAX_ACTIVATIONS_PER_SEC, rate)
        finally:
            JBoxd.finish_thread()

    def handle_if_instance_info(self, is_allowed):
        stats = self.get_argument('instance_info', None)
        if stats is None:
            return False

        if not is_allowed:
            AdminHandler.log_error("Show instance info not allowed for user")
            response = {'code': -1, 'data': 'You do not have permissions to view these stats'}
        else:
            try:
                if stats == 'load':
                    stats = {}
                    # get cluster loads
                    average_load = CloudHost.get_cluster_average_stats('Load')
                    if None != average_load:
                        stats['Average Load'] = average_load;

                    machine_loads = CloudHost.get_cluster_stats('Load')
                    if None != machine_loads:
                        for n, v in machine_loads.iteritems():
                            stats['Instance ' + n] = v

                response = {'code': 0, 'data': stats} if stats is not None else {'code': 1, 'data': {}}
            except:
                AdminHandler.log_error("exception while getting stats")
                response = {'code': -1, 'data': 'error getting stats'}

        self.write(response)
        return True

    def __init__(self):
        dckr = docker.Client()
        cfg = read_config()
        cloud_cfg = cfg['cloud_host']
        user_activation_cfg = cfg['user_activation']

        LoggerMixin.setup_logger(level=cfg['root_log_level'])
        LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']

        db.configure_db(cfg)

        CloudHost.configure(has_s3=cloud_cfg['s3'],
                            has_dynamodb=cloud_cfg['dynamodb'],
                            has_cloudwatch=cloud_cfg['cloudwatch'],
                            has_autoscale=cloud_cfg['autoscale'],
                            has_route53=cloud_cfg['route53'],
                            has_ebs=cloud_cfg['ebs'],
                            has_ses=cloud_cfg['ses'],
                            scale_up_at_load=cloud_cfg['scale_up_at_load'],
                            scale_up_policy=cloud_cfg['scale_up_policy'],
                            autoscale_group=cloud_cfg['autoscale_group'],
                            route53_domain=cloud_cfg['route53_domain'],
                            region=cloud_cfg['region'],
                            install_id=cloud_cfg['install_id'])
        VolMgr.configure(dckr, cfg)
        JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
                                cfg['numlocalmax'], cfg['async_job_port'], async_mode=JBoxAsyncJob.MODE_SUB)
        self.log_debug("Backup daemon listening on port: " + str(cfg['async_job_port']))
        self.queue = JBoxContainer.ASYNC_JOB

        JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg['max_activations_per_sec']
        JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg['max_activations_per_run']
        JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject']
        JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body']
        JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']

 def release(self, backup=False):
     if not JBoxEBSVol.HAS_EBS:
         raise Exception("EBS disks not enabled")
     disk_id = self.disk_path.split('/')[-1]
     CloudHost.unmount_device(disk_id, JBoxEBSVol.FS_LOC)
     if backup:
         self._backup()
     vol_id = CloudHost.get_volume_id_from_device(disk_id)
     CloudHost.detach_volume(vol_id, delete=True)

 def test():
     CloudHost.send_email('[email protected]', '[email protected]', "test SES",
                          """hello world
                          Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
                          ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
                          ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
                          reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
                          Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit
                          anim id est laborum.""")

 def _backup(self, clear_volume=False):
     sess_props = JBoxSessionProps(self.sessname)
     desc = sess_props.get_user_id() + " JuliaBox Backup"
     disk_id = self.disk_path.split('/')[-1]
     snap_id = CloudHost.snapshot_volume(dev_id=disk_id, tag=self.sessname, description=desc)
     old_snap_id = sess_props.get_snapshot_id()
     sess_props.set_snapshot_id(snap_id)
     sess_props.save()
     if old_snap_id is not None:
         CloudHost.delete_snapshot(old_snap_id)

    def __init__(self):
        dckr = docker.Client()
        cfg = JBox.cfg = read_config()
        cloud_cfg = cfg['cloud_host']

        LoggerMixin.setup_logger(level=cfg['root_log_level'])
        LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']

        JBoxHandler.configure(cfg)
        db.configure_db(cfg)

        CloudHost.configure(has_s3=cloud_cfg['s3'],
                            has_dynamodb=cloud_cfg['dynamodb'],
                            has_cloudwatch=cloud_cfg['cloudwatch'],
                            has_autoscale=cloud_cfg['autoscale'],
                            has_route53=cloud_cfg['route53'],
                            has_ebs=cloud_cfg['ebs'],
                            has_ses=cloud_cfg['ses'],
                            scale_up_at_load=cloud_cfg['scale_up_at_load'],
                            scale_up_policy=cloud_cfg['scale_up_policy'],
                            autoscale_group=cloud_cfg['autoscale_group'],
                            route53_domain=cloud_cfg['route53_domain'],
                            region=cloud_cfg['region'],
                            install_id=cloud_cfg['install_id'])

        VolMgr.configure(dckr, cfg)
        JBoxAsyncJob.configure(cfg)
        JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
                                cfg['numlocalmax'], cfg['async_job_ports'])

        self.application = tornado.web.Application([
            (r"/", MainHandler),
            (r"/hostlaunchipnb/", AuthHandler),
            (r"/hostadmin/", AdminHandler),
            (r"/ping/", PingHandler),
            (r"/cors/", CorsHandler),
            (r"/hw/", HomeworkHandler)
        ])
        cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
        self.application.settings["cookie_secret"] = cookie_secret
        self.application.settings["google_oauth"] = cfg["google_oauth"]
        self.application.listen(cfg["port"])

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)

    def sendrecv(self, cmd, data, dest=None, port=None):
        if (dest is None) or (dest == 'localhost'):
            dest = '127.0.0.1'
        else:
            dest = CloudHost.instance_local_ip(dest)
        if port is None:
            port = self._rrport
        rraddr = 'tcp://%s:%d' % (dest, port)

        JBoxAsyncJob.log_debug("sendrecv to %s. connecting...", rraddr)
        sock = self._ctx.socket(zmq.REQ)
        sock.setsockopt(zmq.LINGER, 5*1000)
        sock.connect(rraddr)

        poller = zmq.Poller()
        poller.register(sock, zmq.POLLOUT)

        if poller.poll(10*1000):
            sock.send_json(self._make_msg(cmd, data))
        else:
            sock.close()
            raise IOError("could not connect to %s", rraddr)

        poller.modify(sock, zmq.POLLIN)
        if poller.poll(10*1000):
            msg = sock.recv_json()
        else:
            sock.close()
            raise IOError("did not receive anything from %s", rraddr)

        JBoxAsyncJob.log_debug("sendrecv to %s. received.", rraddr)
        sock.close()
        return msg

    def __init__(self, ports, mode):
        self._mode = mode
        self._ctx = zmq.Context()

        ppmode = zmq.PUSH if (mode == JBoxAsyncJob.MODE_PUB) else zmq.PULL
        self._push_pull_sock = self._ctx.socket(ppmode)

        rrmode = zmq.REQ if (mode == JBoxAsyncJob.MODE_PUB) else zmq.REP

        local_ip = CloudHost.instance_local_ip()
        JBoxAsyncJob.log_debug("local hostname [%s]", local_ip)

        ppbindaddr = 'tcp://%s:%d' % (local_ip, ports[0],)
        ppconnaddr = 'tcp://%s:%d' % (local_ip, ports[0],)
        rraddr = 'tcp://%s:%d' % (local_ip, ports[1],)
        self._rrport = ports[1]
        self._poller = zmq.Poller()

        if mode == JBoxAsyncJob.MODE_PUB:
            self._push_pull_sock.bind(ppbindaddr)
        else:
            self._push_pull_sock.connect(ppconnaddr)
            self._poller.register(self._push_pull_sock, zmq.POLLIN)
            self._req_rep_sock = self._ctx.socket(rrmode)
            self._req_rep_sock.bind(rraddr)

    def _backup(self, clear_volume=False):
        JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC))

        bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
        bkup_tar = tarfile.open(bkup_file, 'w:gz')

        for f in os.listdir(self.disk_path):
            if f.startswith('.') and (f in ['.julia', '.ipython']):
                continue
            full_path = os.path.join(self.disk_path, f)
            bkup_tar.add(full_path, os.path.join('juser', f))
        bkup_tar.close()
        os.chmod(bkup_file, 0666)

        if clear_volume:
            ensure_delete(self.disk_path)

        # Upload to S3 if so configured. Delete from local if successful.
        bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
            datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
        if JBoxVol.BACKUP_BUCKET is not None:
            if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file,
                                           metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
                os.remove(bkup_file)
                JBoxVol.log_info("Moved backup to S3 " + self.sessname)

    def get_disk_for_user(user_email):
        JBoxEBSVol.log_debug("creating EBS volume for %s", user_email)
        if not JBoxEBSVol.HAS_EBS:
            raise Exception("EBS disks not enabled")

        disk_id = JBoxEBSVol._reserve_disk_id()
        if disk_id is None:
            raise Exception("No free disk available")

        sess_id = unique_sessname(user_email)
        sess_props = JBoxSessionProps(sess_id, create=True, user_id=user_email)
        if sess_props.is_new:
            sess_props.save()
        snap_id = sess_props.get_snapshot_id()
        if snap_id is None:
            snap_id = JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT

        JBoxEBSVol.log_debug("will use snapshot id %s for %s", snap_id, user_email)

        _dev_path, mnt_path = CloudHost.create_new_volume(snap_id, disk_id, JBoxEBSVol.FS_LOC, tag=user_email)
        ebsvol = JBoxEBSVol(mnt_path, user_email=user_email)

        if snap_id == JBoxEBSVol.DISK_TEMPLATE_SNAPSHOT:
            JBoxEBSVol.log_debug("creating home folder on blank volume for %s", user_email)
            ebsvol.restore_user_home()
            ebsvol.restore()
        #else:
        #    snap_age_days = CloudHost.get_snapshot_age(snap_id).total_seconds()/(60*60*24)
        #    if snap_age_days > 7:
        #        ebsvol.restore_user_home()
        JBoxEBSVol.log_debug("setting up instance configuration on disk for %s", user_email)
        ebsvol.setup_instance_config()

        return ebsvol

def is_proposed_cluster_leader():
    if not CloudHost.ENABLED['cloudwatch']:
        return False

    cluster = CloudHost.INSTALL_ID
    leader = JBoxDynConfig.get_cluster_leader(cluster)
    return leader == CloudHost.instance_id()

    def push_backup(user_email, disk_path):
        sessname = unique_sessname(user_email)
        S3Disk.log_info("pushing %s.tar.gz from %s to %s", sessname, disk_path, JBoxVol.BACKUP_BUCKET)

        bkup_file = os.path.join('/tmp', sessname + ".tar.gz")
        bkup_tar = tarfile.open(bkup_file, 'w:gz')

        def set_perms(tinfo):
            tinfo.uid = 1000
            tinfo.gid = 1000
            tinfo.uname = 'ubuntu'
            tinfo.gname = 'ubuntu'
            return tinfo

        for f in os.listdir(disk_path):
            if f.startswith('.') and (f in ['.julia', '.juliabox']):
                continue
            full_path = os.path.join(disk_path, f)
            bkup_tar.add(full_path, os.path.join('juser', f), filter=set_perms)
        bkup_tar.close()
        os.chmod(bkup_file, 0666)

        # Upload to S3 if so configured. Delete from local if successful.
        bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
            datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
        if JBoxVol.BACKUP_BUCKET is not None:
            if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file,
                                         metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
                os.remove(bkup_file)
                S3Disk.log_info("Moved backup to S3 " + sessname)

    def calc_stat(user_email):
        VolMgr.STATS['num_users'] += 1
        sessname = unique_sessname(user_email)

        k = CloudHost.pull_file_from_s3(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
        if k is not None:
            VolMgr.STATS['loopback']['sizes'].append(k.size)

 def setup_instance_config(self, profiles=('julia', 'jboxjulia')):
     for profile in profiles:
         profile_path = '.ipython/profile_' + profile
         profile_path = os.path.join(self.disk_path, profile_path)
         if not os.path.exists(profile_path):
             continue
         nbconfig = os.path.join(profile_path, 'ipython_notebook_config.py')
         wsock_cfg = "c.NotebookApp.websocket_url = '" + JBoxVol.NOTEBOOK_WEBSOCK_PROTO + \
                     CloudHost.notebook_websocket_hostname() + "'\n"
         JBoxVol.replace_in_file(nbconfig, "c.NotebookApp.websocket_url", wsock_cfg)