Python qubole.Qubole類代碼示例

 def terminate(cls, cluster_id):
     """
     Terminate the cluster with id `cluster_id`.
     """
     conn = Qubole.agent()
     data = {"state": "terminate"}
     return conn.put(cls.element_path(cluster_id) + "/state", data)

 def start(cls, cluster_id):
     """
     Start the cluster with id `cluster_id`.
     """
     conn = Qubole.agent()
     data = {"state": "start"}
     return conn.put(cls.element_path(cluster_id) + "/state", data)

 def update(cls, cluster_id, cluster_info):
     """
     Update the cluster with id `cluster_id` using information provided in
     `cluster_info`.
     """
     conn = Qubole.agent()
     return conn.put(cls.element_path(cluster_id), data=cluster_info)

    def list(cls, label=None, state=None):
        """
        List existing clusters present in your account.

        Kwargs:
            `label`: list cluster with this label

            `state`: list only those clusters which are in this state

        Returns:
            List of clusters satisfying the given criteria

        Raises:
            Exception if both label and state options are provided
        """
        conn = Qubole.agent()
        if label is None and state is None:
            return conn.get(cls.rest_entity_path)
        elif label is not None and state is None:
            cluster_list = conn.get(cls.rest_entity_path)
            result = []
            for cluster in cluster_list:
                if label in cluster['cluster']['label']:
                    result.append(cluster)
            return result
        elif label is None and state is not None:
            cluster_list = conn.get(cls.rest_entity_path)
            result = []
            for cluster in cluster_list:
                if state.lower() == cluster['cluster']['state'].lower():
                    result.append(cluster)
            return result
        else:
            raise Exception("Can filter either by label or" +
                            " by state but not both")

    def cancel_id(cls, id):
        """
        Cancels command denoted by this id

        Args:
            `id` - command id
        """
        conn=Qubole.agent()
        data={"status":"kill"}
        conn.put(cls.element_path(id), data)

    def get_log_id(cls, id):
        """
        Fetches log for the command represented by this id

        Args:
            `id`: command id
        """
        conn = Qubole.agent()
        r = conn.get_raw(cls.element_path(id) + "/logs")
        return r.text

    def get_log(self):
        """
        Fetches log for the command represented by this object

        Returns:
            The log as a string
        """
        log_path = self.meta_data['logs_resource']
        conn=Qubole.agent()
        r=conn.get_raw(log_path)
        return r.text

    def save_results_locally(self, local_file_location, boto_client=None):
        """
        Saves the results to the passed in file_location

        Args:
            local_file_location: the place locally where you want to store the file
            boto_client: the boto client to use if we're fetching results from s3.

        Returns:
            dictionary of:
             'file_location' : <file path + file name>
             'size' : <file size>
             'success' : True or False
             'error' : error message if there was any
        """
        result_path = self.meta_data['results_resource']
        conn=Qubole.agent()
        r = conn.get(result_path)
        if r.get('inline'):
            error = ''
            success = True
            try:
                f = open(local_file_location, 'w')
                f.write(r['results'])
                f.close()
            except Exception as e:
                error = e
                success = False
            result = {
                'file_location': local_file_location,
                'size': os.path.getsize(local_file_location),
                'success': success,
                'error': error
            }
            return result
        else:
            # TODO:finish
            s3_locations = r.get('result_location')
            if not boto_client:
                log.error("Unable to download results, no boto client provided.\
                           Please fetch from S3: %s" % s3_locations)    
                return {'success':False, 'error': 'Not boto client'}
            print s3_locations
            result = {
                'file_location': local_file_location,
                'size': 0, #os.path.getsize(local_file_location),
                'success': False,
                'error': 'Not implemented'
            }
            return result

    def reassign_label(cls, destination_cluster, label):
        """
        Reassign a label from one cluster to another.

        Args:
            `destination_cluster`: id of the cluster to move the label to

            `label`: label to be moved from the source cluster
        """
        conn = Qubole.agent()
        data = {
                    "destination_cluster": destination_cluster,
                    "label": label
                }
        return conn.put(cls.rest_entity_path + "/reassign-label", data)

    def list(page = None, per_page = None):
        conn = Qubole.agent()
        url_path = Action.rest_entity_path
        params = {}
        if page is not None:
            params['page'] = page  
        if per_page is not None:
            params['per_page'] = per_page  

        #Todo Page numbers are thrown away right now
        actjson = conn.get(url_path, params)
        actlist = []
        for a in actjson["actions"]:
            actlist.append(Action(a))
        return actlist

    def get_results(self):
        """
        Fetches the result for the command represented by this object

        Returns:
            The result as a string
        """
        result_path = self.meta_data['results_resource']
        conn=Qubole.agent()
        r = conn.get(result_path)
        if r.get('inline'):
            return r['results'] 
        else:
            # TODO - this will be implemented in future
            log.error("Unable to download results, please fetch from S3")

    def create(cls, **kwargs):
        """
        Create a command object by issuing a POST request to the /command endpoint
        Note - this does not wait for the command to complete

        Args:
            `\**kwargs` - keyword arguments specific to command type

        Returns:
            Command object
        """

        conn=Qubole.agent()
        if kwargs.get('command_type') is None:
            kwargs['command_type'] = cls.__name__
        return cls(conn.post(cls.rest_entity_path, data=kwargs))

    def list(cls, state=None):
        """
        List existing clusters present in your account.

        Kwargs:
            `state`: list only those clusters which are in this state

        Returns:
            List of clusters satisfying the given criteria
        """
        conn = Qubole.agent()
        if state is None:
            return conn.get(cls.rest_entity_path)
        elif state is not None:
            cluster_list = conn.get(cls.rest_entity_path)
            result = []
            for cluster in cluster_list:
                if state.lower() == cluster['cluster']['state'].lower():
                    result.append(cluster)
            return result

    def get_results(self, fp=sys.stdout, inline=True):
        """
        Fetches the result for the command represented by this object

        @param fp: a file object to write the results to directly
        """
        result_path = self.meta_data['results_resource']
        
        conn=Qubole.agent()
        
        r = conn.get(result_path , {'inline': inline})
        if r.get('inline'):
            fp.write(r['results'].encode('utf8'))
        else:    
            acc = Account.find()
            boto_conn = boto.connect_s3(aws_access_key_id=acc.storage_access_key,
                                        aws_secret_access_key=acc.storage_secret_key)

            log.info("Starting download from result locations: [%s]" % ",".join(r['result_location']))

            for s3_path in r['result_location']:
                _download_to_local(boto_conn, s3_path, fp)

 def create(cls, **kwargs):
     conn=Qubole.agent()
     return cls(conn.post(cls.rest_entity_path, data=kwargs))