Python api.BigML类代码示例

    def __init__(self, anomaly, api=None):

        self.resource_id = None
        self.sample_size = None
        self.input_fields = None
        self.mean_depth = None
        self.expected_mean_depth = None
        self.iforest = None
        self.top_anomalies = None
        self.id_fields = []
        if not (isinstance(anomaly, dict) and 'resource' in anomaly and
                anomaly['resource'] is not None):
            if api is None:
                api = BigML(storage=STORAGE)
            self.resource_id = get_anomaly_id(anomaly)
            if self.resource_id is None:
                raise Exception(api.error_message(anomaly,
                                                  resource_type='anomaly',
                                                  method='get'))
            query_string = ONLY_MODEL
            anomaly = retrieve_resource(api, self.resource_id,
                                        query_string=query_string)
        else:
            self.resource_id = get_anomaly_id(anomaly)
        if 'object' in anomaly and isinstance(anomaly['object'], dict):
            anomaly = anomaly['object']
            self.sample_size = anomaly.get('sample_size')
            self.input_fields = anomaly.get('input_fields')
            self.id_fields = anomaly.get('id_fields', [])
        if 'model' in anomaly and isinstance(anomaly['model'], dict):
            ModelFields.__init__(self, anomaly['model'].get('fields'))
            if ('top_anomalies' in anomaly['model'] and
                    isinstance(anomaly['model']['top_anomalies'], list)):
                self.mean_depth = anomaly['model'].get('mean_depth')
                status = get_status(anomaly)
                if 'code' in status and status['code'] == FINISHED:
                    self.expected_mean_depth = None
                    if self.mean_depth is None or self.sample_size is None:
                        raise Exception("The anomaly data is not complete. "
                                        "Score will"
                                        " not be available")
                    else:
                        default_depth = (
                            2 * (DEPTH_FACTOR + \
                            math.log(self.sample_size - 1) - \
                            (float(self.sample_size - 1) / self.sample_size)))
                        self.expected_mean_depth = min(self.mean_depth,
                                                       default_depth)
                    iforest = anomaly['model'].get('trees', [])
                    if iforest:
                        self.iforest = [
                            AnomalyTree(anomaly_tree['root'], self.fields)
                            for anomaly_tree in iforest]
                    self.top_anomalies = anomaly['model']['top_anomalies']
                else:
                    raise Exception("The anomaly isn't finished yet")
            else:
                raise Exception("Cannot create the Anomaly instance. Could not"
                                " find the 'top_anomalies' key in the"
                                " resource:\n\n%s" % anomaly['model'].keys())

class BigMLAPIMixIn(object):

    BIGML_AUTH_ERRMSG = (
        "{errtype:s} BigML credentials. Please supply "
        "BIGML_USERNAME and BIGML_API_KEY as either Scrapy "
        "settings or environment variables."
    )

    # XXX: This should get a method to read BigML configuration from settings

    def get_bigml_api(self, *args, **kwargs):
        try:
            self.bigml = BigML(*args, **kwargs)
        except AttributeError:
            raise NotConfigured(self.BIGML_AUTH_ERRMSG.format(errtype="Missing"))
        if not self.check_bigml_auth():
            raise NotConfigured(self.BIGML_AUTH_ERRMSG.format(errtype="Invalid"))

    def check_bigml_auth(self):
        return self.bigml.list_projects("limit=1")["code"] == 200

    def export_to_bigml(self, path, name, as_dataset=False):
        source = self.bigml.create_source(file, {"name": name})
        if not as_dataset:
            return source
        return self.bigml.create_dataset(source, {"name": name})

def get_resource_dict(resource, resource_type, api=None):
    """Extracting the resource JSON info as a dict from the first argument of
       the local object constructors, that can be:

        - the path to a file that contains the JSON
        - the ID of the resource
        - the resource dict itself

    """
    if api is None:
        api = BigML(storage=STORAGE)
    get_id = ID_GETTERS[resource_type]
    resource_id = None
    # the string can be a path to a JSON file
    if isinstance(resource, basestring):
        try:
            with open(resource) as resource_file:
                resource = json.load(resource_file)
                resource_id = get_id(resource)
                if resource_id is None:
                    raise ValueError("The JSON file does not seem"
                                     " to contain a valid BigML %s"
                                     " representation." % resource_type)
        except IOError:
            # if it is not a path, it can be a model id
            resource_id = get_id(resource)
            if resource_id is None:
                if resource.find("%s/" % resource_type) > -1:
                    raise Exception(
                        api.error_message(resource,
                                          resource_type=resource_type,
                                          method="get"))
                else:
                    raise IOError("Failed to open the expected JSON file"
                                  " at %s." % resource)
        except ValueError:
            raise ValueError("Failed to interpret %s."
                             " JSON file expected." % resource)

    # checks whether the information needed for local predictions is in
    # the first argument
    if isinstance(resource, dict) and \
            not check_model_fields(resource):
        # if the fields used by the model are not
        # available, use only ID to retrieve it again
        resource = get_id(resource)
        resource_id = resource

    if not (isinstance(resource, dict) and 'resource' in resource and
            resource['resource'] is not None):
        query_string = ONLY_MODEL
        resource = retrieve_resource(api, resource_id,
                                     query_string=query_string)
    else:
        resource_id = get_id(resource)

    return resource_id, resource

def get_model():
    
    bigml_model = memcache.Client().get(model_id)

    if bigml_model is None :
        bigml_api = BigML(BIGML_USERNAME, BIGML_API_KEY, dev_mode=dev_mode)
        bigml_model = bigml_api.get_model('model/%s' % model_id, query_string='only_model=true;limit=-1')
        memcache.Client().add(model_id, bigml_model, time=memcache_timeout)

    return bigml_model

    def __init__(self, model, api=None):

        if check_model_structure(model):
            self.resource_id = model["resource"]
        else:
            # If only the model id is provided, the short version of the model
            # resource is used to build a basic summary of the model
            if api is None:
                api = BigML()
            self.resource_id = get_model_id(model)
            if self.resource_id is None:
                raise Exception(api.error_message(model, resource_type="model", method="get"))
            query_string = ONLY_MODEL
            model = retrieve_resource(api, self.resource_id, query_string=query_string)
            # Stored copies of the model structure might lack some necessary
            # keys
            if not check_model_structure(model):
                model = api.get_model(self.resource_id, query_string=query_string)

        if "object" in model and isinstance(model["object"], dict):
            model = model["object"]

        if "model" in model and isinstance(model["model"], dict):
            status = get_status(model)
            if "code" in status and status["code"] == FINISHED:
                if "model_fields" in model["model"] or "fields" in model["model"]:
                    fields = model["model"].get("model_fields", model["model"].get("fields", []))
                    # pagination or exclusion might cause a field not to
                    # be in available fields dict
                    if not all(key in model["model"]["fields"] for key in fields.keys()):
                        raise Exception(
                            "Some fields are missing"
                            " to generate a local model."
                            " Please, provide a model with"
                            " the complete list of fields."
                        )
                    for field in fields:
                        field_info = model["model"]["fields"][field]
                        if "summary" in field_info:
                            fields[field]["summary"] = field_info["summary"]
                        fields[field]["name"] = field_info["name"]
                objective_field = model["objective_fields"]
                ModelFields.__init__(self, fields, objective_id=extract_objective(objective_field))
                self.description = model["description"]
                self.field_importance = model["model"].get("importance", None)
                if self.field_importance:
                    self.field_importance = [element for element in self.field_importance if element[0] in fields]
                self.locale = model.get("locale", DEFAULT_LOCALE)

            else:
                raise Exception("The model isn't finished yet")
        else:
            raise Exception(
                "Cannot create the BaseModel instance. Could not" " find the 'model' key in the resource:\n\n%s" % model
            )

    def __init__(self, cluster, api=None):

        if not (isinstance(cluster, dict) and 'resource' in cluster and
                cluster['resource'] is not None):
            if api is None:
                api = BigML(storage=STORAGE)
            self.resource_id = get_cluster_id(cluster)
            if self.resource_id is None:
                raise Exception(api.error_message(cluster,
                                                  resource_type='cluster',
                                                  method='get'))
            query_string = ONLY_MODEL
            cluster = retrieve_resource(api, self.resource_id,
                                        query_string=query_string)
        if 'object' in cluster and isinstance(cluster['object'], dict):
            cluster = cluster['object']

        if 'clusters' in cluster and isinstance(cluster['clusters'], dict):
            status = get_status(cluster)
            if 'code' in status and status['code'] == FINISHED:
                clusters = cluster['clusters']['clusters']
                self.centroids = [Centroid(centroid) for centroid in clusters]
                self.scales = {}
                self.scales.update(cluster['scales'])
                self.term_forms = {}
                self.tag_clouds = {}
                self.term_analysis = {}
                fields = cluster['clusters']['fields']
                for field_id, field in fields.items():
                    if field['optype'] == 'text':

                        self.term_forms[field_id] = {}
                        self.term_forms[field_id].update(field[
                            'summary']['term_forms'])
                        self.tag_clouds[field_id] = {}
                        self.tag_clouds[field_id].update(field[
                            'summary']['tag_cloud'])
                        self.term_analysis[field_id] = {}
                        self.term_analysis[field_id].update(
                            field['term_analysis'])
                ModelFields.__init__(self, fields)
                if not all([field_id in self.fields for
                            field_id in self.scales]):
                    raise Exception("Some fields are missing"
                                    " to generate a local cluster."
                                    " Please, provide a cluster with"
                                    " the complete list of fields.")
            else:
                raise Exception("The cluster isn't finished yet")
        else:
            raise Exception("Cannot create the Cluster instance. Could not"
                            " find the 'clusters' key in the resource:\n\n%s" %
                            cluster)

    def points_in_cluster(self, centroid_id):
        """Returns the list of data points that fall in one cluster.

        """

        cluster_datasets = self.datasets
        centroid_dataset = cluster_datasets.get(centroid_id)
        if self.api is None:
            self.api = BigML(storage=STORAGE)
        if centroid_dataset in [None, ""]:
            centroid_dataset = self.api.create_dataset( \
                self.resource_id, {"centroid": centroid_id})
            self.api.ok(centroid_dataset)
        else:
            centroid_dataset = self.api.check_resource( \
                "dataset/%s" % centroid_dataset)
        # download dataset to compute local predictions
        downloaded_data = self.api.download_dataset( \
            centroid_dataset["resource"])
        if PY3:
            text_reader = codecs.getreader("utf-8")
            downloaded_data = text_reader(downloaded_data)
        reader = csv.DictReader(downloaded_data)
        points = []
        for row in reader:
            points.append(row)
        return points

 def print_connection_info(self):
     self.USERNAME = os.environ.get('BIGML_USERNAME')
     self.API_KEY = os.environ.get('BIGML_API_KEY')
     try:
         self.debug = bool(os.environ.get('BIGMLER_DEBUG', 0))
         self.api_debug = bool(os.environ.get('BIGML_DEBUG', 0))
     except ValueError:
         pass
     if self.USERNAME is None or self.API_KEY is None:
         assert False, ("Tests use the BIGML_USERNAME and BIGML_API_KEY"
                        " environment variables to authenticate the"
                        " connection, but they seem to be unset. Please,"
                        "set them before testing.")
     else:
         assert True
     self.api = BigML(self.USERNAME, self.API_KEY, debug=self.api_debug)
     print self.api.connection_info()
     output_dir = "./last_run"
     dirs = []
     for _, subFolders, _ in os.walk("./"):
         for folder in subFolders:
             if folder.startswith("scenario"):
                 dirs.append(folder)
     dirs.reverse()
     for folder in dirs:
         bigmler_delete(folder, output_dir=output_dir)
     if os.path.exists(output_dir):
         shutil.rmtree(output_dir)

 def get_bigml_api(self, *args, **kwargs):
     try:
         self.bigml = BigML(*args, **kwargs)
     except AttributeError:
         raise NotConfigured(self.BIGML_AUTH_ERRMSG.format(errtype="Missing"))
     if not self.check_bigml_auth():
         raise NotConfigured(self.BIGML_AUTH_ERRMSG.format(errtype="Invalid"))

    def __init__(self, model, api=None):

        if not (isinstance(model, dict) and 'resource' in model and
                model['resource'] is not None):
            if api is None:
                api = BigML(storage=STORAGE)
            self.resource_id = get_model_id(model)
            if self.resource_id is None:
                raise Exception(api.error_message(model,
                                                  resource_type='model',
                                                  method='get'))
            query_string = ONLY_MODEL
            model = retrieve_model(api, self.resource_id,
                                   query_string=query_string)
        BaseModel.__init__(self, model, api=api)
        if ('object' in model and isinstance(model['object'], dict)):
            model = model['object']

        if ('model' in model and isinstance(model['model'], dict)):
            status = get_status(model)
            if ('code' in status and status['code'] == FINISHED):
                distribution = model['model']['distribution']['training']
                self.ids_map = {}
                self.tree = Tree(
                    model['model']['root'],
                    self.fields,
                    objective_field=self.objective_field,
                    root_distribution=distribution,
                    parent_id=None,
                    ids_map=self.ids_map)
                self.terms = {}
            else:
                raise Exception("The model isn't finished yet")
        else:
            raise Exception("Cannot create the Model instance. Could not"
                            " find the 'model' key in the resource:\n\n%s" %
                            model)
        if self.tree.regression:
            try:
                import numpy
                import scipy
                self.regression_ready = True
            except ImportError:
                self.regression_ready = False

    def reset_api(self):
        """Reset the api connection values

        """
        if self.api is not None and self.api.dev_mode:
            world.project_id = None
        if self.api is None or self.api.dev_mode:
            self.api = BigML(self.USERNAME, self.API_KEY)
            self.api_dev_mode = BigML(self.USERNAME, self.API_KEY,
                                      dev_mode=True)

 def print_connection_info(self):
     self.USERNAME = os.environ.get('BIGML_USERNAME')
     self.API_KEY = os.environ.get('BIGML_API_KEY')
     if self.USERNAME is None or self.API_KEY is None:
         assert False, ("Tests use the BIGML_USERNAME and BIGML_API_KEY"
                        " environment variables to authenticate the"
                        " connection, but they seem to be unset. Please,"
                        "set them before testing.")
     self.api = BigML(self.USERNAME, self.API_KEY, debug=self.debug)
     print self.api.connection_info()

def main(args=sys.argv[1:]):
    """Parses command-line parameters and calls the actual main function.

    """
    # Process arguments
    parser = argparse.ArgumentParser(
        description="JSON PML to DOT",
        epilog="BigML, Inc")

    # Model
    parser.add_argument('--model',
                        type=str,
                        required=True,
                        action='store',
                        dest='model',
                        default=None,
                        help="Model identifier")

    # Output file
    parser.add_argument('--output',
                        type=str,
                        action='store',
                        dest='output',
                        default=None,
                        help="Output file")

    # Parse args
    args = parser.parse_args(args)

    # Instantiate BigML API
    api = BigML()

    model = api.get_model(args.model)
    api.ok(model)

    if args.output:
        output = open(args.output, 'w')
        write_tree(model, output)
        output.close()
    else:
        write_tree(model)

 def authenticate(self,bigml_user,bigml_key):
     """
     initialize the BigML API, do a short test to check authentication
     """
     
     self.api = BigML(username=bigml_user,api_key=bigml_key)
     
     result = self.api.list_sources()
     if result['code'] == 200:
         self.authenticated = True
     else:
         self.authenticated = False

 def print_connection_info(self):
     self.USERNAME = os.environ.get('BIGML_USERNAME')
     self.API_KEY = os.environ.get('BIGML_API_KEY')
     if self.USERNAME is None or self.API_KEY is None:
         assert False, ("Tests use the BIGML_USERNAME and BIGML_API_KEY"
                        " environment variables to authenticate the"
                        " connection, but they seem to be unset. Please,"
                        "set them before testing.")
     else:
         assert True
     self.api = BigML(self.USERNAME, self.API_KEY)
     print self.api.connection_info()
     output_dir = "./last_run"
     for _, subFolders, _ in os.walk("./"):
         for folder in subFolders:
             if folder.startswith("scenario"):
                 bigmler_delete(folder, output_dir=output_dir)
     if os.path.exists(output_dir):
         shutil.rmtree(output_dir)