Python bson.SON类代码示例

 def __init__(self, object, format='TEXT', _hsid=0, _hflags=0):
     SON.__init__(self)
     self.hdr = Hdr(144, _hsid, 0, _hflags)
     self.type = 144
     self['t'] = 144
     self['object'] = object
     self['format'] = format

def specific_config_gen(IC,args):
    IC.base_dir = args['base_dir']
    IC.annotate_dir = args['annotate_dir']
    IC.groundtruth_dir = args['groundtruth_dir']
    IC.correspondence = tb.tabarray(SVfile = args['frame_correspondence'])
    IC.size = args['size']
    IC.prefix = prefix = args.get('image_extension','.jpg')
    IC.current_frame_path = None
    csvs = [x for x in os.listdir(IC.annotate_dir) if x.endswith('.csv')]
    csvs.sort()
    Xs = [tb.tabarray(SVfile = os.path.join(IC.annotate_dir,csv)) for csv in csvs]
    cns = [csv.split('.')[0] for csv in csvs]
    cns = [[cn]*len(X) for (cn,X) in zip(cns,Xs)]
    Xs = [X.addcols(cn,names=['clip_num']) for (cn,X) in zip(cns,Xs)]

    csvs = [x for x in os.listdir(IC.groundtruth_dir) if x.endswith('.csv')]
    csvs.sort()
    Gs = []
    fields = ['clip_num','Frame'] + xfields + yfields
    for ind,csv in enumerate(csvs):
        try:
            g = tb.tabarray(SVfile = os.path.join(IC.groundtruth_dir,csv))
        except:
            x = Xs[ind].addcols([-1]*len(Xs[ind]),names=['Correctness'])
        else:
            g = g.addcols([csv.split('.')[0]]*len(g),names = ['clip_num'])
            g = g[fields + ['Confidence']]
            g.renamecol('Confidence','Correctness')
            x = Xs[ind].join(g,keycols=fields)
        Gs.append(x)
    X = tb.tab_rowstack(Gs)
    X.sort(order=['clip_num','Frame'])
    
    Y = IC.correspondence
    F = tb.fast.recarrayisin(Y[['clip_num','Frame']],X[['clip_num','Frame']])
    Y = Y[F]
    X = X.join(Y,keycols=['clip_num','Frame'])

    params = []
    for t in X:
        print(t)  
        cn = t['clip_num']
        fr = t['Frame']
        box = get_darpa_box(t)
        bb = box.pop('box')
        xc,yc = bb.center
        center = correct_center((xc,yc),IC.size,(1920,1080))
        bb_new = bbox.BoundingBox(center = center,width = IC.size[0], height = IC.size[1])
        p = SON([('size',IC.size),
                     ('bounding_box',SON([('xfields',list(bb_new.xs)),('yfields',list(bb_new.ys))])),
                     ('original_bounding_box',SON([('xfields',list(bb.xs)),('yfields',list(bb.ys))])),
                     ('clip_num',cn),
                     ('Frame',int(t['Original'])),
                     ('base_dir',IC.base_dir),
                     ('correctness',int(t['Correctness']))])
        p.update(box)
        p['GuessObjectType'] = p['ObjectType']
        p['ObjectType'] = p['ObjectType'] if t['Correctness'] == 1 else ''
        params.append(SON([('image',p)]))
    return params

    def to_mongo(self, use_db_field=True, fields=None):
        """
        Return as SON data ready for use with MongoDB.
        """
        if not fields:
            fields = []

        data = SON()
        data['_id'] = None
        data['_cls'] = self._class_name

        # only root fields ['test1.a', 'test2'] => ['test1', 'test2']
        root_fields = {f.split('.')[0] for f in fields}

        for field_name in self:
            if root_fields and field_name not in root_fields:
                continue

            value = self._data.get(field_name, None)
            field = self._fields.get(field_name)

            if field is None and self._dynamic:
                field = self._dynamic_fields.get(field_name)

            if value is not None:
                f_inputs = field.to_mongo.__code__.co_varnames
                ex_vars = {}
                if fields and 'fields' in f_inputs:
                    key = '%s.' % field_name
                    embedded_fields = [
                        i.replace(key, '') for i in fields
                        if i.startswith(key)]

                    ex_vars['fields'] = embedded_fields

                if 'use_db_field' in f_inputs:
                    ex_vars['use_db_field'] = use_db_field

                value = field.to_mongo(value, **ex_vars)

            # Handle self generating fields
            if value is None and field._auto_gen:
                value = field.generate()
                self._data[field_name] = value

            if (value is not None) or (field.null):
                if use_db_field:
                    data[field.db_field] = value
                else:
                    data[field.name] = value

        # Only add _cls if allow_inheritance is True
        if not self._meta.get('allow_inheritance'):
            data.pop('_cls')

        return data

 def __init__(self, stderr, stack=None, etype=None, value=None, syntax=None, _hsid=0, _hflags=0):
     SON.__init__(self)
     self.hdr = Hdr(10, _hsid, 0, _hflags)
     self.type = 10
     self['t'] = 10
     self['stderr'] = stderr
     self['stack'] = stack
     self['etype'] = etype
     self['value'] = value
     self['syntax'] = syntax

def near(col, params):
    if params is None or params.count(':') != 2:
        raise TypeError("$near requires three arguments.  Use like /%s/$near=-73.10:42.18:0.5/ to return all records within a 0.5-mile radius of %s" % (col, col))
    params = params.split(":")
    params[0] = float(params[0])
    params[1] = float(params[1])
    params[2] = float(params[2]) / 69.0
    near_dict = {"$near": [params[0], params[1]]}
    dist_dict = {"$maxDistance": params[2]}
    q = SON(near_dict)
    q.update(dist_dict)
    return q

    def command(self, command, value=1, read_preference=None,
        callback=None, check=True, allowable_errors=[], connection=None, **kwargs):
        """Issue a MongoDB command.

        Send command `command` to the database and return the
        response. If `command` is an instance of :class:`basestring`
        then the command {`command`: `value`} will be sent. Otherwise,
        `command` must be an instance of :class:`dict` and will be
        sent as is.

        Any additional keyword arguments will be added to the final
        command document before it is sent.

        For example, a command like ``{buildinfo: 1}`` can be sent
        using:

        >>> db.command("buildinfo")

        For a command where the value matters, like ``{collstats:
        collection_name}`` we can do:

        >>> db.command("collstats", collection_name)

        For commands that take additional arguments we can use
        kwargs. So ``{filemd5: object_id, root: file_root}`` becomes:

        >>> db.command("filemd5", object_id, root=file_root)

        :Parameters:
          - `command`: document representing the command to be issued,
            or the name of the command (for simple commands only).

            .. note:: the order of keys in the `command` document is
               significant (the "verb" must come first), so commands
               which require multiple keys (e.g. `findandmodify`)
               should use an instance of :class:`~bson.son.SON` or
               a string and kwargs instead of a Python `dict`.

          - `value` (optional): value to use for the command verb when
            `command` is passed as a string
          - `**kwargs` (optional): additional keyword arguments will
            be added to the command document before it is sent

        """
        if isinstance(command, basestring):
            command = SON([(command, value)])
        command.update(kwargs)

        if read_preference is None:
            read_preference = self._read_preference

        Client(self, '$cmd').find_one(command, is_command=True,
            connection=connection, read_preference=read_preference, callback=callback)

def generate_splits(task_config,hash,colname):
    base_query = SON([('__hash__',hash)])
    ntrain = task_config['ntrain']
    ntest = task_config['ntest']
    ntrain_pos = task_config.get('ntrain_pos')
    ntest_pos = task_config.get('ntest_pos')
    N = task_config.get('N',10)
    query = task_config['query']  
    base_query.update(reach_in('config',task_config.get('universe',SON([]))))    
    cquery = reach_in('config',query)
    
    print('q',cquery)
    print('u',base_query)
 
    return traintest.generate_split2(DB_NAME,colname,cquery,N,ntrain,ntest,ntrain_pos=ntrain_pos,ntest_pos = ntest_pos,universe=base_query,use_negate = True)

    def to_mongo(self):
        from bson import SON
        from mongoengine import Document
        from mongoengine.base.common import ALLOW_INHERITANCE
        """Return as SON data ready for use with MongoDB.
        """
        data = SON()
        data["_id"] = None
        data['_cls'] = self._class_name

        for field_name in self._fields_ordered:
            value = self._data.get(field_name, None)
            field = self._fields.get(field_name)
            if field is None and self._dynamic:
                field = self._dynamic_fields.get(field_name)

            if value is not None:
                value = field.to_mongo(value)

            # Handle self generating fields
            if value is None and field._auto_gen:
                value = field.generate()
                self._data[field_name] = value

            if value is not None:
                data[field.db_field] = value

        # If "_id" has not been set, then try and set it
        if isinstance(self, Document):
            if data["_id"] is None:
                data["_id"] = self._data.get("id", None)

        if data['_id'] is None:
            data.pop('_id')

        # Only add _cls if allow_inheritance is True
        if (not hasattr(self, '_meta') or
           not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)):
            data.pop('_cls')

        return data
        
        

def put_in_split_result(res,image_config_gen,m,task,ext_hash,split_id,splitres_fs):
    out_record = SON([('model',m['config']['model']),
                      ('images',son_escape(image_config_gen['images'])),
                      ('task',son_escape(task)),
                      ('split_id',split_id),
                 ])   
                 
    split_result = SON([])
    for stat in STATS:
        if stat in res and res[stat] != None:
            split_result[stat] = res[stat] 

    filename = get_filename(out_record)
    out_record['filename'] = filename
    out_record['__hash__'] = ext_hash
    out_record.update(split_result)
    print('dump out split result...')
    out_data = cPickle.dumps(SON([('split_result',res)]))
    splitres_fs.put(out_data,**out_record)          

def put_in_performance(split_results,image_config_gen,m,model_hash,image_hash,perf_coll,task,ext_hash):
    
    model_results = SON([])
    for stat in STATS:
        if stat in split_results[0] and split_results[0][stat] != None:
            model_results[stat] = sp.array([split_result[stat] for split_result in split_results]).mean()           

    out_record = SON([('model',m['config']['model']),
                      ('model_hash',model_hash), 
                      ('model_filename',m['filename']), 
                      ('images',son_escape(image_config_gen['images'])),
                      ('image_hash',image_hash),
                      ('task',son_escape(task)),
                      ('__hash__',ext_hash)
                 ])
                 
    out_record.update(model_results)

    perf_coll.insert(out_record)

def MRsimple(collection, FunMap, FunReduce=None, query={}, out={"replace": 'mr_tmp'}, finalize=None,
             scope={}, sort=None, jsMode=False, verbose=1):
    """ simplified generic Map Reduce
        see: http://docs.mongodb.org/manual/reference/method/db.collection.mapReduce/
        returns (MR response object, results collection or results list if out={"inline":1})
        Reduce function defaults to one that increments value count
        optimize by sorting on emit fields
        see: http://edgystuff.tumblr.com/post/7624019777/optimizing-map-reduce-with-mongodb
        docs.mongodb.org/manual/reference/method/db.collection.mapReduce/#db.collection.mapReduce
        sort      i.e: sort= { "_id":1 }
        jsMode    should be False if we expect more than 500K dictinct ids
    """
    if len(out.viewkeys()) > 1:
        command = MRCommand_(out)
        out = SON([(command, out[command]), ('db', out.get('db')),
                   ('nonAtomic', out.get('nonAtomic', False))])
        #nonAtomic not allowed on replace
    FunMap = Code(FunMap, {})
    if FunReduce is None:
        FunReduce = u"""function (key, values) {var total = 0; for (var i = 0;
                        i < values.length; i++) { total += values[i]; } return total;}
                     """
    FunReduce = Code(FunReduce, {})
    if verbose > 2:
        print "Start MRsimple collection = %s"\
              "query = %s\nMap=\n%s\nReduce=\n%s\nFinalize=%s\nscope=%s sort=%s" \
              % tuple(map(str, (out, query, FunMap, FunReduce, finalize, scope, sort)))
    if sort:
        sort = SON(sort)
    r = collection.map_reduce(FunMap, FunReduce, out=out, query=query,
                              finalize=finalize, scope=scope, sort=sort, full_response=True)
    if verbose > 1:
        print  "End MRsimple collection=%s, query=%s\nresulsts=\n %s"\
                % (collection.name, str(query), str(r))
    if 'db' in out.viewkeys():
        #@note:  can be dict or SON, either way it has property viewkeys
        results = collection.database.connection[r['result']['db']][r['result']['collection']]
    else:
        results = r['results'] if  out.keys()[0] == 'inline' else collection.database[r['result']]
        #@note:  results is a list if inline else a collection
    return r, results

    def get_last_error(self, db, **options):
        command = SON([("getlasterror", 1)])
        db = "%s.$cmd" % db.split('.', 1)[0]
        command.update(options)

        query = Query(collection=db, query=command)
        reply = yield self.send_QUERY(query)

        assert len(reply.documents) == 1

        document = reply.documents[0].decode()
        err = document.get("err", None)
        code = document.get("code", None)

        if err is not None:
            if code == 11000:
                raise DuplicateKeyError(err, code=code)
            else:
                raise OperationFailure(err, code=code)

        defer.returnValue(document)

def to_dict_document(self, serializer=None):
    data = SON()
    data["_id"] = None
    # data['_cls'] = self._class_name

    for field_name in self:
        value = self._data.get(field_name, None)
        field = self._fields.get(field_name)
        if field is None and self._dynamic:
            field = self._dynamic_fields.get(field_name)
        if value and isinstance(field, (ResourceField, ResourceIntField, DynamicResourceField)):
            pass
            # value = value.to_dict(fields=field.fields)
        if value is not None:
            value = field.to_dict(value, serializer=serializer)

        # Handle self generating fields
        if value is None and field._auto_gen:
            value = field.generate()
            self._data[field_name] = value

        if value is not None:
            data[field.db_field] = value

    # If "_id" has not been set, then try and set it
    Document = _import_class("Document")
    if isinstance(self, Document):
        if data["_id"] is None:
            data["_id"] = self._data.get("id", None)

    if data['_id'] is None:
        data.pop('_id')

    # Only add _cls if allow_inheritance is True
    # if (not hasattr(self, '_meta') or
    # not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)):
    #     data.pop('_cls')

    return data

def write_outcerts(func, configs, incertdicts, outcertpaths, db):
    if incertdicts:
        old_param_names = dict_union([op["param_names"] for op in incertdicts])
    else:
        old_param_names = SON([])

    new_param_names = uniqify(ListUnion([x.keys() for x in configs]))
    for (outcertpath, outroot) in zip(outcertpaths, func.outroots):
        param_names = old_param_names.copy()
        param_names[outroot] = new_param_names
        remove_incorrect(db, outroot, func.outconfig_string, func.outrun_hash)
        createCertificateDict(
            outcertpath,
            {
                "run_hash": func.outrun_hash,
                "db": func.dbname,
                "out_args": func.out_args,
                "root": outroot,
                "config_hash": func.outconfig_string,
                "param_names": param_names,
            },
        )

def get_op_gen(op, oplist):

    if op.get("outcertpaths") is None:
        func = op["func"]
        params = op.get("params")
        inroots = func.inroots
        outroots = func.outroots
        if func.action_name == "inject":
            args = op["params"]
            out_args = SON([(outroot, params) for outroot in outroots])

        else:
            params = op.get("params", SON([]))

            parents = []
            for ir in inroots:
                try:
                    parent = [op0 for op0 in oplist if ir in op0["func"].outroots][0]
                except IndexError:
                    raise IndexError, "No parent found for at least one collection in " + repr(op0["func"].outroots)
                else:
                    parents.append(parent)

            for parent in parents:
                get_op_gen(parent, oplist)

            in_args = [parent["out_args"] for parent in parents]
            op["incertpaths"] = [
                get_cert_path(func.dbname, inroot, get_config_string(in_arg))
                for (inroot, in_arg) in zip(inroots, in_args)
            ]
            out_args = dict_union(in_args)
            out_args.update(params)

        op["out_args"] = out_args
        op["outcertpaths"] = [
            get_cert_path(func.dbname, outroot, get_config_string(out_args)) for outroot in func.outroots
        ]