Python asizeof.asizeof函数代码示例

    def _get_lines(self, file1, file2, lines, file1_dest, file2_dest):
        """
        Given two files to open and a sorted list of lines to get, open the files,
        retrieves the desired lines, and dumps them to specified file locations.
        """
        lines = deque(lines)
        buf1, buf2 = [], []
        line_counter, target_line = 0, lines.popleft()

        for f1_line, f2_line in zip(open(file1, 'r'), open(file2, 'r')):
            if target_line == line_counter:
                buf1.append(f1_line.strip())
                buf2.append(f2_line.strip())

                if asizeof.asizeof(buf1) + asizeof.asizeof(buf2) > \
                    self.mem_limit:
                    self._dump_bufs_to( [file1_dest, file2_dest],
                                        [buf1, buf2])

                if len(lines) != 0:
                    target_line = lines.popleft()
                else:
                    break
            line_counter += 1

        self._dump_bufs_to( [file1_dest, file2_dest],
                            [buf1, buf2])

 def get_memory_usage(self):
     """
     Returns the sizes (in bytes) of the four main models that the classifier keeps in memory.
     """
     with self.lock:
         return asizeof.asizeof(self.hc), asizeof.asizeof(self.htc), asizeof.asizeof(self.tc), \
             asizeof.asizeof(self.thc)

def test_slots_being_used():
    """
    The class is really using __slots__.
    """
    non_slot_instance = C1(x=1, y="test")
    slot_instance = C1Slots(x=1, y="test")

    assert "__dict__" not in dir(slot_instance)
    assert "__slots__" in dir(slot_instance)

    assert "__dict__" in dir(non_slot_instance)
    assert "__slots__" not in dir(non_slot_instance)

    assert set(["x", "y"]) == set(slot_instance.__slots__)

    if has_pympler:
        assert asizeof(slot_instance) < asizeof(non_slot_instance)

    non_slot_instance.t = "test"
    with pytest.raises(AttributeError):
        slot_instance.t = "test"

    assert 1 == non_slot_instance.method()
    assert 1 == slot_instance.method()

    assert attr.fields(C1Slots) == attr.fields(C1)
    assert attr.asdict(slot_instance) == attr.asdict(non_slot_instance)

    def cleanse(self, src_lang_file, tar_lang_file):
        """
        Cleans the file provided by lowercasing all words and ensuring each line in
        the text file is within min_len and max_len. Operates on two streams
        simultaneously in order to keep line to line correspondence
        """
        self._validate_file(src_lang_file), self._validate_file(tar_lang_file)
        src_dest_file = self.destdir + utilities.strip_filename_from_path(src_lang_file) + ".cleansed"
        tar_dest_file = self.destdir + utilities.strip_filename_from_path(tar_lang_file) + ".cleansed"

        if utilities.files_exist([src_dest_file, tar_dest_file]):
            return
        else:
            utilities.wipe_files([src_dest_file, tar_dest_file])
        self._print("""Cleaning data.  Ensuring uniformity of data...""")

        src_buf, tar_buf = [], []
        for src_line, tar_line in zip(open(src_lang_file), open(tar_lang_file)):
            src_line = src_line.lower().split()
            tar_line = tar_line.lower().split()

            if len(src_line) > self.min_len and len(src_line) < self.max_len and \
                len(tar_line) > self.min_len and len(tar_line) < self.max_len:
                src_buf.append(' '.join(src_line))
                tar_buf.append(' '.join(tar_line))

            if asizeof.asizeof(src_buf) + asizeof.asizeof(tar_buf) > self.mem_limit:
                self._dump_bufs_to( [src_dest_file, tar_dest_file],
                                    [src_buf, tar_buf])

        self._dump_bufs_to([src_dest_file, tar_dest_file], [src_buf, tar_buf])
        self._print("Done\n")

def test_copy_features_does_not_copy_entityset(es):
    agg = Sum(es['log']['value'], es['sessions'])
    agg_where = Sum(es['log']['value'], es['sessions'],
                    where=IdentityFeature(es['log']['value']) == 2)
    agg_use_previous = Sum(es['log']['value'], es['sessions'],
                           use_previous='4 days')
    agg_use_previous_where = Sum(es['log']['value'], es['sessions'],
                                 where=IdentityFeature(es['log']['value']) == 2,
                                 use_previous='4 days')
    features = [agg, agg_where, agg_use_previous, agg_use_previous_where]
    in_memory_size = asizeof(locals())
    copied = [f.copy() for f in features]
    new_in_memory_size = asizeof(locals())
    assert new_in_memory_size < 2 * in_memory_size

    for f, c in zip(features, copied):
        assert f.entityset
        assert c.entityset
        assert id(f.entityset) == id(c.entityset)
        if f.where:
            assert c.where
            assert id(f.where.entityset) == id(c.where.entityset)
        for bf, bf_c in zip(f.base_features, c.base_features):
            assert id(bf.entityset) == id(bf_c.entityset)
            if bf.where:
                assert bf_c.where
                assert id(bf.where.entityset) == id(bf_c.where.entityset)

 def process_response(self, request, response):
         req = request.META['PATH_INFO']
         if req.find('static') == -1 and req.find('media') == -1:
                 print req
                 self.end_objects = muppy.get_objects()
                 sum_start = summary.summarize(self.start_objects)
                 sum_end = summary.summarize(self.end_objects)
                 diff = summary.get_diff(sum_start, sum_end)
                 summary.print_(diff)
                 #print '~~~~~~~~~'
                 #cb = refbrowser.ConsoleBrowser(response, maxdepth=2, \
                         #str_func=output_function)
                 #cb.print_tree()
                 print '~~~~~~~~~'
                 a = asizeof(response)
                 print 'Total size of response object in kB: %s' % \
                     str(a / 1024.0)
                 print '~~~~~~~~~'
                 a = asizeof(self.end_objects)
                 print 'Total size of end_objects in MB: %s' % \
                     str(a / 1048576.0)
                 b = asizeof(self.start_objects)
                 print 'Total size of start_objects in MB: %s' % \
                     str(b / 1048576.0)
                 print '~~~~~~~~~'
         return response

 def push(self, msg):
     serialized_msg = pickle.dumps(msg)
     from pympler.asizeof import asizeof
     print('unpickled: {}, pickled: {}'.format(
         asizeof(msg),
         asizeof(serialized_msg)
     ))
     self.output(serialized_msg)

    def test_globals(self):
        '''Test globals examples'''
        self._printf('%sasizeof(%s, limit=%s, code=%s) ... %s', os.linesep, 'globals()', 'MAX', False, '-glob[als]')
        asizeof.asizeof(globals(), limit=self.MAX, code=False, stats=1)
        self._print_functions(globals(), 'globals()', opt='-glob[als]')

        self._printf('%sasizesof(%s, limit=%s, code=%s) ... %s', os.linesep, 'globals(), locals()', 'MAX', False, '-glob[als]')
        asizeof.asizesof(globals(), locals(), limit=self.MAX, code=False, stats=1)
        asizeof.asized(globals(), align=0, detail=self.MAX, limit=self.MAX, code=False, stats=1)

    def test_methods(self):
        '''Test sizing methods and functions
        '''
        def foo():
            pass

        s1 = asizeof.asizeof(self.test_methods, code=True)
        s2 = asizeof.asizeof(TypesTest.test_methods, code=True)
        s3 = asizeof.asizeof(foo, code=True)

 def test_adict(self):
     '''Test asizeof.adict()
     '''
     pdict = PseudoDict()
     size1 = asizeof.asizeof(pdict)
     asizeof.adict(PseudoDict)
     size2 = asizeof.asizeof(pdict)
     # TODO: come up with useful assertions
     self.assertEqual(size1, size2)

def run(dicp="~/dev/kaggle/fb5/pdic.map", datap="~/dev/kaggle/fb5/train.tab", lr=1., numbats=100, epochs=10):
    dic, revdic = loaddict(expanduser(dicp))
    print len(dic)
    traindata, golddata = loaddata(expanduser(datap), top=10000)
    print asizeof(traindata), golddata.dtype
    m = SpatialEmb(dim=len(dic))

    m.train([traindata], golddata).adagrad(lr=lr).cross_entropy()\
        .split_validate(splits=100, random=True).cross_entropy().accuracy()\
        .train(numbats, epochs)

 def test_asizer(self):
     '''Test Asizer properties.
     '''
     sizer = asizeof.Asizer()
     obj = 'unladen swallow'
     mutable = [obj]
     sizer.asizeof(obj)
     self.assertEqual(sizer.total, asizeof.asizeof(obj))
     sizer.asizeof(mutable, mutable)
     self.assertEqual(sizer.duplicate, 1)
     self.assertEqual(sizer.total, asizeof.asizeof(obj, mutable))

 def add_results_data(self, results):
     if SIZE_CONTROL:
         if not self.MEM_LIMIT:
             mem_size = asizeof(self.current_task.results)
             add_size = asizeof(results)
             if (mem_size + add_size) < 15000000:
                 self._add_results(results)
             else:
                 self.MEM_LIMIT = True
     else:
         self._add_results(results)

 def getSizeOfMgrs(self):
     """ get size of object """
     appGlobal = config['pylons.app_globals']
     
     result = {}
     result['threadmgr'] = asizeof(appGlobal.threadMgr)
     result['packagemgr'] = asizeof(appGlobal.packageMgr)
     result['montior'] = asizeof(appGlobal.agentMonitor)
     result['all'] = asizeof(appGlobal)
     
     return doneResult(request, response, result = result, controller = self)

    def test_private_slots(self):
        class PrivateSlot(object):
            __slots__ = ('__data',)
            def __init__(self, data):
                self.__data = data

        data = [42] * 100
        container = PrivateSlot(data)
        size1 = asizeof.asizeof(container)
        size2 = asizeof.asizeof(data)
        self.assertTrue(size1 > size2, (size1, size2))