Python ioutils.get_root_logger函数代码示例

    def test_version_mismatch(self):
        traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
                          add_time=True)

        traj.f_add_parameter('group1.test',42)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj._version='0.1a.1'

        traj.f_store()

        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('testversionmismatch.hdf5'))

        with self.assertRaises(pex.VersionMismatchError):
            traj2.f_load(load_parameters=2, load_results=2)

        traj2.f_load(load_parameters=2, load_results=2, force=True)

        self.compare_trajectories(traj,traj2)

        get_root_logger().info('Mismatch testing done!')

    def test_run_large(self):

        self.explore(self.traj, trials=30)
        self.add_array_params(self.traj)

        self.traj.f_add_parameter('TEST', 'test_run')

        self.env.f_run(write_into_shared_storage)

        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)


        newtraj = self.load_trajectory(trajectory_name=self.traj.v_name, as_new=False)

        self.check_insertions(self.traj)
        self.check_insertions(newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 10.0, 'Size is %sMB > 10MB' % str(size_in_mb))

        for res in self.traj.results.f_iter_leaves():
            if isinstance(res, SharedResult):
                for key in res.f_to_dict():
                    item = res[key]
                    if isinstance(item, SharedData):
                        make_ordinary_result(res, key, trajectory=self.traj)

        newtraj = self.load_trajectory(trajectory_name=self.traj.v_name, as_new=False)
        self.compare_trajectories(self.traj, newtraj)

    def test_expand_after_reload(self):

        self.traj.f_add_parameter('TEST', 'test_expand_after_reload')
        ###Explore
        self.explore(self.traj)

        self.make_run()

        traj_name = self.traj.v_name


        self.env = Environment(trajectory=self.traj,
                          log_stdout=False,
                          log_config=get_log_config())

        self.traj = self.env.v_trajectory

        self.traj.f_load(name=traj_name)
        self.traj.res.f_remove()
        self.traj.dpar.f_remove()

        self.expand()

        get_root_logger().info('\n $$$$$$$$$$$$ Second Run $$$$$$$$$$ \n')
        self.make_run()

        newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj, newtraj)

def postproc(traj, results, idx):
    get_root_logger().info(idx)

    traj.f_load_skeleton()

    if len(results) <= 4 and len(traj) == 4:
        return {'x':[1,2], 'y':[1,2]}
    if len(results) <= 6 and len(traj) == 6:
        traj.f_expand({'x':[2,3], 'y':[0,1]})

def postproc_with_iter_args(traj, results, idx):
    get_root_logger().info(idx)

    if isinstance(traj.v_storage_service, (LockWrapper, ReferenceWrapper)):
        traj.f_load_skeleton()

    if len(results) <= 4 and len(traj) == 4:
        assert idx == 42
        return {'x':[1,2], 'y':[1,2]}, ([5,6],), {'w':[7,8]}, (43,)
    if len(results) <= 6 and len(traj) == 6:
        assert idx == 43
        traj.f_expand({'x':[2,3], 'y':[0,1]})
        return {}, ([7,8],), {'w':[9,10]}

    def test_compacting(self):
        filename = make_temp_dir("hdf5compacting.hdf5")
        traj = Trajectory(name=make_trajectory_name(self), filename=filename)
        trajname = traj.v_name
        traj.v_storage_service.complevel = 7

        first_row = {"ha": compat.tobytes("hi"), "haha": np.zeros((3, 3))}

        traj.f_store(only_init=True)

        res1 = traj.f_add_result("My.Tree.Will.Be.Deleted", 42)
        res2 = traj.f_add_result("Mine.Too.HomeBoy", 42, comment="Don`t cry for me!")

        res = traj.f_add_result(SharedResult, "myres")

        res["myres"] = SharedTable()

        res["myres"].create_shared_data(first_row=first_row)

        with StorageContextManager(traj):
            tab = traj.myres
            for irun in range(10000):
                row = traj.myres.row
                for key in first_row:
                    row[key] = first_row[key]
                row.append()
        traj.f_store()
        del traj
        traj = load_trajectory(name=trajname, filename=filename, load_all=2)
        with StorageContextManager(traj) as cm:
            tb = traj.myres.get_data_node()
            ptcompat.remove_rows(tb, 1000, 10000)

            cm.f_flush_store()
            self.assertTrue(traj.myres.nrows == 1001)

        traj.f_delete_item(traj.My, recursive=True)
        traj.f_delete_item(traj.Mine, recursive=True)

        size = os.path.getsize(filename)
        get_root_logger().info("Filesize is %s" % str(size))
        name_wo_ext, ext = os.path.splitext(filename)
        backup_file_name = name_wo_ext + "_backup" + ext
        code = compact_hdf5_file(filename, keep_backup=True)
        if code != 0:
            raise RuntimeError("ptrepack fail")
        backup_size = os.path.getsize(backup_file_name)
        self.assertTrue(backup_size == size)
        new_size = os.path.getsize(filename)
        get_root_logger().info("New filesize is %s" % str(new_size))
        self.assertTrue(new_size < size, "%s > %s" % (str(new_size), str(size)))

    def test_file_size_many_params(self):
        filename = make_temp_dir('filesize.hdf5')
        traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
        npars = 700
        traj.f_store()
        for irun in range(npars):
            par = traj.f_add_parameter('test.test%d' % irun, 42+irun, comment='duh!')
            traj.f_store_item(par)


        size =  os.path.getsize(filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 10.0, 'Size is %sMB > 10MB' % str(size_in_mb))

    def test_a_large_run(self):
        get_root_logger().info('Testing large run')
        self.traj.f_add_parameter('TEST', 'test_run')
        ###Explore
        self.explore_large(self.traj)
        self.make_run_large_data()

        self.assertTrue(self.traj.f_is_completed())

        # Check if printing and repr work
        get_root_logger().info(str(self.env))
        get_root_logger().info(repr(self.env))

        newtraj = Trajectory()
        newtraj.f_load(name=self.traj.v_name, as_new=False, load_data=2, filename=self.filename)

        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj,newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 30.0, 'Size is %sMB > 30MB' % str(size_in_mb))

def multiply_with_storing(traj):
    rootlogger = get_root_logger()
    z=traj.x*traj.y
    rootlogger.info('z=x*y: '+str(z)+'='+str(traj.x)+'*'+str(traj.y))
    traj.f_add_result('z',z)
    traj.f_store()
    return z

def write_into_shared_storage(traj):
    traj.f_add_result("ggg", 42)
    traj.f_add_derived_parameter("huuu", 46)

    root = get_root_logger()
    daarrays = traj.res.daarrays
    idx = traj.v_idx
    ncores = traj[traj.v_environment_name].f_get_default("ncores", 1)
    root.info("1. This")
    a = daarrays.a
    a[idx] = idx
    root.info("2. is")
    ca = daarrays.ca
    ca[idx] = idx
    root.info("3. a")
    ea = daarrays.ea
    ea.append(np.ones((1, 10)) * idx)
    root.info("4. sequential")
    vla = daarrays.vla
    vla.append(np.ones(idx + 2) * idx)
    root.info("5. Block")
    the_range = list(range(max(0, idx - 2 * ncores), max(0, idx)))
    for irun in the_range:
        x, y = a[irun], irun
        if x != y and x != 0:
            raise RuntimeError("ERROR in write_into_shared_storage %s != %s" % (str(x), str(y)))
        x, y = ca[irun], irun
        if x != y and x != 0:
            raise RuntimeError("ERROR in write_into_shared_storage %s != %s" % (str(x), str(y)))
        try:
            x, y = ea[irun, 9], ea[irun, 8]
            if x != y and x != 0:
                raise RuntimeError("ERROR in write_into_shared_storage %s != %s" % (str(x), str(y)))
        except IndexError:
            pass  # Array is not at this size yet
        try:
            x, y = vla[irun][0], vla[irun][1]
            if x != y and x != 0:
                raise RuntimeError("ERROR in write_into_shared_storage %s != %s" % (str(x), str(y)))
        except IndexError:
            pass  # Array is not at this size yet
    root.info("6. !!!!!!!!!")

    tabs = traj.tabs

    with StorageContextManager(traj) as cm:
        t1 = tabs.t1
        row = t1.row
        row["run_name"] = compat.tobytes(traj.v_crun)
        row["idx"] = idx
        row.append()
        t1.flush()

    t2 = tabs.t2
    row = t2[idx]
    if row["run_name"] != compat.tobytes(traj.v_crun):
        raise RuntimeError("Names in run table do not match, Run: %s != %s" % (row["run_name"], traj.v_crun))

    df = traj.df
    df.append(pd.DataFrame({"idx": [traj.v_idx], "run_name": traj.v_crun}))

def multiply_args(traj, arg1=0, arg2=0, arg3=0):
    rootlogger = get_root_logger()
    z=traj.x*traj.y + arg1 + arg2 + arg3
    rootlogger.info('z=x*y: '+str(z)+'='+str(traj.x)+'*'+str(traj.y) +
                    '+'+str(arg1)+'+'+str(arg2)+'+'+str(arg3))
    traj.f_add_result('z',z)
    return z

    def test_two_runs(self):
        self.traj.f_add_parameter('TEST', 'test_run')
        self.traj.hdf5.purge_duplicate_comments = False
        ###Explore
        self.explore(self.traj)

        self.make_run()

        newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj, newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 6.0, 'Size is %sMB > 6MB' % str(size_in_mb))

        mp_traj = self.traj

        old_multiproc = self.multiproc
        self.multiproc = False

        ### Make a new single core run
        self.setUp()

        self.traj.f_add_parameter('TEST', 'test_run')
        self.traj.hdf5.purge_duplicate_comments = False
        ###Explore
        self.explore(self.traj)

        self.make_run()

        # newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj, newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 6.0, 'Size is %sMB > 6MB' % str(size_in_mb))

        self.compare_trajectories(mp_traj, self.traj)
        self.multiproc = old_multiproc

def postproc(traj, results, idx):
    get_root_logger().info(idx)

    if isinstance(traj.v_storage_service, (LockWrapper, ReferenceWrapper)):
        traj.f_load_skeleton()

    if isinstance(traj.v_storage_service, (QueueStorageServiceSender, PipeStorageServiceSender)):
        try:
            traj.f_load()
            raise RuntimeError('Should not load')
        except NotImplementedError:
            pass

    if len(results) <= 4 and len(traj) == 4:
        return {'x':[1,2], 'y':[1,2]}
    if len(results) <= 6 and len(traj) == 6:
        traj.f_expand({'x':[2,3], 'y':[0,1]})

    def test_just_one_run(self):
        self.make_run()
        newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj, newtraj)

        self.assertTrue(len(newtraj) == 1)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 2.0, 'Size is %sMB > 6MB' % str(size_in_mb))

        with self.assertRaises(TypeError):
            self.explore(self.traj)

    def test_run(self):
        self.traj.f_add_parameter('TEST', 'test_run')
        ###Explore
        self.explore(self.traj)

        self.make_run()

        newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
        self.traj.f_load_skeleton()
        self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)

        self.compare_trajectories(self.traj, newtraj)

        size=os.path.getsize(self.filename)
        size_in_mb = size/1000000.
        get_root_logger().info('Size is %sMB' % str(size_in_mb))
        self.assertTrue(size_in_mb < 6.0, 'Size is %sMB > 6MB' % str(size_in_mb))