Python pypet.Trajectory类代码示例

 def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False):
     ### Load The Trajectory and check if the values are still the same
     newtraj = Trajectory(filename=self.filename)
     newtraj.f_load(name=trajectory_name, index=trajectory_index, as_new=as_new,
                    load_parameters=2, load_derived_parameters=2, load_results=2,
                    load_other_data=2)
     return newtraj

def main():
    # We don't use an environment so we enable logging manually
    logging.basicConfig(level=logging.INFO)

    filename = os.path.join('hdf5','example_16.hdf5')
    traj = Trajectory(filename=filename, overwrite_file=True)

    # The result that will be manipulated
    traj.f_add_result('last_process_name', 'N/A',
                      comment='Name of the last process that manipulated the trajectory')

    with MultiprocContext(trajectory=traj, wrap_mode='LOCK') as mc:
        # The multiprocessing context manager wraps the storage service of the trajectory
        # and passes the wrapped service to the trajectory.
        # Also restores the original storage service in the end.
        # Moreover, wee need to use the `MANAGER_LOCK` wrapping because the locks
        # are pickled and send to the pool for all function executions

        # Start a pool of processes manipulating the trajectory
        iterable = (traj for x in range(20))
        pool = mp.Pool(processes=4)
        # Pass the trajectory and the function to the pool and execute it 20 times
        pool.map_async(manipulate_multiproc_safe, iterable)
        pool.close()
        # Wait for all processes to join
        pool.join()

    # Reload the data from disk and overwrite the existing result in RAM
    traj.results.f_load(load_data=3)
    # Print the name of the last process the trajectory was manipulated by
    print('The last process to manipulate the trajectory was: `%s`' % traj.last_process_name)

    def test_partially_delete_stuff(self):
        traj = Trajectory(name='TestDelete',
                          filename=make_temp_dir('testpartiallydel.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        self.assertTrue('a' in res)
        traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)

        self.assertTrue('c' in res)
        self.assertTrue('a' not in res)

        res['a'] = 'offf'

        self.assertTrue('a' in res)

        traj.f_load(load_results=3)

        self.assertTrue('a' not in res)
        self.assertTrue('c' in res)

        traj.f_delete_item(res, remove_from_trajectory=True)

        self.assertTrue('results' in traj)
        self.assertTrue(res not in traj)

    def test_store_and_load_large_dictionary(self):
        traj = Trajectory(name='Testlargedict', filename=make_temp_dir('large_dict.hdf5'))

        large_dict = {}

        for irun in range(1025):
            large_dict['item_%d' % irun] = irun

        large_dict2 = {}

        for irun in range(33):
            large_dict2['item_%d' % irun] = irun

        traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
        traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')

        traj.f_store()

        traj_name = traj.v_name

        traj2 = Trajectory(filename=make_temp_dir('large_dict.hdf5'))

        traj2.f_load(name=traj_name, load_data=2)

        self.compare_trajectories(traj, traj2)

    def test_iteration_failure(self):
        traj = Trajectory()

        traj.f_add_parameter_group('test.test3')
        traj.f_add_parameter_group('test2')
        traj.test2.f_add_link(traj.test3)

        with self.assertRaises(pex.NotUniqueNodeError):
            traj.test3

    def test_storage_service_errors(self):

        traj = Trajectory(filename=make_temp_dir('testnoservice.hdf5'))

        traj_name = traj.v_name

        # you cannot store stuff before the trajectory was stored once:
        with self.assertRaises(ValueError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)

        traj.f_store()

        with self.assertRaises(ValueError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = 'test')

        with self.assertRaises(pex.NoSuchServiceError):
            traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)

        with self.assertRaises(ValueError):
            traj.f_load(name='test', index=1)

        with self.assertRaises(RuntimeError):
            traj.v_storage_service.store('LIST', [('LEAF',None,None,None,None)],
                                         trajectory_name = traj.v_name)

        with self.assertRaises(ValueError):
            traj.f_load(index=9999)

        with self.assertRaises(ValueError):
            traj.f_load(name='Non-Existising-Traj')

    def test_errors(self):
        filename = make_temp_dir("hdf5errors.hdf5")
        traj = Trajectory(name=make_trajectory_name(self), filename=filename)
        trajname = traj.v_name

        npearray = np.ones((2, 10, 3), dtype=np.float)
        thevlarray = np.array([compat.tobytes("j"), 22.2, compat.tobytes("gutter")])

        with self.assertRaises(TypeError):
            traj.f_add_result(SharedResult, "arrays.vlarray", SharedVLArray()).create_shared_data(obj=thevlarray)
        traj.f_store()
        traj.arrays.vlarray.create_shared_data(obj=thevlarray)
        traj.f_add_result(SharedResult, "arrays.array", SharedArray()).create_shared_data(data=npearray)
        traj.arrays.f_add_result(SharedResult, "super.carray", SharedCArray(), comment="carray").create_shared_data(
            shape=(10, 10), atom=pt.atom.FloatAtom()
        )
        traj.arrays.f_add_result(SharedResult, "earray", SharedEArray()).create_shared_data("earray", obj=npearray)

        traj.f_store()

        with self.assertRaises(TypeError):
            traj.arrays.array.iter_rows()

        with StorageContextManager(traj) as cm:
            with self.assertRaises(RuntimeError):
                with StorageContextManager(traj) as cm2:
                    pass
            self.assertTrue(traj.v_storage_service.is_open)
            with self.assertRaises(RuntimeError):
                StorageContextManager(traj).f_open_store()

        self.assertFalse(traj.v_storage_service.is_open)

    def test_links_according_to_run(self):

        traj = Trajectory()

        traj.f_add_parameter('test.hi', 44)
        traj.f_explore({'hi': [1,2,3]})

        traj.f_add_parameter_group('test.test.test2')
        traj.f_add_parameter_group('test2')
        traj.test2.f_add_link('test', traj.test)

        traj.v_idx = 1

    def test_link_of_link(self):

        traj = Trajectory()

        traj.f_add_parameter_group('test')
        traj.f_add_parameter_group('test2')

        traj.test.f_add_link('circle1' , traj.test2)
        traj.test2.f_add_link('circle2' , traj.test)
        traj.test.f_add_link('circle2' , traj.test.circle1.circle2)


        self.assertTrue(traj.test.circle2 is traj.test)

    def test_max_depth_loading_and_storing(self):
        filename = make_temp_dir('newassignment.hdf5')
        traj = Trajectory(filename=filename, overwrite_file=True)

        traj.par.d1 = Parameter('d1.d2.d3.d4.d5', 55)
        traj.f_store(max_depth=4)

        traj = load_trajectory(index=-1, filename=filename)
        traj.f_load(load_data=2)
        self.assertTrue('d3' in traj)
        self.assertFalse('d4' in traj)

        traj = load_trajectory(index=-1, filename=filename, max_depth=3)
        self.assertTrue('d2' in traj)
        self.assertFalse('d3' in traj)

        traj.par.f_remove(recursive=True)
        traj.dpar.d1 = Parameter('d1.d2.d3.d4.d5', 123)

        traj.dpar.f_store_child('d1', recursive=True, max_depth=3)
        traj.dpar.f_remove_child('d1', recursive=True)

        self.assertTrue('d1' not in traj)
        traj.dpar.f_load_child('d1', recursive=True)

        self.assertTrue('d3' in traj)
        self.assertTrue('d4' not in traj)

        traj.dpar.f_remove_child('d1', recursive=True)
        self.assertTrue('d1' not in traj)
        traj.dpar.f_load_child('d1', recursive=True, max_depth=2)

        self.assertTrue('d2' in traj)
        self.assertTrue('d3' not in traj)

        traj.dpar.l1 = Parameter('l1.l2.l3.l4.l5', 123)
        traj.dpar.f_store(recursive=True, max_depth=0)
        self.assertFalse(traj.dpar.l1._stored)

        traj.dpar.f_store(recursive=True, max_depth=4)
        traj.dpar.f_remove()
        self.assertTrue('l1' not in traj)
        traj.dpar.f_load(recursive=True)
        self.assertTrue('l4' in traj)
        self.assertTrue('l5' not in traj)

        traj.dpar.f_remove()
        self.assertTrue('l1' not in traj)
        traj.dpar.f_load(recursive=True, max_depth=3)
        self.assertTrue('l3' in traj)
        self.assertTrue('l4' not in traj)

    def test_backwards_compatibility(self):
        # Test only makes sense with python 2.7 or lower
        old_pypet_traj = Trajectory()
        module_path, init_file = os.path.split(pypet.__file__)
        filename= os.path.join(module_path, 'tests','testdata','pypet_v0_1b_6.hdf5')
        old_pypet_traj.f_load(index=-1, load_data=2, force=True, filename=filename)

        self.assertTrue(old_pypet_traj.v_version=='0.1b.6')
        self.assertTrue(old_pypet_traj.par.x==0)
        self.assertTrue(len(old_pypet_traj)==9)
        self.assertTrue(old_pypet_traj.res.runs.r_4.z==12)
        nexplored = len(old_pypet_traj._explored_parameters)
        self.assertGreater(nexplored, 0)
        for param in old_pypet_traj.f_get_explored_parameters():
            self.assertTrue(old_pypet_traj.f_contains(param))

 def test_store_overly_long_comment(self):
     filename = make_temp_dir('remove_errored.hdf5')
     traj = Trajectory(name='traj', add_time=True, filename=filename)
     res=traj.f_add_result('iii', 42, 43, comment=7777 * '6')
     traj.f_store()
     traj.f_remove_child('results', recursive=True)
     traj.f_load_child('results', recursive=True)
     self.assertTrue(traj.iii.v_comment == 7777 * '6')

    def test_no_run_information_loading(self):
        filename = make_temp_dir('testnoruninfo.hdf5')
        traj = Trajectory(name='TestDelete',
                          filename=filename,
                          add_time=True)

        length = 100000
        traj.par.x = Parameter('', 42)
        traj.f_explore({'x': range(length)})

        traj.f_store()

        traj = load_trajectory(index=-1, filename=filename, with_run_information=False)
        self.assertEqual(len(traj), length)
        self.assertEqual(len(traj._run_information), 1)

    def test_storing_and_manipulating(self):
        filename = make_temp_dir("hdf5manipulation.hdf5")
        traj = Trajectory(name=make_trajectory_name(self), filename=filename)
        trajname = traj.v_name

        thedata = np.zeros((1000, 1000))
        res = traj.f_add_result(SharedResult, "shared")
        myarray = SharedArray("array", res, trajectory=traj, add_to_parent=True)
        mytable = SharedTable("t1", res, trajectory=traj, add_to_parent=True)
        mytable2 = SharedTable("t2", res, trajectory=traj, add_to_parent=True)
        mytable3 = SharedTable("t3", res, trajectory=traj, add_to_parent=True)

        traj.f_store(only_init=True)
        myarray.create_shared_data(data=thedata)
        mytable.create_shared_data(first_row={"hi": compat.tobytes("hi"), "huhu": np.ones(3)})
        mytable2.create_shared_data(description={"ha": pt.StringCol(2, pos=0), "haha": pt.FloatCol(pos=1)})
        mytable3.create_shared_data(description={"ha": pt.StringCol(2, pos=0), "haha": pt.FloatCol(pos=1)})

        traj.f_store()

        newrow = {"ha": "hu", "haha": 4.0}

        with self.assertRaises(TypeError):
            row = traj.shared.t2.row

        with StorageContextManager(traj) as cm:
            row = traj.shared.t2.row
            for irun in range(11):
                for key, val in newrow.items():
                    row[key] = val
                row.append()
            traj.shared.t3.flush()

        data = myarray.read()
        arr = myarray.get_data_node()
        self.assertTrue(np.all(data == thedata))

        with StorageContextManager(traj) as cm:
            myarray[2, 2] = 10
            data = myarray.read()
            self.assertTrue(data[2, 2] == 10)

        self.assertTrue(data[2, 2] == 10)
        self.assertFalse(traj.v_storage_service.is_open)

        traj = load_trajectory(name=trajname, filename=filename)

        traj.f_load(load_data=2)

        traj.shared.t2.traj = traj
        traj.shared.t1.traj = traj
        traj.shared.array.traj = traj

        self.assertTrue(traj.shared.t2.nrows == 11, "%s != 11" % str(traj.shared.t2.nrows))
        self.assertTrue(traj.shared.t2[0]["ha"] == compat.tobytes("hu"), traj.shared.t2[0]["ha"])
        self.assertTrue(traj.shared.t2[1]["ha"] == compat.tobytes("hu"), traj.shared.t2[1]["ha"])
        self.assertTrue("huhu" in traj.shared.t1.colnames)
        self.assertTrue(traj.shared.array[2, 2] == 10)

    def test_loading_explored_parameters(self):

        filename = make_temp_dir('load_explored.hdf5')
        traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
        traj.par.x = Parameter('x', 42, comment='answer')
        traj.f_explore({'x':[1,2,3,4]})
        traj.f_store()
        name = traj.v_name

        traj = Trajectory(filename=filename, add_time=False)
        traj.f_load()
        x = traj.f_get('x')
        self.assertIs(x, traj._explored_parameters['parameters.x'])