Python generators.InstanceGenerator类代码示例

 def test_complex_relationships(self):
     dics = [
         {'record': 40, 'numero': 'uno'},
         {'record': 40, 'numero': 'due'},
         {'record': 43, 'numero': 'tres',
             'related': [
                 {'record': '10', 'ilosc': 'jedynka zero'},
                 {'record': '21', 'ilosc': 'dwadziescia jeden'}
             ]},
         {'record': 43, 'numero': 'quattre',
             'related': [
                 {'record': '10', 'ilosc': 'jedynka zero'},
                 {'record': '21', 'ilosc': 'jeden'},
                 {'record': '19', 'ilosc': 'jeden'}
             ]},
         {'record': 43, 'zahl': None, 'numero': 'uno'},
         {'record': 43, 'zahl': None, 'numero': None},
         {'record': 43, 'numero': 'tres',
             'related': [
                 {'record': '10', 'ilosc': 'jedynka zero'},
                 {'record': '21', 'ilosc': 'dwadziescia jeden'}
             ]
             }
     ]
     for dic in dics:
         generator = InstanceGenerator(
             models.HashTestModel, persistence='record')
         generator.get_instance(dic)
     self.assertEqual(models.Polish.objects.count(), 3)

 def test_selectrelated(self):
     generator = InstanceGenerator(models.RelatedRelated)
     dic = {
         'value': 'test',
         'key': {
             'numero': {
                 'name': 'hello',
             },
             'another': {
                 'record': 'r10',
                 'last_name': 'Mueller',
             },
             'etl_create': True,
             'etl_update': False
         }
     }
     generator.get_instance(dic)
     self.assertEqual(models.Numero.objects.count(), 1)
     self.assertEqual(models.AnotherModel.objects.count(), 1)
     self.assertEqual(models.TwoRelatedAsUnique.objects.count(), 1)
     qs = models.RelatedRelated.objects.all()
     self.assertEqual(qs.count(), 1)
     item = qs[0]
     self.assertEqual(item.key.numero.name, 'hello')
     self.assertEqual(item.key.another.last_name, 'Mueller')
     self.assertEqual(item.value, 'test')
     generator.get_instance(dic)
     qs = models.RelatedRelated.objects.all()
     self.assertEqual(qs.count(), 2)
     item = qs[1]
     self.assertEqual(item.key.numero.name, 'hello')
     self.assertEqual(item.key.another.last_name, 'Mueller')
     self.assertEqual(item.value, 'test')

 def test_full_instance_generator(self):
     dics = [
         {'record': '1', 'name': 'one', 'zahl': 'eins', 'trash': 'rubish'},
     ]
     generator = InstanceGenerator(
         TestModel, dics[0], persistence=['record'])
     generator.get_instance()

 def test_onetoone(self):
     ins = models.Nombre.objects.create(name='un', id=1)
     dos = models.Nombre.objects.create(name='dos', id=2)
     dics = [
         {'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
          'numero': 'uno'},
         {'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
          'numero': 'due'},
         {'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
          {'name': 'troix'}, 'numero': 'tre'},
         {'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
          'numero': 'quattro'},
         {'record': '1', 'name': 'one again', 'zahl': 'fuenf',
          'nombre': dos, 'numero': 'cinque'},
          {'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
           'numero': 'test'},
         {'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
         {'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
          'nombre': '2'},
         {'record': '7', 'name': 'test', 'numero': '1'}
     ]
     for dic in dics:
         generator = InstanceGenerator(
             models.TestOnetoOneModel, persistence='record')
         generator.get_instance(dic)
     res = models.Nombre.objects.all()
     self.assertEqual(res.count(), 5)
     res = models.TestOnetoOneModel.objects.all()
     self.assertEqual(res.count(), 7)
     rec = res.filter(record='1')[0]
     self.assertEqual(rec.nombre.name, 'dos')
     self.assertEqual(rec.nombre.testonetoonemodel, rec)

 def load(self):
     """
     Loads data into database using Django models and error logging.
     """
     print('Opening {0} using {1}'.format(self.filename, self.encoding))
     with FileReaderLogManager(self.filename,
                               logname=self.logfilename,
                               reader_class=self.reader_class,
                               reader_kwargs=self.reader_kwargs,
                               encoding=self.encoding) as reader:
         reader.log(
             'Data extraction started {0}\n\nStart line: '
             '{1}\nEnd line: {2}\n'.format(
                 datetime.now().strftime(
                     '%Y-%m-%d'), self.slice_begin, self.slice_end))
         counter = FeedbackCounter(
             feedbacksize=self.feedbacksize, message=self.message)
         while self.slice_begin and self.slice_begin > counter.counter:
             reader.next()
             counter.increment()
         while not self.slice_end or self.slice_end >= counter.counter:
             try:
                 csv_dic = reader.next()
             except (UnicodeDecodeError, csv.Error):
                 reader.log(
                     'Text decoding or CSV error in line {0} '
                     '=> rejected'.format(counter.counter))
                 counter.reject()
                 continue
             except StopIteration:
                 reader.log('End of file.')
                 break
             transformer = self.transformer_class(csv_dic, self.defaults)
             if transformer.is_valid():
                 dic = transformer.cleaned_data
             else:
                 reader.log(
                     'Validation error in line {0}: {1} '
                     '=> rejected'.format(
                         counter.counter, transformer.error))
                 counter.reject()
                 continue
             # remove keywords conflicting with Django model
             # TODO: I think that is done in several places now
             # determine the correct one and get rid of the others
             if 'id' in dic:
                 del dic['id']
             generator = InstanceGenerator(
                 self.model_class, dic,
                 persistence=self.etl_persistence)
             try:
                 generator.get_instance()
             except (ValidationError, IntegrityError, DatabaseError), e:
                 reader.log('Error in line {0}: {1} => rejected'.format(
                     counter.counter, str(e)))
                 counter.reject()
                 continue
             else:
                 counter.use_result(generator.res)
         reader.log(counter.finished())

 def test_prepare_integer(self):
     generator = InstanceGenerator(models.SomeModel)
     tests = [
         ('1', 1), ('', None), (0, 0), (1, 1), ('bla', None)]
     for test in tests:
         self.assertEqual(
             generator.prepare_integer(None, test[0]), test[1])

 def test_onetoone(self):
     ins = Nombre(name='un')
     dos = Nombre(name='dos')
     dics = [
         {'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
          'numero': 'uno'},
         {'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
          'numero': 'due'},
         {'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
          {'name': 'troix'}, 'numero': 'tre'},
         {'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
          'numero': 'quattro'},
         {'record': '1', 'name': 'one again', 'zahl': 'fuenf',
          'nombre': dos, 'numero': 'cinque'},
         {'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
          'numero': 'test'},
         {'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
         {'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
          'nombre': '2'},
         {'record': '7', 'name': 'test', 'numero': '1'}
     ]
     for dic in dics:
         generator = InstanceGenerator(TestOnetoOneModel, dic, persistence='record')
         generator.get_instance()
     res = Nombre.objects.all()
     self.assertEqual(res.count(), 5)
     res = TestOnetoOneModel.objects.all()
     self.assertEqual(res.count(), 7)
     rec1 = res.filter(record='1')[0]
     self.assertEqual(rec1.nombre.name, 'dos')
     #OneToOneField has the related object as a property
     self.assertEqual(rec1.nombre.testonetoonemodel, rec1)
     res.delete()

 def test_rel_creation(self):
     dics = [
         {'record': '1', 'name': 'one', 'related': [
             {'record': '10', 'ilosc': 'dziesiec',
                 'persistence': 'record'},
             {'record': '20', 'ilosc': 'dwadziescia',
                 'persistence': 'record'},
             ],
             'numero': 'uno'},
         {'record': '2', 'name': 'one', 'related': [
             {'record': '10', 'ilosc': 'jedynka zero',
                 'persistence': 'record'},
             {'record': '21', 'ilosc': 'dwadziescia jeden',
                 'persistence': 'record'},
             ],
             'numero': 'tre'}]
     for dic in dics:
         generator = InstanceGenerator(TestModel, dic, persistence='record')
         generator.get_instance()
     res = Polish.objects.all()
     self.assertEqual(res.count(), 3)
     # tests also field truncation
     self.assertEqual(res.filter(record='10')[0].ilosc, 'jedynka ze')
     res = TestModel.objects.all()
     self.assertEqual(res.count(), 2)
     self.assertEqual(len(res.filter(record='1')[0].related.values()), 2)

 def test_fk(self):
     ins = Nombre(name='un')
     dics = [
         {'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
          'numero': 'uno'},
         {'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
          'numero': 'due'},
         {'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
          {'name': 'troix'}, 'numero': 'tre'},
         {'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
          'numero': 'quattro'},
         {'record': '1', 'name': 'one again', 'zahl': 'fuenf',
          'nombre': 'quatre', 'numero': 'cinque'},
         {'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
          'numero': 'test'},
         {'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
         {'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
          'nombre': '2'},
         {'record': '7', 'name': 'test', 'numero': '1'}
     ]
     for dic in dics:
         generator = InstanceGenerator(TestModel, dic, persistence='record')
         generator.get_instance()
     res = Nombre.objects.all()
     self.assertEqual(res.count(), 5)
     res = TestModel.objects.all()
     self.assertEqual(res.count(), 7)
     self.assertEqual(res.filter(record='1')[0].nombre.name, 'quatre')
     res.delete()

 def test_prepare_boolean(self):
     generator = InstanceGenerator(models.SomeModel)
     tests = [
         ('0', False), ('1', True), ('false', False), ('true', True),
         ('f', False), ('t', True), (1, True), (0, False)]
     for test in tests:
         self.assertEqual(
             generator.prepare_boolean(None, test[0]), test[1])

 def test_instance_generation(self):
     generator = InstanceGenerator(models.TestModelWoFk)
     res = generator.get_instance({
         'record': 1, 'name': 'test', 'zahl': '1'})
     self.assertIsInstance(res, models.TestModelWoFk)
     self.assertEqual(res.record, '1')
     self.assertEqual(res.name, 'test')
     self.assertEqual(res.zahl, '1')

 def test_prepare_geometry(self):
     from django.contrib.gis.geos import GEOSGeometry
     example3d_string = (
         'MULTIPOLYGON (((2 4 5, 2 6 5, 3 3 3, 2 4 5)),'
         '((10 10 3, 11 10 4, 10 11 4, 10 10 3)))')
     geom = GEOSGeometry(example3d_string)
     generator = InstanceGenerator(
         GeometryModel, {'geom2d': geom, 'geom3d': geom,
                         'name': 'testcase 1'})
     generator.get_instance()
     item = GeometryModel.objects.filter(name='testcase 1')[0]
     self.assertFalse(item.geom2d.hasz)
     self.assertTrue(item.geom3d.hasz)
     example2d_string = item.geom2d
     geom = GEOSGeometry(example2d_string)
     generator = InstanceGenerator(
         GeometryModel, {'geom2d': geom, 'geom3d': geom,
                         'name': 'testcase 2'})
     generator.get_instance()
     item = GeometryModel.objects.filter(name='testcase 2')[0]
     self.assertFalse(item.geom2d.hasz)
     generator = InstanceGenerator(
         GeometryModel, {'geom2d': None, 'geom3d': None,
                         'name': 'emptytest'})
     generator.get_instance()
     item = GeometryModel.objects.filter(name='emptytest')[0]
     self.assertFalse(item.geom2d)
     self.assertFalse(item.geom3d)

 def test_fk_with_rel_field_specified(self):
     generator = InstanceGenerator(
         models.TestModel, persistence=['record'])
     instance = generator.get_instance({
         'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': 'un',
         'numero': 'uno', 'elnumero': 'el uno'})
     self.assertEqual(generator.res, 'created')
     self.assertEqual(instance.elnumero.rec, 'el uno')
     instance = generator.get_instance({
         'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
         'numero': 'due', 'elnumero': 'el dos'})
     self.assertEqual(generator.res, 'created')

 def test_fk_rejection(self):
     generator = InstanceGenerator(models.TestModel)
     generator.get_instance({
         'record': '1', 'name': 'one', 'zahl': 'eins',
         'nombre': {
             'name': 'un', 'etl_create': False},
         'numero': 'quattre'})
     self.assertEqual(generator.res, 'created')
     with self.assertRaises((ValueError, IntegrityError)):
         generator.get_instance({
             'record': '2', 'name': 'two', 'zahl': 'eins',
             'numero': {
                 'name': 'uno', 'etl_create': False}})

 def test_update(self):
     """
     Test record update.
     """
     dic = {'record': '100', 'numero': 'cento', 'zahl': 'hundert'}
     generator = InstanceGenerator(HashTestModel, dic, persistence='record')
     res = generator.get_instance()
     self.assertEqual(res.numero.name, 'cento')
     self.assertTrue(generator.res['created'])
     self.assertFalse(generator.res['updated'])
     generator.get_instance()
     self.assertFalse(generator.res['created'])
     self.assertFalse(generator.res['updated'])
     self.assertTrue(generator.res['exists'])
     dic = {'record': '100', 'numero': 'hundert', 'zahl': 'hundert'}
     generator = InstanceGenerator(HashTestModel, dic, persistence='record')
     res = generator.get_instance()
     self.assertTrue(generator.res['updated'])
     self.assertEqual(res.numero.name, 'hundert')
     dic = {'record': '101', 'name': 'test', 'zahl': 'vier'}
     generator = InstanceGenerator(
         TestModelWoFk, dic, persistence='record')
     res = generator.get_instance()
     res = generator.get_instance()
     self.assertTrue(generator.res['updated'])