Python xform.XFormPillow类代码示例

    def finish_handle(self):
        filepath = os.path.join(settings.FILEPATH, "corehq", "pillows", "mappings", "xform_mapping.py")
        xform_pillow = XFormPillow(create_index=False)

        # current index
        # check current index
        aliased_indices = xform_pillow.check_alias()

        #        current_index = '%s_%s' % (xform_pillow.es_index_prefix, xform_pillow.calc_meta())
        current_index = xform_pillow.es_index

        sys.stderr.write("current index:\n")
        sys.stderr.write('XFORM_INDEX="%s"\n' % current_index)

        # regenerate the mapping dict
        mapping = xform_mapping.XFORM_MAPPING
        xform_pillow.default_mapping = mapping
        delattr(xform_pillow, "_calc_meta_cache")
        # xform_pillow.calc_meta.reset_cache()
        calc_index = "%s_%s" % (xform_pillow.es_index_prefix, xform_pillow.calc_meta())

        if calc_index not in aliased_indices and calc_index != current_index:
            sys.stderr.write("\n\tWarning, current index %s is not aliased at the moment\n" % current_index)
            sys.stderr.write("\tCurrent live aliased index: %s\n\n" % (",".join(aliased_indices)))

        if calc_index != current_index:
            sys.stderr.write(
                "XFORM_INDEX hash has changed, please update \n\t%s\n\tXFORM_INDEX property with the line below:\n"
                % filepath
            )
            sys.stdout.write('XFORM_INDEX="%s"\n' % calc_index)
        else:
            sys.stderr.write("XFORM_INDEX unchanged\n")

class XFormPillowTest(TestCase):

    domain = 'xform-pillowtest-domain'

    def setUp(self):
        FormProcessorTestUtils.delete_all_xforms()
        with trap_extra_setup(ConnectionError):
            self.pillow = XFormPillow()
        self.elasticsearch = self.pillow.get_es_new()
        delete_es_index(self.pillow.es_index)

    def tearDown(self):
        ensure_index_deleted(self.pillow.es_index)

    def test_xform_pillow_couch(self):
        metadata = TestFormMetadata(domain=self.domain)
        form = get_form_ready_to_save(metadata)
        FormProcessorInterface(domain=self.domain).save_processed_models([form])
        self.pillow.process_changes(since=0, forever=False)
        self.elasticsearch.indices.refresh(self.pillow.es_index)
        results = FormES().run()
        self.assertEqual(1, results.total)
        form_doc = results.hits[0]
        self.assertEqual(self.domain, form_doc['domain'])
        self.assertEqual(metadata.xmlns, form_doc['xmlns'])
        self.assertEqual('XFormInstance', form_doc['doc_type'])
        form.delete()

    @override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True)
    def test_xform_pillow_sql(self):
        consumer = get_test_kafka_consumer(topics.FORM_SQL)

        # have to get the seq id before the change is processed
        kafka_seq = consumer.offsets()['fetch'][(topics.FORM_SQL, 0)]

        metadata = TestFormMetadata(domain=self.domain)
        form = get_form_ready_to_save(metadata, is_db_test=True)
        form_processor = FormProcessorInterface(domain=self.domain)
        form_processor.save_processed_models([form])

        # confirm change made it to kafka
        message = consumer.next()
        change_meta = change_meta_from_kafka_message(message.value)
        self.assertEqual(form.form_id, change_meta.document_id)
        self.assertEqual(self.domain, change_meta.domain)

        # send to elasticsearch
        sql_pillow = get_sql_xform_to_elasticsearch_pillow()
        sql_pillow.process_changes(since=kafka_seq, forever=False)
        self.elasticsearch.indices.refresh(self.pillow.es_index)

        # confirm change made it to elasticserach
        results = FormES().run()
        self.assertEqual(1, results.total)
        form_doc = results.hits[0]
        self.assertEqual(self.domain, form_doc['domain'])
        self.assertEqual(metadata.xmlns, form_doc['xmlns'])
        self.assertEqual('XFormInstance', form_doc['doc_type'])

    def testXFormPillowSingleCaseProcess(self):
        """
        Test that xform pillow can process and cleanup a single xform with a case submission
        """
        xform = XFORM_SINGLE_CASE
        pillow = XFormPillow(create_index=False, online=False)
        changed = pillow.change_transform(xform)

        self.assertIsNone(changed['form']['case'].get('@date_modified'))
        self.assertIsNotNone(xform['form']['case']['@date_modified'])

    def testXFormMapping(self):
        """
        Verify that a simple case doc will yield the basic mapping
        """

        pillow = XFormPillow(create_index=False, online=False)
        t1 = pillow.get_mapping_from_type(XFORM_SINGLE_CASE)
        t2 = pillow.get_mapping_from_type(XFORM_MULTI_CASES)

        self.assertEqual(t1, t2)

def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None):
    metadata = TestFormMetadata(
        domain=domain,
        time_end=received_on,
        received_on=received_on,
        app_id=app_id,
        user_id=user_id,
        device_id=device_id,
        username=username

    )
    form_pair = make_es_ready_form(metadata)
    pillow = XFormPillow()
    pillow.change_transport(form_pair.json_form)

    def test_get_list(self):
        """
        Any form in the appropriate domain should be in the list from the API.
        """

        # The actual infrastructure involves saving to CouchDB, having PillowTop
        # read the changes and write it to ElasticSearch.

        # In order to test just the API code, we set up a fake XFormES (this should
        # really be a parameter to the XFormInstanceResource constructor)
        # and write the translated form directly; we are not trying to test
        # the ptop infrastructure.

        #the pillow is set to offline mode - elasticsearch not needed to validate
        pillow = XFormPillow(online=False)
        fake_xform_es = FakeXFormES()
        v0_4.MOCK_XFORM_ES = fake_xform_es

        backend_form = XFormInstance(xmlns = 'fake-xmlns',
                                     domain = self.domain.name,
                                     received_on = datetime.utcnow(),
                                     form = {
                                         '#type': 'fake-type',
                                         '@xmlns': 'fake-xmlns'
                                     })
        backend_form.save()
        translated_doc = pillow.change_transform(backend_form.to_json())
        fake_xform_es.add_doc(translated_doc['_id'], translated_doc)

        self.client.login(username=self.username, password=self.password)

        response = self.client.get(self.list_endpoint)
        self.assertEqual(response.status_code, 200)

        api_forms = simplejson.loads(response.content)['objects']
        self.assertEqual(len(api_forms), 1)

        api_form = api_forms[0]
        self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns)
        self.assertEqual(api_form['received_on'], backend_form.received_on.isoformat())

        backend_form.delete()

    def setUpClass(cls):
        form_pillow = XFormPillow(online=False)
        case_pillow = CasePillow(online=False)
        cls.pillows = [form_pillow, case_pillow]

        es = get_es_new()
        with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"):
            for pillow in cls.pillows:
                completely_initialize_pillow_index(pillow)
            initialize_index_and_mapping(es, GROUP_INDEX_INFO)

        case = new_case(closed=True)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=True, owner_id="foo")
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False, owner_id="bar")
        case_pillow.send_robust(case.to_json())

        group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
        cls.group_id = group._id
        send_to_elasticsearch('groups', group.to_json())

        form = new_form(form={"meta": {"userID": None}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": ""}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"deviceID": "abc"}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": uuid.uuid4().hex}})
        form_pillow.send_robust(form.to_json())

        for pillow in cls.pillows:
            pillow.get_es_new().indices.refresh(pillow.es_index)
        es.indices.refresh(GROUP_INDEX_INFO.index)

 def setUp(self):
     FormProcessorTestUtils.delete_all_xforms()
     with trap_extra_setup(ConnectionError):
         self.pillow = XFormPillow()
     self.elasticsearch = self.pillow.get_es_new()
     delete_es_index(self.pillow.es_index)

 def setUp(self):
     FormProcessorTestUtils.delete_all_xforms()
     self.pillow = XFormPillow()
     self.elasticsearch = self.pillow.get_es_new()
     delete_es_index(self.pillow.es_index)