Python models.RepeatRecord类代码示例

 def testLocks(self):
     r = RepeatRecord(domain='test')
     r.save()
     r2 = RepeatRecord.get(r._id)
     self.assertTrue(r.acquire_lock(datetime.utcnow()))
     r3 = RepeatRecord.get(r._id)
     self.assertFalse(r2.acquire_lock(datetime.utcnow()))
     self.assertFalse(r3.acquire_lock(datetime.utcnow()))
     r.release_lock()
     r4 = RepeatRecord.get(r._id)
     self.assertTrue(r4.acquire_lock(datetime.utcnow()))

 def tearDown(self):
     self.case_repeater.delete()
     self.form_repeater.delete()
     XFormInstance.get(instance_id).delete()
     repeat_records = RepeatRecord.all()
     for repeat_record in repeat_records:
         repeat_record.delete()

def check_repeaters():
    start = datetime.utcnow()
    LIMIT = 100
    while True:
        number_locked = 0
        # take LIMIT records off the top
        # the assumption is that they all get 'popped' in the for loop
        # the only exception I can see is if there's a problem with the
        # locking, a large number of locked tasks could pile up at the top,
        # so make a provision for that worst case
        repeat_records = RepeatRecord.all(due_before=start, limit=LIMIT + number_locked)
        if repeat_records.count() <= number_locked:
            # don't keep spinning if there's nothing left to fetch
            return

        for repeat_record in repeat_records:
            now = datetime.utcnow()

            # abort if taking too long, so the next task can take over
            if now - start > CHECK_REPEATERS_INTERVAL:
                return

            if repeat_record.acquire_lock(start):
                repeat_record.fire()
                repeat_record.save()
                repeat_record.release_lock()
            else:
                number_locked += 1

def check_repeaters(request, domain):
    if request.method == 'GET':
        return HttpResponse(PAGE % {'status': '', 'domain': domain})
    elif request.method == 'POST':
        start = datetime.utcnow()
        repeat_records = RepeatRecord.all(domain, due_before=start, limit=100)
        process_repeater_list(repeat_records)
        return HttpResponse(PAGE % {'status': 'Done', 'domain': domain})

    def test_repeater(self):

        CommCareCase.get(case_id)

        now = datetime.utcnow()


        repeat_records = RepeatRecord.all(domain=self.domain, due_before=now)
        self.assertEqual(len(repeat_records), 2)

        self.clear_log()


        for repeat_record in repeat_records:
            repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
            repeat_record.save()

        for (url, status, data) in self.log:
            self.assertEqual(status, 404)

        self.clear_log()

        in30min = now + timedelta(minutes=30)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=now + timedelta(minutes=15))
        self.assertEqual(len(repeat_records), 0)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min + timedelta(seconds=1))
        self.assertEqual(len(repeat_records), 2)

        for repeat_record in repeat_records:
            self.assertLess(abs(in30min - repeat_record.next_check), timedelta(seconds=1))
            repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
            repeat_record.save()

        self.assertEqual(len(self.log), 4)
        self.assertEqual(self.log[1], (self.form_repeater.url, 200, xform_xml))
        self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200))
        check_xml_line_by_line(self, self.log[3][2], case_block)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=in30min)
        for repeat_record in repeat_records:
            self.assertEqual(repeat_record.succeeded, True)
            self.assertEqual(repeat_record.next_check, None)

    def handle(self, *args, **options):
        if len(args) == 1:
            domain = args[0]
        else:
            raise CommandError("Usage: %s\n%s" % (self.args, self.help))

        next_year = datetime.datetime.now() + datetime.timedelta(days=365)
        records = RepeatRecord.all(domain=domain, due_before=next_year)
        for record in records:
            record.fire(post_fn=simple_post)
            record.save()
            print "{} {}".format(record._id, "successful" if record.succeeded else "failed")

def check_repeaters():
    now = datetime.utcnow()

    repeat_records = RepeatRecord.all(due_before=now)

    for repeat_record in repeat_records:
        repeat_record.fire()
        try:
            repeat_record.save()
        except ResourceConflict:
            logging.error("ResourceConflict with repeat_record %s: %s" % (repeat_record.get_id, repeat_record.to_json()))
            raise

    def test_repeater(self):

        CommCareCase.get(case_id)

        def now():
            return datetime.utcnow()


        repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
        self.assertEqual(len(repeat_records), 2)

        self.clear_log()


        for repeat_record in repeat_records:
            repeat_record.fire(post_fn=self.make_post_fn([404, 404, 404]))
            repeat_record.save()

        for (url, status, data, headers) in self.log:
            self.assertEqual(status, 404)

        self.clear_log()

        next_check_time = now() + timedelta(minutes=60)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=now() + timedelta(minutes=15))
        self.assertEqual(len(repeat_records), 0)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time + timedelta(seconds=2))
        self.assertEqual(len(repeat_records), 2)

        for repeat_record in repeat_records:
            self.assertLess(abs(next_check_time - repeat_record.next_check), timedelta(seconds=2))
            repeat_record.fire(post_fn=self.make_post_fn([404, 200]))
            repeat_record.save()

        self.assertEqual(len(self.log), 4)
        self.assertEqual(self.log[1][:3], (self.form_repeater.url, 200, xform_xml))
        self.assertIn('received-on', self.log[1][3])
        self.assertEqual(self.log[3][:2], (self.case_repeater.url, 200))
        self.assertIn('server-modified-on', self.log[3][3])
        check_xml_line_by_line(self, self.log[3][2], case_block)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=next_check_time)
        for repeat_record in repeat_records:
            self.assertEqual(repeat_record.succeeded, True)
            self.assertEqual(repeat_record.next_check, None)


        repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
        self.assertEqual(len(repeat_records), 0)

        self.post_xml(update_xform_xml)

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=now())
        self.assertEqual(len(repeat_records), 2)

    def test_repeat_record_created(self):
        '''
        Tests that whenever an application with a repeater is saved that a repeat record is created.
        '''
        application = Application(domain=self.domain)
        application.save()

        repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())
        self.assertEqual(len(repeat_records), 0)
        
        app_structure_repeater = AppStructureRepeater(domain=self.domain, url=self.forwarding_url)
        app_structure_repeater.save()

        application.save()
        repeat_records = RepeatRecord.all(domain=self.domain, due_before=datetime.utcnow())

        self.assertEqual(len(repeat_records), 1)
        for repeat_record in repeat_records:
                self.assertEqual(repeat_record.url, self.forwarding_url)
                self.assertEqual(repeat_record.get_payload(), application.get_id)
                repeat_record.delete()
        
        application.delete()
        app_structure_repeater.delete()

    def handle(self, *args, **options):
        domain = args[0]
        old_db = Domain.get_db()
        new_db = RepeatRecord.get_db()
        assert old_db.dbname != new_db.dbname
        doc_ids = get_doc_ids(domain, 'RepeatRecord', old_db)
        count = len(doc_ids)
        chunksize = 250

        for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)):
            for doc in docs:
                if '_rev' in doc:
                    del doc['_rev']
            new_db.bulk_save(docs, new_edits=False)
            print 'checked %s / %s' % (i * chunksize, count)

 def loop():
     # take LIMIT records off the top
     # the assumption is that they all get 'popped' in the for loop
     # the only exception I can see is if there's a problem with the
     # locking, a large number of locked tasks could pile up at the top,
     # so make a provision for that worst case
     number_locked = progress_report['number_locked']
     repeat_records = RepeatRecord.all(
         due_before=start,
         limit=LIMIT + number_locked
     )
     return process_repeater_list(
         repeat_records,
         start=start,
         cutoff=start + CHECK_REPEATERS_INTERVAL,
         progress_report=progress_report
     )

    def test_ignore_document(self):
        """
        When get_payload raises IgnoreDocument, fire should call update_success
        """

        @RegisterGenerator(FormRepeater, 'new_format', 'XML')
        class NewFormGenerator(BasePayloadGenerator):
            def get_payload(self, repeat_record, payload_doc):
                raise IgnoreDocument

        repeat_records = RepeatRecord.all(
            domain=self.domain,
        )
        for repeat_record_ in repeat_records:
            repeat_record_.fire()

            self.assertIsNone(repeat_record_.next_check)
            self.assertTrue(repeat_record_.succeeded)

    def handle(self, *args, **options):
        domain = args[0]
        old_db = Domain.get_db()
        new_db = RepeatRecord.get_db()
        assert old_db.dbname != new_db.dbname
        doc_ids = [
            r["id"]
            for r in old_db.view(
                "domain/docs", startkey=[domain, "RepeatRecord"], endkey=[domain, "RepeatRecord", {}], reduce=False
            )
        ]
        count = len(doc_ids)
        chunksize = 250

        for i, docs in enumerate(chunked(iter_docs(old_db, doc_ids, chunksize), chunksize)):
            for doc in docs:
                if "_rev" in doc:
                    del doc["_rev"]
            new_db.bulk_save(docs, new_edits=False)
            print "checked %s / %s" % (i * chunksize, count)

    def handle(self, *args, **options):
        domain = args[0]
        db = Domain.get_db()
        doc_ids = get_doc_ids(domain, "RepeatRecord", db)
        count = len(doc_ids)
        print "found %s doc ids" % count
        latest = datetime.min
        latest_doc = None
        for i, doc in enumerate(iter_docs(db, doc_ids)):
            wrapped = RepeatRecord.wrap(doc)
            if i % 100 == 0:
                print "checked %s / %s" % (i, count)
            if wrapped.last_checked and wrapped.last_checked > latest:
                latest = wrapped.last_checked
                latest_doc = wrapped
                print "new latest: %s" % latest

        if latest_doc:
            print "latest repeater date is %s" % latest
            print "latest repeater is %s" % latest_doc._id
        else:
            print "no relevant repeaters found"

    def loop():
        number_locked = 0
        # take LIMIT records off the top
        # the assumption is that they all get 'popped' in the for loop
        # the only exception I can see is if there's a problem with the
        # locking, a large number of locked tasks could pile up at the top,
        # so make a provision for that worst case
        repeat_records = RepeatRecord.all(
            due_before=start,
            limit=LIMIT + number_locked
        )
        if repeat_records.count() <= number_locked:
            # don't keep spinning if there's nothing left to fetch
            return False

        for repeat_record in repeat_records:
            now = datetime.utcnow()

            # abort if taking too long, so the next task can take over
            if now - start > CHECK_REPEATERS_INTERVAL:
                return False

            if repeat_record.acquire_lock(start):
                if repeat_record.repeater.doc_type.endswith(DELETED):
                    if not repeat_record.doc_type.endswith(DELETED):
                        repeat_record.doc_type += DELETED
                    progress_report['deleted'].append(repeat_record.get_id)
                else:
                    repeat_record.fire()
                    if repeat_record.succeeded:
                        progress_report['success'].append(repeat_record.get_id)
                    else:
                        progress_report['fail'].append(repeat_record.get_id)
                repeat_record.save()
                repeat_record.release_lock()
            else:
                progress_report['locked'].append(repeat_record.get_id)
                number_locked += 1
        return True