Python tasks.MultipartUploadContext类代码示例

 def test_normal_non_threaded(self):
     # The context object is pretty straightforward.
     # This shows the non threaded usage of this object.
     context = MultipartUploadContext(expected_parts=3)
     # First you can announce an upload id.
     context.announce_upload_id('my_upload_id')
     # Then a thread that was waiting on the id would be notified.
     self.assertEqual(context.wait_for_upload_id(), 'my_upload_id')
     # Then thread would chug away at the parts.
     context.announce_finished_part(etag='etag1', part_number=1)
     context.announce_finished_part(etag='etag2', part_number=2)
     context.announce_finished_part(etag='etag3', part_number=3)
     # Then a thread that was waiting for all the parts to finish
     # would be notified.
     self.assertEqual(context.wait_for_parts_to_finish(), [
         {'ETag': 'etag1', 'PartNumber': 1},
         {'ETag': 'etag2', 'PartNumber': 2},
         {'ETag': 'etag3', 'PartNumber': 3}])

    def test_streaming_threaded_parts(self):
        # This is similar to the basic threaded parts test but instead
        # the thread has to wait to know exactly how many parts are 
        # expected from the stream.  This is indicated when the expected
        # parts of the context changes from ... to an integer.

        self.context = MultipartUploadContext(expected_parts='...')
        upload_part_thread = threading.Thread(target=self.upload_part,
                                              args=(1,))
        # Once this thread starts it will immediately block.
        self.start_thread(upload_part_thread)

        # Also, let's start the thread that will do the complete
        # multipart upload.  It will also block because it needs all
        # the parts so it's blocked up the upload_part_thread.  It also
        # needs the upload_id so it's blocked on that as well.
        complete_upload_thread = threading.Thread(target=self.complete_upload)
        self.start_thread(complete_upload_thread)

        # Then finally the CreateMultipartUpload completes and we
        # announce the upload id.
        self.create_upload('my_upload_id')
        # The complete upload thread should still be waiting for an expect
        # parts number.
        with self.call_lock:
            was_completed = (len(self.calls) > 2)

        # The upload_part thread can now proceed as well as the complete
        # multipart upload thread.
        self.context.announce_total_parts(1)
        self.join_threads()

        self.assertIsNone(self.caught_exception)

        # Make sure that the completed task was never called since it was
        # waiting to announce the parts.
        self.assertFalse(was_completed)        

        # We can verify that the invariants still hold.
        self.assertEqual(len(self.calls), 3)
        # First there should be three calls, create, upload, complete.
        self.assertEqual(self.calls[0][0], 'create_multipart_upload')
        self.assertEqual(self.calls[1][0], 'upload_part')
        self.assertEqual(self.calls[2][0], 'complete_upload')

        # Verify the correct args were used.
        self.assertEqual(self.calls[0][1], 'my_upload_id')
        self.assertEqual(self.calls[1][1:], (1, 'my_upload_id'))
        self.assertEqual(
            self.calls[2][1:],
            ('my_upload_id', [{'ETag': 'etag1', 'PartNumber': 1}]))

    def test_randomized_stress_test(self):
        # Now given that we've verified the functionality from
        # the two tests above, we randomize the threading to ensure
        # that the order doesn't actually matter.  The invariant that
        # the CreateMultipartUpload is called first, then UploadPart
        # operations are called with the appropriate upload_id, then
        # CompleteMultipartUpload with the appropriate upload_id and
        # parts list should hold true regardless of how the threads
        # are ordered.

        # I've run this with much larger values, but 100 is a good
        # tradeoff with coverage vs. execution time.
        for i in range(100):
            expected_parts = random.randint(2, 50)
            self.context = MultipartUploadContext(expected_parts=expected_parts)
            self.threads = []
            self.calls = []
            all_threads = [
                threading.Thread(target=self.complete_upload),
                threading.Thread(target=self.create_upload,
                                args=('my_upload_id',)),
                threading.Thread(target=self.wait_for_upload_complete),
            ]
            for i in range(1, expected_parts + 1):
                all_threads.append(
                    threading.Thread(target=self.upload_part, args=(i,))
                )
            random.shuffle(all_threads)
            for thread in all_threads:
                self.start_thread(thread)
            self.join_threads()
            self.assertEqual(self.calls[0][0], 'create_multipart_upload')
            self.assertEqual(self.calls[-1][0],
                             'arbitrary_post_complete_operation')
            self.assertEqual(self.calls[-2][0], 'complete_upload')
            parts = set()
            for call in self.calls[1:-2]:
                self.assertEqual(call[0], 'upload_part')
                self.assertEqual(call[2], 'my_upload_id')
                parts.add(call[1])
            self.assertEqual(len(parts), expected_parts)

 def setUp(self):
     self.context = MultipartUploadContext(expected_parts=1)
     self.calls = []
     self.threads = []
     self.call_lock = threading.Lock()
     self.caught_exception = None

class TestMultipartUploadContext(unittest.TestCase):

    def setUp(self):
        self.context = MultipartUploadContext(expected_parts=1)
        self.calls = []
        self.threads = []
        self.call_lock = threading.Lock()
        self.caught_exception = None

    def tearDown(self):
        self.join_threads()

    def join_threads(self):
        for thread in self.threads:
            thread.join()

    def upload_part(self, part_number):
        # This simulates what a thread would do if it wanted to upload
        # a part.  First it would wait for the upload id.
        try:
            upload_id = self.context.wait_for_upload_id()
        except Exception as e:
            self.caught_exception = e
            return
        with self.call_lock:
            self.calls.append(('upload_part', part_number, upload_id))
        # Then it would call UploadPart here.
        # Then it would announce that it's finished with a part.
        self.context.announce_finished_part(etag='etag%s' % part_number,
                                            part_number=part_number)

    def complete_upload(self):
        try:
            upload_id = self.context.wait_for_upload_id()
            parts = self.context.wait_for_parts_to_finish()
        except Exception as e:
            self.caught_exception = e
            return
        with self.call_lock:
            self.calls.append(('complete_upload', upload_id, parts))
            self.context.announce_completed()

    def wait_for_upload_complete(self):
        try:
            self.context.wait_for_completion()
        except Exception as e:
            self.caught_exception = e
            return
        with self.call_lock:
            self.calls.append(('arbitrary_post_complete_operation',))

    def create_upload(self, upload_id):
        with self.call_lock:
            self.calls.append(('create_multipart_upload', 'my_upload_id'))
        self.context.announce_upload_id(upload_id)

    def start_thread(self, thread):
        thread.start()
        self.threads.append(thread)

    def test_normal_non_threaded(self):
        # The context object is pretty straightforward.
        # This shows the non threaded usage of this object.
        context = MultipartUploadContext(expected_parts=3)
        # First you can announce an upload id.
        context.announce_upload_id('my_upload_id')
        # Then a thread that was waiting on the id would be notified.
        self.assertEqual(context.wait_for_upload_id(), 'my_upload_id')
        # Then thread would chug away at the parts.
        context.announce_finished_part(etag='etag1', part_number=1)
        context.announce_finished_part(etag='etag2', part_number=2)
        context.announce_finished_part(etag='etag3', part_number=3)
        # Then a thread that was waiting for all the parts to finish
        # would be notified.
        self.assertEqual(context.wait_for_parts_to_finish(), [
            {'ETag': 'etag1', 'PartNumber': 1},
            {'ETag': 'etag2', 'PartNumber': 2},
            {'ETag': 'etag3', 'PartNumber': 3}])
        context.announce_completed()
        # This will return right away since we've already announced completion.
        self.assertIsNone(context.wait_for_completion())

    def test_basic_threaded_parts(self):
        # Now while test_normal_non_threaded showed the conceptual idea,
        # the real strength of MultipartUploadContext is that it works
        # when there are threads and when these threads operate out of
        # sequence.
        # For example, let's say a thread comes along that wants
        # to upload a part.  It needs to wait until the upload id
        # is announced.
        upload_part_thread = threading.Thread(target=self.upload_part,
                                              args=(1,))
        # Once this thread starts it will immediately block.
        self.start_thread(upload_part_thread)

        # Also, let's start the thread that will do the complete
        # multipart upload.  It will also block because it needs all
        # the parts so it's blocked up the upload_part_thread.  It also
        # needs the upload_id so it's blocked on that as well.
        complete_upload_thread = threading.Thread(target=self.complete_upload)
#.........这里部分代码省略.........