Python task_scheduler.bot_reap_task函数代码示例

 def test_cron_handle_bot_died_second(self):
   # Test two tries internal_failure's leading to a BOT_DIED status.
   self.mock(random, 'getrandbits', lambda _: 0x88)
   now = utils.utcnow()
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       created_ts=now,
       expiration_ts=now+datetime.timedelta(seconds=600))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
   self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local'))
   now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
   # It must be a different bot.
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost-second', 'abc')
   now_2 = self.mock_now(self.now + 2 * task_result.BOT_PING_TOLERANCE, 3)
   self.assertEqual(
       (['1d69b9f088008812'], 0, 0),
       task_scheduler.cron_handle_bot_died('f.local'))
   self.assertEqual(([], 0, 0), task_scheduler.cron_handle_bot_died('f.local'))
   expected = {
     'abandoned_ts': now_2,
     'bot_dimensions': bot_dimensions,
     'bot_id': u'localhost-second',
     'bot_version': u'abc',
     'children_task_ids': [],
     'completed_ts': None,
     'costs_usd': [0., 0.],
     'cost_saved_usd': None,
     'created_ts': self.now,
     'deduped_from': None,
     'durations': [],
     'exit_codes': [],
     'failure': False,
     'id': '1d69b9f088008810',
     'internal_failure': True,
     'modified_ts': now_2,
     'name': u'Request name',
     'outputs_ref': None,
     'properties_hash': None,
     'server_versions': [u'v1a'],
     'started_ts': now_1,
     'state': task_result.State.BOT_DIED,
     'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
     'try_number': 2,
     'user': u'Jesus',
   }
   self.assertEqual(expected, run_result.result_summary_key.get().to_dict())

  def test_bot_kill_task(self):
    self.mock(random, 'getrandbits', lambda _: 0x88)
    dimensions = {u'OS': u'Windows-3.1.1'}
    request = task_request.make_request(
        _gen_request(properties={'dimensions': dimensions}), True)
    result_summary = task_scheduler.schedule_request(request)
    reaped_request, run_result = task_scheduler.bot_reap_task(
        {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')

    self.assertEqual(
        None, task_scheduler.bot_kill_task(run_result.key, 'localhost'))
    expected = {
      'abandoned_ts': self.now,
      'bot_dimensions': dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': True,
      'modified_ts': self.now,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': State.BOT_DIED,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, result_summary.key.get().to_dict())
    expected = {
      'abandoned_ts': self.now,
      'bot_dimensions': dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'cost_usd': 0.,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008811',
      'internal_failure': True,
      'modified_ts': self.now,
      'outputs_ref': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': State.BOT_DIED,
      'try_number': 1,
    }
    self.assertEqual(expected, run_result.key.get().to_dict())

  def test_bot_update_pubsub_error(self):
    data = _gen_request(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        pubsub_topic='projects/abc/topics/def')
    request = task_request.make_request(data, True)
    task_scheduler.schedule_request(request)
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    self.assertEqual('localhost', run_result.bot_id)

    # Attempt to terminate the task with success, but make PubSub call fail.
    self.mock_pub_sub(publish_successful=False)
    self.assertEqual(
        (False, False),
        task_scheduler.bot_update_task(
            run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
            0.1, None))

    # Bot retries bot_update, now PubSub works and notification is sent.
    pub_sub_calls = self.mock_pub_sub(publish_successful=True)
    self.assertEqual(
        (True, True),
        task_scheduler.bot_update_task(
            run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
            0.1, None))
    self.assertEqual(1, len(pub_sub_calls)) # notification is sent

  def _bot_update_timeouts(self, hard, io):
    self.mock(random, 'getrandbits', lambda _: 0x88)
    data = _gen_request_data(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
    request = task_request.make_request(data)
    result_summary = task_scheduler.schedule_request(request)
    reaped_request, run_result = task_scheduler.bot_reap_task(
        {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
    self.assertEqual(
        (True, True),
        task_scheduler.bot_update_task(
            run_result.key, 'localhost', 'hi', 0, 0, 0.1, hard, io, 0.1))
    expected = {
      'abandoned_ts': None,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': self.now,
      'costs_usd': [0.1],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [0.1],
      'exit_codes': [0],
      'failure': True,
      'id': '1d69b9f088008810',
      'internal_failure': False,
      'modified_ts': self.now,
      'name': u'Request name',
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': State.TIMED_OUT,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, result_summary.key.get().to_dict())

    expected = {
      'abandoned_ts': None,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': self.now,
      'cost_usd': 0.1,
      'durations': [0.1],
      'exit_codes': [0],
      'failure': True,
      'id': '1d69b9f088008811',
      'internal_failure': False,
      'modified_ts': self.now,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': State.TIMED_OUT,
      'try_number': 1,
    }
    self.assertEqual(expected, run_result.key.get().to_dict())

def _quick_reap():
  """Reaps a task."""
  data = _gen_request(
      properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
  request = task_request.make_request(data, True)
  _result_summary = task_scheduler.schedule_request(request)
  reaped_request, run_result = task_scheduler.bot_reap_task(
      {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
  return run_result

  def test_cron_abort_expired_task_to_run_retry(self):
    self.mock(random, 'getrandbits', lambda _: 0x88)
    now = utils.utcnow()
    data = _gen_request(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=600))
    request = task_request.make_request(data, True)
    result_summary = task_scheduler.schedule_request(request)

    # Fake first try bot died.
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
    self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())
    self.assertEqual(task_result.State.BOT_DIED, run_result.key.get().state)
    self.assertEqual(
        task_result.State.PENDING, run_result.result_summary_key.get().state)

    # BOT_DIED is kept instead of EXPIRED.
    abandoned_ts = self.mock_now(self.now, request.expiration_secs+1)
    self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run())
    self.assertEqual(1, len(task_result.TaskRunResult.query().fetch()))
    expected = {
      'abandoned_ts': abandoned_ts,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': True,
      'modified_ts': abandoned_ts,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': task_result.State.BOT_DIED,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, result_summary.key.get().to_dict())

 def test_cancel_task_running(self):
   data = _gen_request_data(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
   request = task_request.make_request(data)
   result_summary = task_scheduler.schedule_request(request)
   reaped_request, run_result = task_scheduler.bot_reap_task(
       {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
   ok, was_running = task_scheduler.cancel_task(result_summary.key)
   self.assertEqual(False, ok)
   self.assertEqual(True, was_running)
   result_summary = result_summary.key.get()
   self.assertEqual(task_result.State.RUNNING, result_summary.state)

 def test_bot_kill_task_wrong_bot(self):
   self.mock(random, 'getrandbits', lambda _: 0x88)
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
   request = task_request.make_request(data, True)
   result_summary = task_scheduler.schedule_request(request)
   reaped_request, run_result = task_scheduler.bot_reap_task(
       {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
   expected = (
     'Bot bot1 sent task kill for task 1d69b9f088008811 owned by bot '
     'localhost')
   self.assertEqual(
       expected, task_scheduler.bot_kill_task(run_result.key, 'bot1'))

 def _task_deduped(
     self, new_ts, deduped_from, task_id='1d8dc670a0008810', now=None):
   data = _gen_request(
       name='yay',
       user='Raoul',
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}, idempotent=True))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   self.assertEqual(None, task_to_run.TaskToRun.query().get().queue_number)
   actual_request_2, run_result_2 = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(None, actual_request_2)
   result_summary_duped, run_results_duped = get_results(request.key)
   expected = {
     'abandoned_ts': None,
     'bot_dimensions': bot_dimensions,
     'bot_id': u'localhost',
     'bot_version': u'abc',
     'children_task_ids': [],
     'completed_ts': now or self.now,
     'costs_usd': [],
     'cost_saved_usd': 0.1,
     'created_ts': new_ts,
     'deduped_from': deduped_from,
     'durations': [0.1],
     'exit_codes': [0],
     'failure': False,
     'id': task_id,
     'internal_failure': False,
     # Only this value is updated to 'now', the rest uses the previous run
     # timestamps.
     'modified_ts': new_ts,
     'name': u'yay',
     'outputs_ref': None,
     # A deduped task cannot be deduped against.
     'properties_hash': None,
     'server_versions': [u'v1a'],
     'started_ts': now or self.now,
     'state': State.COMPLETED,
     'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Raoul'],
     'try_number': 0,
     'user': u'Raoul',
   }
   self.assertEqual(expected, result_summary_duped.to_dict())
   self.assertEqual([], run_results_duped)

 def test_cancel_task_running(self):
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       pubsub_topic='projects/abc/topics/def')
   pub_sub_calls = self.mock_pub_sub()
   request = task_request.make_request(data, True)
   result_summary = task_scheduler.schedule_request(request)
   reaped_request, run_result = task_scheduler.bot_reap_task(
       {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
   ok, was_running = task_scheduler.cancel_task(result_summary.key)
   self.assertEqual(False, ok)
   self.assertEqual(True, was_running)
   result_summary = result_summary.key.get()
   self.assertEqual(task_result.State.RUNNING, result_summary.state)
   self.assertEqual(0, len(pub_sub_calls)) # no notifications

 def test_bot_reap_task(self):
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   actual_request, run_result  = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(request, actual_request)
   self.assertEqual('localhost', run_result.bot_id)
   self.assertEqual(None, task_to_run.TaskToRun.query().get().queue_number)

  def test_task_parent_isolated(self):
    request = task_request.make_request(
        _gen_request(
            properties={
              'commands': None,
              'dimensions': {u'OS': u'Windows-3.1.1'},
              'inputs_ref': {
                'isolated': '1' * 40,
                'isolatedserver': 'http://localhost:1',
                'namespace': 'default-gzip',
              },
            }),
        True)
    _result_summary = task_scheduler.schedule_request(request)
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    actual_request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    self.assertEqual(request, actual_request)
    self.assertEqual('localhost', run_result.bot_id)
    self.assertEqual(None, task_to_run.TaskToRun.query().get().queue_number)
    # It's important to terminate the task with success.
    self.assertEqual(
        (True, True),
        task_scheduler.bot_update_task(
            run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
            0.1, None))

    parent_id = run_result.task_id
    request = task_request.make_request(
        _gen_request(
            parent_task_id=parent_id,
            properties={'dimensions':{u'OS': u'Windows-3.1.1'}}),
        True)
    result_summary = task_scheduler.schedule_request(request)
    self.assertEqual([], result_summary.children_task_ids)
    self.assertEqual(parent_id, request.parent_task_id)

    parent_run_result_key = task_pack.unpack_run_result_key(parent_id)
    parent_res_summary_key = task_pack.run_result_key_to_result_summary_key(
        parent_run_result_key)
    expected = [result_summary.task_id]
    self.assertEqual(expected, parent_run_result_key.get().children_task_ids)
    self.assertEqual(expected, parent_res_summary_key.get().children_task_ids)

 def test_cron_handle_bot_died_ignored_expired(self):
   self.mock(random, 'getrandbits', lambda _: 0x88)
   data = _gen_request_data(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       scheduling_expiration_secs=600)
   request = task_request.make_request(data)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601)
   self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died())

 def test_cron_handle_bot_died_ignored_expired(self):
   self.mock(random, 'getrandbits', lambda _: 0x88)
   now = utils.utcnow()
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       created_ts=now,
       expiration_ts=now+datetime.timedelta(seconds=600))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601)
   self.assertEqual(
       (['1d69b9f088008811'], 0, 0),
       task_scheduler.cron_handle_bot_died('f.local'))

 def _task_ran_successfully(self):
   """Runs a task successfully and returns the task_id."""
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}, idempotent=True))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   actual_request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(request, actual_request)
   self.assertEqual('localhost', run_result.bot_id)
   self.assertEqual(None, task_to_run.TaskToRun.query().get().queue_number)
   # It's important to terminate the task with success.
   self.assertEqual(
       (True, True),
       task_scheduler.bot_update_task(
           run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
           0.1, None))
   return unicode(run_result.task_id)