Python task_request.make_request函数代码示例

 def test_make_request_parent(self):
   parent = task_request.make_request(_gen_request_data())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   child = task_request.make_request(
       _gen_request_data(parent_task_id=parent_id))
   self.assertEqual(parent_id, child.parent_task_id)

 def test_make_request_clone(self):
   # Compare with test_make_request().
   parent = task_request.make_request(_gen_request_data())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   data = _gen_request_data(
       properties=dict(idempotent=True), parent_task_id=parent_id)
   request = task_request.make_request_clone(task_request.make_request(data))
   # Differences from make_request() are:
   # - idempotent was reset to False.
   # - parent_task_id was reset to None.
   expected_properties = {
     'commands': [[u'command1', u'arg1']],
     'data': [
       # Items were sorted.
       [u'http://localhost/bar', u'bar.zip'],
       [u'http://localhost/foo', u'foo.zip'],
     ],
     'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
     'env': {u'foo': u'bar', u'joe': u'2'},
     'execution_timeout_secs': 30,
     'extra_args': None,
     'grace_period_secs': 30,
     'idempotent': False,
     'io_timeout_secs': None,
     'isolated': None,
     'isolatedserver': None,
     'namespace': None,
   }
   # Differences from make_request() are:
   # - parent_task_id was reset to None.
   # - tag 'user:' was replaced
   # - user was replaced.
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name (Retry #1)',
     'parent_task_id': None,
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': None,
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'priority:49',
       u'tag:1',
       u'user:[email protected]',
     ],
     'user': u'[email protected]',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == deadline_to_run.
   created = actual.pop('created_ts')
   expiration = actual.pop('expiration_ts')
   self.assertEqual(
       int(round((expiration - created).total_seconds())),
       data['scheduling_expiration_secs'])
   self.assertEqual(expected_request, actual)
   self.assertEqual(
       data['scheduling_expiration_secs'], request.scheduling_expiration_secs)

    def test_new_task_to_run(self):
        self.mock(random, "getrandbits", lambda _: 0x12)
        request_dimensions = {u"OS": u"Windows-3.1.1"}
        data = _gen_request_data(
            properties={
                "commands": [[u"command1", u"arg1"]],
                "data": [[u"http://localhost/foo", u"foo.zip"]],
                "dimensions": request_dimensions,
                "env": {u"foo": u"bar"},
                "execution_timeout_secs": 30,
            },
            priority=20,
            scheduling_expiration_secs=31,
        )
        task_to_run.new_task_to_run(task_request.make_request(data)).put()

        # Create a second with higher priority.
        self.mock(random, "getrandbits", lambda _: 0x23)
        data = _gen_request_data(
            properties={
                "commands": [[u"command1", u"arg1"]],
                "data": [[u"http://localhost/foo", u"foo.zip"]],
                "dimensions": request_dimensions,
                "env": {u"foo": u"bar"},
                "execution_timeout_secs": 30,
            },
            priority=10,
            scheduling_expiration_secs=31,
        )
        task_to_run.new_task_to_run(task_request.make_request(data)).put()

        expected = [
            {
                "dimensions_hash": _hash_dimensions(request_dimensions),
                "expiration_ts": self.now + datetime.timedelta(seconds=31),
                "request_key": "0x7e296460f77ffdce",
                # Lower priority value means higher priority.
                "queue_number": "0x00060dc5849f1346",
            },
            {
                "dimensions_hash": _hash_dimensions(request_dimensions),
                "expiration_ts": self.now + datetime.timedelta(seconds=31),
                "request_key": "0x7e296460f77ffede",
                "queue_number": "0x00072c96fd65d346",
            },
        ]

        def flatten(i):
            out = _task_to_run_to_dict(i)
            out["request_key"] = "0x%016x" % i.request_key.integer_id()
            return out

        # Warning: Ordering by key doesn't work because of TaskToRunShard; e.g.
        # the entity key ordering DOES NOT correlate with .queue_number
        # Ensure they come out in expected order.
        q = task_to_run.TaskToRun.query().order(task_to_run.TaskToRun.queue_number)
        self.assertEqual(expected, map(flatten, q.fetch()))

 def test_different(self):
     # Two TestRequest with different properties.
     request_1 = task_request.make_request(
         _gen_request(properties=dict(execution_timeout_secs=30, idempotent=True)), True
     )
     request_2 = task_request.make_request(
         _gen_request(properties=dict(execution_timeout_secs=129, idempotent=True)), True
     )
     self.assertNotEqual(request_1.properties.properties_hash, request_2.properties.properties_hash)

  def test_new_task_to_run(self):
    self.mock(random, 'getrandbits', lambda _: 0x12)
    request_dimensions = {u'OS': u'Windows-3.1.1', u'pool': u'default'}
    now = utils.utcnow()
    data = _gen_request(
        properties={
          'command': [u'command1', u'arg1'],
          'dimensions': request_dimensions,
          'env': {u'foo': u'bar'},
          'execution_timeout_secs': 30,
        },
        priority=20,
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=31))
    task_to_run.new_task_to_run(task_request.make_request(data, True)).put()

    # Create a second with higher priority.
    self.mock(random, 'getrandbits', lambda _: 0x23)
    data = _gen_request(
        properties={
          'command': [u'command1', u'arg1'],
          'dimensions': request_dimensions,
          'env': {u'foo': u'bar'},
          'execution_timeout_secs': 30,
        },
        priority=10,
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=31))
    task_to_run.new_task_to_run(task_request.make_request(data, True)).put()

    expected = [
      {
        'dimensions_hash': _hash_dimensions(request_dimensions),
        'expiration_ts': self.now + datetime.timedelta(seconds=31),
        'request_key': '0x7e296460f77ffdce',
        # Lower priority value means higher priority.
        'queue_number': '0x00060dc5849f1346',
      },
      {
        'dimensions_hash': _hash_dimensions(request_dimensions),
        'expiration_ts': self.now + datetime.timedelta(seconds=31),
        'request_key': '0x7e296460f77ffede',
        'queue_number': '0x00072c96fd65d346',
      },
    ]

    def flatten(i):
      out = _task_to_run_to_dict(i)
      out['request_key'] = '0x%016x' % i.request_key.integer_id()
      return out

    # Warning: Ordering by key doesn't work because of TaskToRunShard; e.g.
    # the entity key ordering DOES NOT correlate with .queue_number
    # Ensure they come out in expected order.
    q = task_to_run.TaskToRun.query().order(task_to_run.TaskToRun.queue_number)
    self.assertEqual(expected, map(flatten, q.fetch()))

 def test_make_request_isolated(self):
   parent = task_request.make_request(
       _gen_request(
           properties={
             'commands': None,
             'data': None,
             'inputs_ref': {
               'isolated': '0123456789012345678901234567890123456789',
               'isolatedserver': 'http://localhost:1',
               'namespace': 'default-gzip',
             },
           }),
       True)
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   request = task_request.make_request(
       _gen_request(properties={'idempotent':True}, parent_task_id=parent_id),
       True)
   expected_properties = {
     'commands': [[u'command1', u'arg1']],
     'data': [
       # Items were sorted.
       [u'http://localhost/bar', u'bar.zip'],
       [u'http://localhost/foo', u'foo.zip'],
     ],
     'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
     'env': {u'foo': u'bar', u'joe': u'2'},
     'extra_args': [],
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'inputs_ref': None,
     'io_timeout_secs': None,
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': 'b45f6f868f9227c3035cd82b4a5b0360f5ce6f61',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'priority:49',
       u'tag:1',
       u'user:Jesus',
     ],
     'user': u'Jesus',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   actual.pop('created_ts')
   actual.pop('expiration_ts')
   self.assertEqual(expected_request, actual)
   self.assertEqual(30, request.expiration_secs)

 def test_make_request_isolated(self):
     parent = task_request.make_request(
         _gen_request(
             properties={
                 "commands": None,
                 "data": None,
                 "inputs_ref": {
                     "isolated": "0123456789012345678901234567890123456789",
                     "isolatedserver": "http://localhost:1",
                     "namespace": "default-gzip",
                 },
             }
         ),
         True,
     )
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + "1"
     request = task_request.make_request(
         _gen_request(properties={"idempotent": True}, parent_task_id=parent_id), True
     )
     expected_properties = {
         "commands": [[u"command1", u"arg1"]],
         "data": [
             # Items were sorted.
             [u"http://localhost/bar", u"bar.zip"],
             [u"http://localhost/foo", u"foo.zip"],
         ],
         "dimensions": {u"OS": u"Windows-3.1.1", u"hostname": u"localhost"},
         "env": {u"foo": u"bar", u"joe": u"2"},
         "extra_args": [],
         "execution_timeout_secs": 30,
         "grace_period_secs": 30,
         "idempotent": True,
         "inputs_ref": None,
         "io_timeout_secs": None,
     }
     expected_request = {
         "authenticated": auth_testing.DEFAULT_MOCKED_IDENTITY,
         "name": u"Request name",
         "parent_task_id": unicode(parent_id),
         "priority": 49,
         "properties": expected_properties,
         "properties_hash": "b45f6f868f9227c3035cd82b4a5b0360f5ce6f61",
         "pubsub_topic": None,
         "pubsub_userdata": None,
         "tags": [u"OS:Windows-3.1.1", u"hostname:localhost", u"priority:49", u"tag:1", u"user:Jesus"],
         "user": u"Jesus",
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == scheduling_expiration_secs.
     actual.pop("created_ts")
     actual.pop("expiration_ts")
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)

 def test_make_request(self):
   # Compare with test_make_request_clone().
   parent = task_request.make_request(_gen_request_data())
   # Hack: Would need to know about TaskResultSummary.
   parent_id = task_pack.pack_request_key(parent.key) + '1'
   data = _gen_request_data(
       properties=dict(idempotent=True), parent_task_id=parent_id)
   request = task_request.make_request(data)
   expected_properties = {
     'commands': [[u'command1', u'arg1']],
     'data': [
       # Items were sorted.
       [u'http://localhost/bar', u'bar.zip'],
       [u'http://localhost/foo', u'foo.zip'],
     ],
     'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
     'env': {u'foo': u'bar', u'joe': u'2'},
     'extra_args': None,
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': True,
     'io_timeout_secs': None,
     'isolated': None,
     'isolatedserver': None,
     'namespace': None,
   }
   expected_request = {
     'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
     'name': u'Request name',
     'parent_task_id': unicode(parent_id),
     'priority': 49,
     'properties': expected_properties,
     'properties_hash': 'e7276ca9999756de386d26d39ae648e641ae1eac',
     'tags': [
       u'OS:Windows-3.1.1',
       u'hostname:localhost',
       u'priority:49',
       u'tag:1',
       u'user:Jesus',
     ],
     'user': u'Jesus',
   }
   actual = request.to_dict()
   # expiration_ts - created_ts == scheduling_expiration_secs.
   created = actual.pop('created_ts')
   expiration = actual.pop('expiration_ts')
   self.assertEqual(
       int(round((expiration - created).total_seconds())),
       data['scheduling_expiration_secs'])
   self.assertEqual(expected_request, actual)
   self.assertEqual(
       data['scheduling_expiration_secs'], request.scheduling_expiration_secs)

 def test_duped(self):
   # Two TestRequest with the same properties.
   request_1 = task_request.make_request(
       _gen_request_data(properties=dict(idempotent=True)))
   request_2 = task_request.make_request(
       _gen_request_data(
           name='Other', user='Other', priority=201,
           scheduling_expiration_secs=129, tags=['tag:2'],
           properties=dict(idempotent=True)))
   self.assertEqual(
       request_1.properties.properties_hash,
       request_2.properties.properties_hash)
   self.assertTrue(request_1.properties.properties_hash)

 def test_make_request_clone(self):
     # Compare with test_make_request().
     parent = task_request.make_request(_gen_request(), True)
     # Hack: Would need to know about TaskResultSummary.
     parent_id = task_pack.pack_request_key(parent.key) + "1"
     data = _gen_request(properties=dict(idempotent=True), parent_task_id=parent_id)
     request = task_request.make_request_clone(task_request.make_request(data, True))
     # Differences from make_request() are:
     # - idempotent was reset to False.
     # - parent_task_id was reset to None.
     expected_properties = {
         "commands": [[u"command1", u"arg1"]],
         "data": [
             # Items were sorted.
             [u"http://localhost/bar", u"bar.zip"],
             [u"http://localhost/foo", u"foo.zip"],
         ],
         "dimensions": {u"OS": u"Windows-3.1.1", u"hostname": u"localhost"},
         "env": {u"foo": u"bar", u"joe": u"2"},
         "execution_timeout_secs": 30,
         "extra_args": [],
         "grace_period_secs": 30,
         "idempotent": False,
         "inputs_ref": None,
         "io_timeout_secs": None,
     }
     # Differences from make_request() are:
     # - parent_task_id was reset to None.
     # - tag 'user:' was replaced
     # - user was replaced.
     expected_request = {
         "authenticated": auth_testing.DEFAULT_MOCKED_IDENTITY,
         "name": u"Request name (Retry #1)",
         "parent_task_id": None,
         "priority": 49,
         "properties": expected_properties,
         "properties_hash": None,
         "pubsub_topic": None,
         "pubsub_userdata": None,
         "tags": [u"OS:Windows-3.1.1", u"hostname:localhost", u"priority:49", u"tag:1", u"user:[email protected]"],
         "user": u"[email protected]",
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == deadline_to_run.
     actual.pop("created_ts")
     actual.pop("expiration_ts")
     self.assertEqual(expected_request, actual)
     self.assertEqual(30, request.expiration_secs)

 def test_request_to_task_to_run_key(self):
   self.mock(random, 'getrandbits', lambda _: 0x88)
   request = task_request.make_request(_gen_request(), True)
   # Ensures that the hash value is constant for the same input.
   self.assertEqual(
       ndb.Key('TaskRequest', 0x7e296460f77ff77e, 'TaskToRun', 3420117132),
       task_to_run.request_to_task_to_run_key(request))

  def new(self, request):
    """Creates a new task.

    The task will be enqueued in the tasks list and will be executed at the
    earliest opportunity by a bot that has at least the dimensions as described
    in the task request.
    """
    logging.info('%s', request)
    try:
      request = message_conversion.new_task_request_from_rpc(
          request, utils.utcnow())
      posted_request = task_request.make_request(request, acl.is_bot_or_admin())
    except (datastore_errors.BadValueError, TypeError, ValueError) as e:
      raise endpoints.BadRequestException(e.message)

    result_summary = task_scheduler.schedule_request(posted_request)

    previous_result = None
    if result_summary.deduped_from:
      previous_result = message_conversion.task_result_to_rpc(result_summary)

    return swarming_rpcs.TaskRequestMetadata(
        request=message_conversion.task_request_to_rpc(posted_request),
        task_id=task_pack.pack_result_summary_key(result_summary.key),
        task_result=previous_result)

  def test_bot_kill_task(self):
    self.mock(random, 'getrandbits', lambda _: 0x88)
    dimensions = {u'OS': u'Windows-3.1.1'}
    request = task_request.make_request(
        _gen_request(properties={'dimensions': dimensions}), True)
    result_summary = task_scheduler.schedule_request(request)
    reaped_request, run_result = task_scheduler.bot_reap_task(
        {'OS': 'Windows-3.1.1'}, 'localhost', 'abc')

    self.assertEqual(
        None, task_scheduler.bot_kill_task(run_result.key, 'localhost'))
    expected = {
      'abandoned_ts': self.now,
      'bot_dimensions': dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': True,
      'modified_ts': self.now,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': State.BOT_DIED,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, result_summary.key.get().to_dict())
    expected = {
      'abandoned_ts': self.now,
      'bot_dimensions': dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'cost_usd': 0.,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008811',
      'internal_failure': True,
      'modified_ts': self.now,
      'outputs_ref': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': State.BOT_DIED,
      'try_number': 1,
    }
    self.assertEqual(expected, run_result.key.get().to_dict())

 def test_request_to_task_to_run_key(self):
     self.mock(random, "getrandbits", lambda _: 0x88)
     request = task_request.make_request(_gen_request_data())
     self.assertEqual(
         ndb.Key("TaskRequest", 0x7E296460F77FF77E, "TaskToRun", 2471203225),
         task_to_run.request_to_task_to_run_key(request),
     )

  def test_set_from_run_result_two_tries(self):
    request = task_request.make_request(_gen_request(), True)
    result_summary = task_result.new_result_summary(request)
    run_result_1 = task_result.new_run_result(
        request, 1, 'localhost', 'abc', {})
    run_result_2 = task_result.new_run_result(
        request, 2, 'localhost', 'abc', {})
    self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
    run_result_2.modified_ts = utils.utcnow()
    result_summary.modified_ts = utils.utcnow()
    ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))

    self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
    run_result_1.modified_ts = utils.utcnow()
    result_summary.set_from_run_result(run_result_1, request)
    ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1)))

    result_summary = result_summary.key.get()
    self.assertFalse(result_summary.need_update_from_run_result(run_result_1))

    self.assertTrue(result_summary.need_update_from_run_result(run_result_2))
    run_result_2.modified_ts = utils.utcnow()
    result_summary.set_from_run_result(run_result_2, request)
    ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
    result_summary = result_summary.key.get()

    self.assertEqual(2, result_summary.try_number)
    self.assertFalse(result_summary.need_update_from_run_result(run_result_1))