Python control.start_map函数代码示例

 def get(self):
     mapreduce_params = {
           'entity_kind': 'models.Topic',
     }
     
     control.start_map("DeleteOldUpdates", "mapjob.keep_thirty_updates", "mapreduce.input_readers.DatastoreInputReader", mapreduce_params, 2)
     self.response.out.write("ok")

 def get(self):
     # TODO(qyearsley): Add test coverage. See catapult:#1346.
     name = "Update test deprecation status."
     handler = "dashboard.mr.DeprecateTestsMapper"
     reader = "mapreduce.input_readers.DatastoreInputReader"
     mapper_parameters = {"entity_kind": ("dashboard.models.graph_data.TestMetadata"), "filters": []}
     mr_control.start_map(name, handler, reader, mapper_parameters)

  def _run_test(self, num_shards, num_files):
    bucket_name = "testing"
    object_prefix = "file-"
    job_name = "test_map"
    input_class = (input_readers.__name__ + "." +
                   input_readers._GoogleCloudStorageInputReader.__name__)

    expected_content = self.create_test_content(bucket_name,
                                                object_prefix,
                                                num_files)

    control.start_map(
        job_name,
        __name__ + "." + "_input_reader_memory_mapper",
        input_class,
        {
            "input_reader": {
                "bucket_name": bucket_name,
                "objects": [object_prefix + "*"]
            },
        },
        shard_count=num_shards)

    test_support.execute_until_empty(self.taskqueue)
    self.assertEqual(expected_content.sort(), _memory_mapper_data.sort())

    def testHandlerSerialization(self):
        """Test serializable handler works with MR and shard retry."""
        entity_count = 10

        for _ in range(entity_count):
            TestEntity(int_property=-1).put()

        # Force handler to serialize on every call.
        handlers._SLICE_DURATION_SEC = 0

        control.start_map(
            "test_map",
            __name__ + ".SerializableHandler",
            "mapreduce.input_readers.DatastoreInputReader",
            {"entity_kind": __name__ + "." + TestEntity.__name__},
            shard_count=1,
            base_path="/mapreduce_base_path",
        )

        task_run_counts = test_support.execute_until_empty(self.taskqueue)
        self.assertEquals(
            task_run_counts[handlers.MapperWorkerCallbackHandler],
            entity_count + 1 + SerializableHandler.TASKS_CONSUMED_BY_RETRY,
        )
        vals = [e.int_property for e in TestEntity.all()]
        vals.sort()
        # SerializableHandler updates int_property to be incremental from 0 to 9.
        self.assertEquals(range(10), vals)

def start_map(
    fbl,
    name,
    handler_spec,
    entity_kind,
    filters=None,
    handle_batch_size=None,
    output_writer_spec=None,
    output_writer=None,
    queue='slow-queue',
    extra_mapper_params=None,
    randomize_tokens=True,
):
    filters = filters or []
    output_writer = output_writer or {}
    extra_mapper_params = extra_mapper_params or {}
    mapper_params = {
        'entity_kind': entity_kind,
        'handle_batch_size': handle_batch_size,
        'filters': filters,
        'output_writer': output_writer,
    }
    mapper_params.update(get_fblookup_params(fbl, randomize_tokens=randomize_tokens))
    mapper_params.update(extra_mapper_params)
    control.start_map(
        name=name,
        reader_spec='mapreduce.input_readers.DatastoreInputReader',
        handler_spec=handler_spec,
        output_writer_spec=output_writer_spec,
        shard_count=16,  # since we want to stick it in the slow-queue, and don't care how fast it executes
        queue_name=queue,
        mapper_parameters=mapper_params,
    )

  def testHandlerSerialization(self):
    """Test serializable handler works with MR and shard retry."""
    entity_count = 10

    for _ in range(entity_count):
      TestEntity(int_property=-1).put()

    # Force handler to serialize on every call.
    parameters.config._SLICE_DURATION_SEC = 0

    control.start_map(
        "test_map",
        __name__ + ".SerializableHandler",
        "mapreduce.input_readers.DatastoreInputReader",
        {
            "entity_kind": __name__ + "." + TestEntity.__name__,
        },
        shard_count=1,
        base_path="/mapreduce_base_path")

    task_run_counts = test_support.execute_until_empty(self.taskqueue)
    self.assertEquals(
        task_run_counts[handlers.MapperWorkerCallbackHandler],
        # Shard retries + one per entity + one to exhaust input reader + one for
        # finalization.
        SerializableHandler.FAILURES_INDUCED_BY_INSTANCE + entity_count + 1 + 1)
    vals = [e.int_property for e in TestEntity.all()]
    vals.sort()
    # SerializableHandler updates int_property to be incremental from 0 to 9.
    self.assertEquals(range(10), vals)

 def get(self):
     control.start_map(
         name='Compute User-Event Stats',
         reader_spec='mapreduce.input_readers.DatastoreInputReader',
         handler_spec='users.user_event_tasks.map_compute_user_stats',
         mapper_parameters={'entity_kind': 'users.users.User'},
         queue_name='fast-queue',
         shard_count=5,
     )

 def get(self):
   name = 'Update anomalies with units.'
   handler = ('dashboard.mr.StoreUnitsInAnomalyEntity')
   reader = 'mapreduce.input_readers.DatastoreInputReader'
   mapper_parameters = {
       'entity_kind': ('dashboard.models.graph_data.Anomaly'),
       'filters': [],
   }
   mr_control.start_map(name, handler, reader, mapper_parameters)

 def get(self):
     table = self.request.get('table') # users.users.User or events.eventdata.DBEvent or ...
     control.start_map(
         name='Resave %s' % table,
         reader_spec='mapreduce.input_readers.DatastoreInputReader',
         handler_spec='servlets.tools.resave_table',
         mapper_parameters={
             'entity_kind': table,
         },
     )

 def get(self):
     control.start_map(
         name='Fixup Events',
         reader_spec='mapreduce.input_readers.DatastoreInputReader',
         handler_spec='dancedeets.servlets.tools.resave_object',
         mapper_parameters={
             'entity_kind': 'dancedeets.events.eventdata.DBEvent',
         },
         shard_count=16,
     )

 def get(self):
     # TODO(qyearsley): Add test coverage. See catapult:#1346.
     name = 'Update test deprecation status.'
     handler = ('dashboard.mr.DeprecateTestsMapper')
     reader = 'mapreduce.input_readers.DatastoreInputReader'
     mapper_parameters = {
         'entity_kind': ('dashboard.models.graph_data.TestMetadata'),
         'filters': [('has_rows', '=', True), ('deprecated', '=', False)],
     }
     mr_control.start_map(name, handler, reader, mapper_parameters)

def delete_all_questions():
    logging.info("Delete all existing questions")
    ctrl.start_map("Delete all Question entities", 
                   'locql.delete_entity', 
                   'mapreduce.input_readers.DatastoreKeyInputReader', 
                   {'entity_kind': 'locql.Question'})
    ctrl.start_map("Delete all TermStat entities", 
                   'locql.delete_entity', 
                   'mapreduce.input_readers.DatastoreKeyInputReader',
                   {'entity_kind': 'locql.TermStat'})

def begin_user_ranking_calculations():
    control.start_map(
        name='Compute City Rankings by Users',
        reader_spec='mapreduce.input_readers.DatastoreInputReader',
        handler_spec='dancedeets.rankings.rankings.count_user_for_city',
        mapper_parameters={'entity_kind': 'dancedeets.users.users.User'},
        queue_name='fast-queue',
        shard_count=16,
        _app=USER_FOR_CITY_RANKING,
    )
    _compute_summary(expiry=5 * 60)  # 5 minutes

 def get(self):
   mr_control.start_map(
    self.request.get("name"),
    self.request.get("reader_spec", "your_mapreduce.map"),
    self.request.get("reader_parameters",
                     "mapreduce.input_readers.DatastoreInputReader"),
    { "entity_kind": self.request.get("entity_kind", "models.YourModel"),
      "processing_rate": int(self.request.get("processing_rate", 100)) },
    mapreduce_parameters={"done_callback": self.request.get("done_callback",
                                                            None) } )
   self.response.out.write("MapReduce scheduled");

def start_map(name, params=None, eta=None, countdown=None):
  for config in status.MapReduceYaml.to_dict(status.get_mapreduce_yaml()):
    if config.get('name') == name:
      break

  # Add the mapreduce specific parameters to the params dictionary
  config['mapper_params'].update(params if params else {})

  control.start_map(config['name'], config['mapper_handler'],
                    config['mapper_input_reader'], config['mapper_params'],
                    eta=eta,
                    countdown=countdown)