Python db.transactional方法代码示例

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def __add_kickoff_task(cls, job_config, mapreduce_spec):
    """Add kickoff task to taskqueue.

    Args:
      job_config: map_job.JobConfig.
      mapreduce_spec: model.MapreduceSpec,
    """
    params = {"mapreduce_id": job_config.job_id}
    # Task is not named so that it can be added within a transaction.
    kickoff_task = taskqueue.Task(
        # TODO(user): Perhaps make this url a computed field of job_config.
        url=job_config._base_path + "/kickoffjob_callback/" + job_config.job_id,
        headers=util._get_task_headers(job_config.job_id),
        params=params)
    if job_config._hooks_cls:
      hooks = job_config._hooks_cls(mapreduce_spec)
      try:
        hooks.enqueue_kickoff_task(kickoff_task, job_config.queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(job_config.queue_name, transactional=True) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def __add_kickoff_task(cls, job_config, mapreduce_spec):
    """Add kickoff task to taskqueue.

    Args:
      job_config: map_job.JobConfig.
      mapreduce_spec: model.MapreduceSpec,
    """
    params = {"mapreduce_id": job_config.job_id}

    kickoff_task = taskqueue.Task(

        url=job_config._base_path + "/kickoffjob_callback/" + job_config.job_id,
        headers=util._get_task_headers(job_config.job_id),
        params=params)
    if job_config._hooks_cls:
      hooks = job_config._hooks_cls(mapreduce_spec)
      try:
        hooks.enqueue_kickoff_task(kickoff_task, job_config.queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(job_config.queue_name, transactional=True) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def schedule(cls, mapreduce_spec):
    """Schedule finalize task.

    Args:
      mapreduce_spec: mapreduce specification as MapreduceSpec.
    """
    task_name = mapreduce_spec.mapreduce_id + "-finalize"
    finalize_task = taskqueue.Task(
        name=task_name,
        url=(mapreduce_spec.params["base_path"] + "/finalizejob_callback/" +
             mapreduce_spec.mapreduce_id),
        params={"mapreduce_id": mapreduce_spec.mapreduce_id},
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id))
    queue_name = util.get_queue_name(None)
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          finalize_task,
                          queue_name,
                          transactional=False):
      try:
        finalize_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        task_name, e.__class__, e) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def testFindAllByMapreduceState(self):
    mr_state = model.MapreduceState.create_new("mapreduce-id")
    mr_state.mapreduce_spec = model.MapreduceSpec(
        "mapreduce", "mapreduce-id",
        model.MapperSpec("handler", "input-reader",
                         {}, shard_count=304).to_json())
    mr_state.put()
    for i in range(304):
      model.ShardState.create_new("mapreduce-id", i).put()
    @db.transactional(xg=False)
    def non_xg_tx():
      # Open a single non-related entity group to ensure
      # find_all_by_mapreduce_state does not attempt to use outer transaction
      mr_state2 = model.MapreduceState.create_new("unrelated-mapreduce-id")
      mr_state2.put()
      shard_states = model.ShardState.find_all_by_mapreduce_state(mr_state)
      for i, ss in enumerate(shard_states):
        self.assertEqual(i, ss.shard_number)
    non_xg_tx() 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def _add_task(cls,
                worker_task,
                mapreduce_spec,
                queue_name):
    """Schedule slice scanning by adding it to the task queue.

    Args:
      worker_task: a model.HugeTask task for slice. This is NOT a taskqueue
        task.
      mapreduce_spec: an instance of model.MapreduceSpec.
      queue_name: Optional queue to run on; uses the current queue of
        execution or the default queue if unspecified.
    """
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_worker_task",
                          worker_task,
                          queue_name):
      try:
        # Not adding transactionally because worker_task has name.
        # Named task is not allowed for transactional add.
        worker_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        worker_task.name,
                        e.__class__,
                        e) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def _add_kickoff_task(cls,
                        base_path,
                        mapreduce_spec,
                        eta,
                        countdown,
                        parent,
                        queue_name,
                        transactional,
                        _app):
    queue_name = queue_name or os.environ.get("HTTP_X_APPENGINE_QUEUENAME",
                                              "default")
    if queue_name[0] == "_":
      # We are currently in some special queue. E.g. __cron.
      queue_name = "default"

    kickoff_params = {"mapreduce_spec": mapreduce_spec.to_json_str()}
    if _app:
      kickoff_params["app"] = _app
    kickoff_worker_task = model.HugeTask(
        url=base_path + "/kickoffjob_callback",
        params=kickoff_params,
        eta=eta,
        countdown=countdown,
        parent=parent)
    hooks = mapreduce_spec.get_hooks()
    if hooks is not None:
      try:
        hooks.enqueue_kickoff_task(kickoff_worker_task, queue_name)
      except NotImplementedError:
        kickoff_worker_task.add(queue_name, transactional=transactional)
    else:
      kickoff_worker_task.add(queue_name) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def submit(cls, job_config, in_xg_transaction=False):
    """Submit the job to run.

    Args:
      job_config: an instance of map_job.MapJobConfig.
      in_xg_transaction: controls what transaction scope to use to start this MR
        job. If True, there has to be an already opened cross-group transaction
        scope. MR will use one entity group from it.
        If False, MR will create an independent transaction to start the job
        regardless of any existing transaction scopes.

    Returns:
      a Job instance representing the submitted job.
    """
    cls.__validate_job_config(job_config)
    mapper_spec = job_config._get_mapper_spec()

    # Create mr spec.
    mapreduce_params = job_config._get_mr_params()
    mapreduce_spec = model.MapreduceSpec(
        job_config.job_name,
        job_config.job_id,
        mapper_spec.to_json(),
        mapreduce_params,
        util._obj_to_path(job_config._hooks_cls))

    # Save states and enqueue task.
    if in_xg_transaction:
      propagation = db.MANDATORY
    else:
      propagation = db.INDEPENDENT

    state = None
    @db.transactional(propagation=propagation)
    def _txn():
      state = cls.__create_and_save_state(job_config, mapreduce_spec)
      cls.__add_kickoff_task(job_config, mapreduce_spec)
      return state

    state = _txn()
    return cls(state) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def begin_abort(self, root_pipeline_key, abort_message):
    """Kicks off the abort process for a root pipeline and all its children.

    Args:
      root_pipeline_key: db.Key of the root pipeline to abort.
      abort_message: Message explaining why the abort happened, only saved
          into the root pipeline.

    Returns:
      True if the abort signal was sent successfully; False otherwise.
    """
    def txn():
      pipeline_record = db.get(root_pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to abort root pipeline ID "%s" but it does not exist.',
            root_pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status == _PipelineRecord.ABORTED:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; already in state: %s',
            root_pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()
      if pipeline_record.abort_requested:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; abort signal already sent.',
            root_pipeline_key.name())
        raise db.Rollback()

      pipeline_record.abort_requested = True
      pipeline_record.abort_message = abort_message
      pipeline_record.put()

      task = taskqueue.Task(
          url=self.fanout_abort_handler_path,
          params=dict(root_pipeline_key=root_pipeline_key))
      task.add(queue_name=self.queue_name, transactional=True)
      return True

    return db.run_in_transaction(txn) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def submit(cls, job_config, in_xg_transaction=False):
    """Submit the job to run.

    Args:
      job_config: an instance of map_job.MapJobConfig.
      in_xg_transaction: controls what transaction scope to use to start this MR
        job. If True, there has to be an already opened cross-group transaction
        scope. MR will use one entity group from it.
        If False, MR will create an independent transaction to start the job
        regardless of any existing transaction scopes.

    Returns:
      a Job instance representing the submitted job.
    """
    cls.__validate_job_config(job_config)
    mapper_spec = job_config._get_mapper_spec()


    mapreduce_params = job_config._get_mr_params()
    mapreduce_spec = model.MapreduceSpec(
        job_config.job_name,
        job_config.job_id,
        mapper_spec.to_json(),
        mapreduce_params,
        util._obj_to_path(job_config._hooks_cls))


    if in_xg_transaction:
      propagation = db.MANDATORY
    else:
      propagation = db.INDEPENDENT

    state = None
    @db.transactional(propagation=propagation)
    def _txn():
      state = cls.__create_and_save_state(job_config, mapreduce_spec)
      cls.__add_kickoff_task(job_config, mapreduce_spec)
      return state

    state = _txn()
    return cls(state) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def _run_task_hook(hooks, method, task, queue_name, transactional=False):
  """Invokes hooks.method(task, queue_name, transactional).

  Args:
    hooks: A hooks.Hooks instance or None.
    method: The name of the method to invoke on the hooks class e.g.
        "enqueue_kickoff_task".
    task: The taskqueue.Task to pass to the hook method.
    queue_name: The name of the queue to pass to the hook method.
    transactional: Whether the task should be added transactionally.

  Returns:
    True if the hooks.Hooks instance handled the method, False otherwise.
  Raises:
    Exception: if we try to add a named task transactionally.
  """





  if task.name is not None and transactional:
    raise Exception("Named tasks cannot be added transactionally.")

  if hooks is not None:
    try:
      getattr(hooks, method)(task, queue_name, transactional)
    except NotImplementedError:

      return False

    return True
  return False 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def _add_task(cls,
                worker_task,
                mapreduce_spec,
                queue_name,
                transactional=False):
    """Schedule slice scanning by adding it to the task queue.

    Args:
      worker_task: a model.HugeTask task for slice. This is NOT a taskqueue
        task.
      mapreduce_spec: an instance of model.MapreduceSpec.
      queue_name: Optional queue to run on; uses the current queue of
        execution or the default queue if unspecified.
      transactional: If the task should be part of an existing transaction.
    """
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_worker_task",
                          worker_task,
                          queue_name,
                          transactional=transactional):
      try:
        worker_task.add(queue_name, transactional=transactional)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        worker_task.name,
                        e.__class__,
                        e) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def reschedule(cls,
                 mapreduce_state,
                 mapreduce_spec,
                 serial_id,
                 queue_name=None):
    """Schedule new update status callback task.

    Args:
      mapreduce_state: mapreduce state as model.MapreduceState
      mapreduce_spec: mapreduce specification as MapreduceSpec.
      serial_id: id of the invocation as int.
      queue_name: The queue to schedule this task on. Will use the current
        queue of execution if not supplied.
    """

    task_name = ControllerCallbackHandler.get_task_name(
        mapreduce_spec, serial_id)
    task_params = ControllerCallbackHandler.controller_parameters(
        mapreduce_spec, serial_id)
    if not queue_name:
      queue_name = os.environ.get("HTTP_X_APPENGINE_QUEUENAME", "default")

    controller_callback_task = model.HugeTask(
        url=(mapreduce_spec.params["base_path"] + "/controller_callback/" +
             mapreduce_spec.mapreduce_id),
        name=task_name, params=task_params,
        countdown=parameters.config._CONTROLLER_PERIOD_SEC,
        parent=mapreduce_state,
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id))

    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          controller_callback_task,
                          queue_name,
                          transactional=False):
      try:
        controller_callback_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r with params %r already exists. %s: %s",
                        task_name, task_params, e.__class__, e) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def _add_kickoff_task(cls,
                        base_path,
                        mapreduce_spec,
                        eta,
                        countdown,
                        queue_name):
    """Enqueues a new kickoff task."""
    params = {"mapreduce_id": mapreduce_spec.mapreduce_id}







    mapreduce_target = mapreduce_spec.params.get(
        model.MapreduceSpec.PARAM_MAPREDUCE_TARGET)
    headers = util._get_task_headers(mapreduce_spec.mapreduce_id,
                                     set_host_header=(mapreduce_target is None))


    kickoff_task = taskqueue.Task(
        url=base_path + "/kickoffjob_callback/" + mapreduce_spec.mapreduce_id,
        headers=headers,
        params=params,
        eta=eta,
        countdown=countdown,
        target=mapreduce_target)
    hooks = mapreduce_spec.get_hooks()
    if hooks is not None:
      try:
        hooks.enqueue_kickoff_task(kickoff_task, queue_name, True)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(queue_name, transactional=True) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def testStartMapTransactional(self):
    """Test start_map function.

    Most of start_map functionality is already tested by handlers_test.
    Just a smoke test is enough.
    """
    TestEntity().put()

    shard_count = 4
    mapreduce_id = ""

    @db.transactional(xg=True)
    def tx():
      some_entity = TestEntity()
      some_entity.put()
      return control.start_map(
          "test_map",
          __name__ + ".test_handler",
          "mapreduce.input_readers.DatastoreInputReader",
          {
              "entity_kind": __name__ + "." + TestEntity.__name__,
          },
          shard_count,
          mapreduce_parameters={"foo": "bar"},
          base_path="/mapreduce_base_path",
          queue_name=self.QUEUE_NAME,
          in_xg_transaction=True)
    mapreduce_id = tx()
    self.validate_map_started(mapreduce_id) 

# 需要导入模块: from google.appengine.ext import db [as 别名]
# 或者: from google.appengine.ext.db import transactional [as 别名]
def testStartMapTransactional_HugePayload(self):
    """Test start_map function.

    Most of start_map functionality is already tested by handlers_test.
    Just a smoke test is enough.
    """
    TestEntity().put()

    shard_count = 4
    mapreduce_id = ""

    @db.transactional(xg=True)
    def tx():
      some_entity = TestEntity()
      some_entity.put()
      return control.start_map(
          "test_map",
          __name__ + ".test_handler",
          "mapreduce.input_readers.DatastoreInputReader",
          {
              "entity_kind": __name__ + "." + TestEntity.__name__,
              "huge_parameter": random_string(900000)
          },
          shard_count,
          mapreduce_parameters={"foo": "bar"},
          base_path="/mapreduce_base_path",
          queue_name=self.QUEUE_NAME,
          in_xg_transaction=True)
    mapreduce_id = tx()
    self.validate_map_started(mapreduce_id)