Python ray.put方法代码示例

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def full_grad(theta):
    theta_id = ray.put(theta)
    grad_ids = [grad.remote(theta_id, xs_id, ys_id) for (xs_id, ys_id) in batch_ids]
    return sum(ray.get(grad_ids)).astype("float64") # This conversion is necessary for use with fmin_l_bfgs_b.

  # From the perspective of scipy.optimize.fmin_l_bfgs_b, full_loss is simply a
  # function which takes some parameters theta, and computes a loss. Similarly,
  # full_grad is a function which takes some parameters theta, and computes the
  # gradient of the loss. Internally, these functions use Ray to distribute the
  # computation of the loss and the gradient over the data that is represented
  # by the remote object IDs x_batches and y_batches and which is potentially
  # distributed over a cluster. However, these details are hidden from
  # scipy.optimize.fmin_l_bfgs_b, which simply uses it to run the L-BFGS
  # algorithm.

  # Load the mnist data and turn the data into remote objects. 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def testRegisterClass(self):
    ray.init(start_ray_local=True, num_workers=0)

    # Check that putting an object of a class that has not been registered
    # throws an exception.
    class TempClass(object):
      pass
    self.assertRaises(Exception, lambda : ray.put(Foo))
    # Check that registering a class that Ray cannot serialize efficiently
    # raises an exception.
    self.assertRaises(Exception, lambda : ray.register_class(type(True)))
    # Check that registering the same class with pickle works.
    ray.register_class(type(float), pickle=True)
    self.assertEqual(ray.get(ray.put(float)), float)

    ray.worker.cleanup() 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def testPythonMode(self):
    reload(test_functions)
    ray.init(start_ray_local=True, driver_mode=ray.PYTHON_MODE)

    @ray.remote
    def f():
      return np.ones([3, 4, 5])
    xref = f.remote()
    assert_equal(xref, np.ones([3, 4, 5])) # remote functions should return by value
    assert_equal(xref, ray.get(xref)) # ray.get should be the identity
    y = np.random.normal(size=[11, 12])
    assert_equal(y, ray.put(y)) # ray.put should be the identity

    # make sure objects are immutable, this example is why we need to copy
    # arguments before passing them into remote functions in python mode
    aref = test_functions.python_mode_f.remote()
    assert_equal(aref, np.array([0, 0]))
    bref = test_functions.python_mode_g.remote(aref)
    assert_equal(aref, np.array([0, 0])) # python_mode_g should not mutate aref
    assert_equal(bref, np.array([1, 0]))

    ray.worker.cleanup() 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_transfer_performance(benchmark, ray_start_cluster_head, object_number,
                              data_size):
    cluster = ray_start_cluster_head
    cluster.add_node(resources={"my_resource": 1}, object_store_memory=10**9)

    @ray.remote(resources={"my_resource": 1})
    class ObjectActor:
        def f(self, object_ids):
            ray.get(object_ids)

    # setup remote actor
    actor = ObjectActor.remote()
    actor.f.remote([])

    data = bytes(1) * data_size
    object_ids = [ray.put(data) for _ in range(object_number)]

    benchmark(benchmark_transfer_object, actor, object_ids) 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_load_balancing_with_dependencies(ray_start_cluster):
    # This test ensures that tasks are being assigned to all raylets in a
    # roughly equal manner even when the tasks have dependencies.
    cluster = ray_start_cluster
    num_nodes = 3
    for _ in range(num_nodes):
        cluster.add_node(num_cpus=1)
    ray.init(address=cluster.address)

    @ray.remote
    def f(x):
        time.sleep(0.010)
        return ray.worker.global_worker.node.unique_id

    # This object will be local to one of the raylets. Make sure
    # this doesn't prevent tasks from being scheduled on other raylets.
    x = ray.put(np.zeros(1000000))

    attempt_to_load_balance(f, [x], 100, num_nodes, 25) 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_put_pins_object(ray_start_object_store_memory):
    x_id = ray.put("HI")
    x_binary = x_id.binary()
    assert ray.get(ray.ObjectID(x_binary)) == "HI"

    # x cannot be evicted since x_id pins it
    for _ in range(10):
        ray.put(np.zeros(10 * 1024 * 1024))
    assert ray.get(x_id) == "HI"
    assert ray.get(ray.ObjectID(x_binary)) == "HI"

    # now it can be evicted since x_id pins it but x_binary does not
    del x_id
    for _ in range(10):
        ray.put(np.zeros(10 * 1024 * 1024))
    assert not ray.worker.global_worker.core_worker.object_exists(
        ray.ObjectID(x_binary))

    # weakref put
    y_id = ray.put("HI", weakref=True)
    for _ in range(10):
        ray.put(np.zeros(10 * 1024 * 1024))
    with pytest.raises(ray.exceptions.UnreconstructableError):
        ray.get(y_id) 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def _run(self, driver_quota, a_quota, b_quota):
        print("*** Testing ***", driver_quota, a_quota, b_quota)
        try:
            ray.init(
                num_cpus=1,
                object_store_memory=300 * MB,
                driver_object_store_memory=driver_quota)
            z = ray.put("hi", weakref=True)
            a = LightActor._remote(object_store_memory=a_quota)
            b = GreedyActor._remote(object_store_memory=b_quota)
            for _ in range(5):
                r_a = a.sample.remote()
                for _ in range(20):
                    new_oid = b.sample.remote()
                    ray.get(new_oid)
                ray.get(r_a)
            ray.get(z)
        except Exception as e:
            print("Raised exception", type(e), e)
            raise e
        finally:
            print(ray.worker.global_worker.core_worker.
                  dump_object_store_memory_usage())
            ray.shutdown() 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_asyncio_actor_async_get(ray_start_regular_shared):
    @ray.remote
    def remote_task():
        return 1

    @ray.remote
    class AsyncGetter:
        async def get(self):
            return await remote_task.remote()

        async def plasma_get(self, plasma_object):
            return await plasma_object[0]

    plasma_object = ray.put(2)
    getter = AsyncGetter.remote()
    assert ray.get(getter.get.remote()) == 1
    assert ray.get(getter.plasma_get.remote([plasma_object])) == 2 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_multi_node_stats(shutdown_only):
    cluster = Cluster()
    for _ in range(2):
        cluster.add_node(num_cpus=1)

    ray.init(address=cluster.address)

    @ray.remote(num_cpus=1)
    class Actor:
        def __init__(self):
            self.ref = ray.put(np.zeros(100000))

        def ping(self):
            pass

    # Each actor will be on a different node.
    a = Actor.remote()
    b = Actor.remote()
    ray.get(a.ping.remote())
    ray.get(b.ping.remote())

    # Verify we have collected stats across the nodes.
    info = memory_summary()
    print(info)
    assert count(info, PUT_OBJ) == 2, info 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_raylet_crash_when_get(ray_start_regular):
    def sleep_to_kill_raylet():
        # Don't kill raylet before default workers get connected.
        time.sleep(2)
        ray.worker._global_node.kill_raylet()

    object_id = ray.put(None)
    ray.internal.free(object_id)
    while ray.worker.global_worker.core_worker.object_exists(object_id):
        time.sleep(1)

    thread = threading.Thread(target=sleep_to_kill_raylet)
    thread.start()
    with pytest.raises(ray.exceptions.UnreconstructableError):
        ray.get(object_id)
    thread.join() 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_numpy_subclass_serialization(ray_start_regular):
    class MyNumpyConstant(np.ndarray):
        def __init__(self, value):
            super().__init__()
            self.constant = value

        def __str__(self):
            print(self.constant)

    constant = MyNumpyConstant(123)

    def explode(x):
        raise RuntimeError("Expected error.")

    ray.register_custom_serializer(
        type(constant), serializer=explode, deserializer=explode)

    try:
        ray.put(constant)
        assert False, "Should never get here!"
    except (RuntimeError, IndexError):
        print("Correct behavior, proof that customer serializer was used.") 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_internal_config_when_connecting(ray_start_cluster):
    config = json.dumps({
        "object_pinning_enabled": 0,
        "initial_reconstruction_timeout_milliseconds": 200
    })
    cluster = ray.cluster_utils.Cluster()
    cluster.add_node(
        _internal_config=config, object_store_memory=100 * 1024 * 1024)
    cluster.wait_for_nodes()

    # Specifying _internal_config when connecting to a cluster is disallowed.
    with pytest.raises(ValueError):
        ray.init(address=cluster.address, _internal_config=config)

    # Check that the config was picked up (object pinning is disabled).
    ray.init(address=cluster.address)
    oid = ray.put(np.zeros(40 * 1024 * 1024, dtype=np.uint8))

    for _ in range(5):
        ray.put(np.zeros(40 * 1024 * 1024, dtype=np.uint8))

    # This would not raise an exception if object pinning was enabled.
    with pytest.raises(ray.exceptions.UnreconstructableError):
        ray.get(oid) 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def test_cache(ray_start_regular):
    A = np.random.rand(1, 1000000)
    v = np.random.rand(1000000)
    A_id = ray.put(A)
    v_id = ray.put(v)
    a = time.time()
    for i in range(100):
        A.dot(v)
    b = time.time() - a
    c = time.time()
    for i in range(100):
        ray.get(A_id).dot(ray.get(v_id))
    d = time.time() - c

    if d > 1.5 * b:
        print("WARNING: The caching test was too slow. "
              "d = {}, b = {}".format(d, b)) 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def do_db_job(self, method, args, kwargs):
        db_job = RayDbJob(method, args=args, kwargs=kwargs)
        self.db_jobs_queue.put(db_job)
        db_job.wait()

        if db_job.error:
            raise db_job.error

        # self.print_timecheck("db job delay:", db_job.delay)
        # self.print_timecheck("db job duration:", db_job.duration)

        # self.print_timecheck('db job result:', db_job.result)
        return db_job.result 

# 需要导入模块: import ray [as 别名]
# 或者: from ray import put [as 别名]
def _put_notifications_in_ray_object_store(self, notifications):
        notification_ids = [(n["id"], ray.put(n)) for n in notifications]
        return notification_ids