Python Cluster.verify_alive方法代码示例

# 需要导入模块: from libraries.testkit.cluster import Cluster [as 别名]
# 或者: from libraries.testkit.cluster.Cluster import verify_alive [as 别名]
def params_from_base_test_setup(request, params_from_base_suite_setup):
    # Code before the yeild will execute before each test starts

    cluster_config = params_from_base_suite_setup["cluster_config"]
    mode = params_from_base_suite_setup["mode"]

    test_name = request.node.name
    log_info("Setting up test '{}'".format(test_name))

    # This dictionary is passed to each test
    yield {"cluster_config": cluster_config, "mode": mode}

    # Code after the yeild will execute when each test finishes
    log_info("Tearing down test '{}'".format(test_name))

    # Capture testkit socket usage
    network_utils = NetworkUtils()
    network_utils.list_connections()

    # Verify all sync_gateways and sg_accels are reachable
    c = Cluster(cluster_config)
    errors = c.verify_alive(mode)
    assert len(errors) == 0

    # if the test failed pull logs
    if request.node.rep_call.failed:
        logging_helper = Logging()
        logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)

# 需要导入模块: from libraries.testkit.cluster import Cluster [as 别名]
# 或者: from libraries.testkit.cluster.Cluster import verify_alive [as 别名]
def test_bucket_shadow_low_revs_limit_repeated_deletes(params_from_base_test_setup):
    """
    Validate that Sync Gateway doesn't panic (and instead creates a conflict branch
    and prints a warning) after doing the following steps:

    - Set revs_limit to 5
    - Create a doc via SG
    - Issue a delete operation for that doc via SG
    - Repeat step 3 5x. (each additional delete will create a new revision in SG, but the delete on the source bucket will fail with the 'not found' error, which also means that upstream_rev won't get incremented
    - Recreate the doc in the source bucket
    See https://github.com/couchbaselabs/sync-gateway-testcluster/issues/291#issuecomment-191521993
    """

    cluster_config = params_from_base_test_setup["cluster_config"]
    mode = params_from_base_test_setup["mode"]

    default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode)
    default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode)

    log_info("Running 'test_bucket_shadow_low_revs_limit_repeated_deletes'")
    log_info("Using cluster_config: {}".format(cluster_config))

    cluster = Cluster(config=cluster_config)
    sc = init_shadow_cluster(cluster,
                             default_config_path_shadower_low_revs,
                             default_config_path_non_shadower_low_revs)

    # Write doc into shadower SG
    doc_id = sc.alice_shadower.add_doc()

    # Wait until it gets to source bucket
    get_doc_from_source_bucket_retry(doc_id, sc.source_bucket)

    # Wait until upstream-rev in _sync metadata is non empty
    # Otherwise, this will not reproduce a panic
    while True:
        doc = sc.data_bucket.get(doc_id)
        if doc.success:
            if "upstream_rev" in doc.value["_sync"]:
                break
        time.sleep(1)

    # Repeatedly issue a delete operation for that doc via SG
    # Keep adding tombstone revs to the one and only branch
    rev_id_to_delete = None
    for i in xrange(100):
        resp = sc.alice_shadower.delete_doc(doc_id, rev_id_to_delete)
        rev_id_to_delete = resp["rev"]

    # Recreate doc with that ID in the source bucket
    sc.source_bucket.upsert(doc_id, json.loads('{"foo":"bar"}'))

    # Check if SG's are up
    errors = cluster.verify_alive(sc.mode)
    assert len(errors) == 0

    # Restart Shadow SG
    sc.shadower_sg.stop()
    sc.shadower_sg.start(default_config_path_shadower_low_revs)

# 需要导入模块: from libraries.testkit.cluster import Cluster [as 别名]
# 或者: from libraries.testkit.cluster.Cluster import verify_alive [as 别名]
def test_dcp_reshard_sync_gateway_goes_down(params_from_base_test_setup, sg_conf):

    cluster_conf = params_from_base_test_setup["cluster_config"]

    log_info("Running 'test_dcp_reshard_sync_gateway_goes_down'")
    log_info("cluster_conf: {}".format(cluster_conf))

    log_info("sg_conf: {}".format(sg_conf))

    cluster = Cluster(config=cluster_conf)
    mode = cluster.reset(sg_config_path=sg_conf)

    admin = Admin(cluster.sync_gateways[0])

    traun = admin.register_user(target=cluster.sync_gateways[0], db="db", name="traun", password="password", channels=["ABC", "NBC", "CBS"])
    seth = admin.register_user(target=cluster.sync_gateways[0], db="db", name="seth", password="password", channels=["FOX"])

    log_info(">> Users added")

    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:

        futures = dict()

        log_info(">>> Adding Seth docs")  # FOX
        futures[executor.submit(seth.add_docs, 8000)] = "seth"

        log_info(">>> Adding Traun docs")  # ABC, NBC, CBS
        futures[executor.submit(traun.add_docs, 2000, bulk=True)] = "traun"

        # stop sg_accel
        shutdown_status = cluster.sg_accels[0].stop()
        assert shutdown_status == 0

        for future in concurrent.futures.as_completed(futures):
            tag = futures[future]
            log_info("{} Completed:".format(tag))

    # TODO better way to do this
    time.sleep(120)

    verify_changes(traun, expected_num_docs=2000, expected_num_revisions=0, expected_docs=traun.cache)
    verify_changes(seth, expected_num_docs=8000, expected_num_revisions=0, expected_docs=seth.cache)

    # Verify that the sg1 is down but the other sync_gateways are running
    errors = cluster.verify_alive(mode)
    assert len(errors) == 1 and errors[0][0].hostname == "ac1"

    # Restart the failing node so that cluster verification does not blow up in test teardown
    start_status = cluster.sg_accels[0].start(sg_conf)
    assert start_status == 0

# 需要导入模块: from libraries.testkit.cluster import Cluster [as 别名]
# 或者: from libraries.testkit.cluster.Cluster import verify_alive [as 别名]
def test_bucket_shadow_multiple_sync_gateways(params_from_base_test_setup):

    cluster_config = params_from_base_test_setup["cluster_config"]
    mode = params_from_base_test_setup["mode"]

    log_info("Running 'test_bucket_shadow_multiple_sync_gateways'")
    log_info("Using cluster_config: {}".format(cluster_config))

    default_config_path_shadower = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow", mode)
    default_config_path_non_shadower = sync_gateway_config_path_for_mode("sync_gateway_default", mode)

    cluster = Cluster(config=cluster_config)
    sc = init_shadow_cluster(
        cluster,
        default_config_path_shadower,
        default_config_path_non_shadower,
    )

    # Write several docs into shadower SG
    doc_id_alice = sc.alice_shadower.add_doc()

    doc_id_will_delete = sc.alice_shadower.add_doc()

    # Write several docs into non-shadower SG
    doc_id_bob = sc.bob_non_shadower.add_doc()

    # Ditto as above, but bump revs rather than writing brand new docs
    sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
    sc.alice_shadower.update_doc(doc_id_alice, num_revision=10)

    # Get the doc from the source bucket, possibly retrying if needed
    # Otherwise an exception will be thrown and the test will fail
    get_doc_from_source_bucket_retry(doc_id_bob, sc.source_bucket)
    get_doc_from_source_bucket_retry(doc_id_alice, sc.source_bucket)
    get_doc_from_source_bucket_retry(doc_id_will_delete, sc.source_bucket)

    # Stop the SG shadower
    sc.shadower_sg.stop()

    # Write several docs + bump revs into non-shadower SG
    sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
    sc.bob_non_shadower.add_doc()

    # Delete one of the docs
    sc.bob_non_shadower.delete_doc(doc_id_will_delete)

    # Bring SG shadower back up
    sc.shadower_sg.start(default_config_path_shadower)
    time.sleep(5)  # Give tap feed a chance to initialize

    # Verify SG shadower comes up without panicking, given writes from non-shadower during downtime.
    errors = cluster.verify_alive(sc.mode)
    assert len(errors) == 0

    # If SG shadower does come up without panicking, bump a rev on a document that was modified during the downtime of the SG shadower
    sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)

    # Make sure the doc that was added while shadower was down makes it to source bucket
    # FAILING: Currently known to be failing, so temporarily disabled
    # get_doc_from_source_bucket_retry(doc_id_shadower_down, sc.source_bucket)

    # Manual check
    # Grep logs for:
    # WARNING: Error pushing rev of "f6eb40bf-d02a-4b4d-a3ab-779cce2fe9d5" to external bucket: MCResponse status=KEY_ENOENT, opcode=DELETE, opaque=0, msg: Not found -- db.(*Shadower).PushRevision() at shadower.go:164

    # Verify SG shadower comes up without panicking, given writes from non-shadower during downtime.
    time.sleep(5)  # Give tap feed a chance to initialize

# 需要导入模块: from libraries.testkit.cluster import Cluster [as 别名]
# 或者: from libraries.testkit.cluster.Cluster import verify_alive [as 别名]
def test_bucket_shadow_low_revs_limit(params_from_base_test_setup):
    """
    Set revs limit to 40
    Add doc and makes sure it syncs to source bucket
    Take shadower offline
    Update one doc more than 50 times
    Bring shadower online
    Look for panics
    Add more revisions to SG -- expected issue
    Look for panics
    (TODO: Update doc in shadow bucket and look for panics?)
    """

    cluster_config = params_from_base_test_setup["cluster_config"]
    mode = params_from_base_test_setup["mode"]

    log_info("Running 'test_bucket_shadow_low_revs_limit'")
    log_info("Using cluster_config: {}".format(cluster_config))

    default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode)
    default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode)

    cluster = Cluster(config=cluster_config)
    sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs, default_config_path_non_shadower_low_revs)

    # Write doc into shadower SG
    doc_id = sc.alice_shadower.add_doc()

    # Update the doc just so we have a rev_id
    sc.alice_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1)

    # Make sure it makes it to source bucket
    get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket)

    # Stop the SG shadower
    sc.shadower_sg.stop()

    # Update doc more than 50 times in non-shadower SG (since shadower is down)
    sc.bob_non_shadower.update_doc(doc_id, num_revision=100)
    sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1)

    # Bring SG shadower back up
    sc.shadower_sg.start(default_config_path_shadower_low_revs)

    # Look for panics
    time.sleep(5)  # Give tap feed a chance to initialize
    errors = cluster.verify_alive(sc.mode)
    assert len(errors) == 0

    # Verify that the latest revision sync'd to source bucket
    get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket)

    # Add more revisions
    sc.bob_non_shadower.update_doc(doc_id, num_revision=50)
    sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1)

    # Verify that the latest revision sync'd to source bucket
    get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket)

    # Look for panics
    time.sleep(5)  # Wait until the shadower can process