Python utils.pause函数代码示例

  def setUpClass(cls):
    """Set up two keyspaces: one unsharded, one with two shards."""
    topology = vttest_pb2.VTTestTopology()
    topology.cells.append('test')
    topology.cells.append('test2')
    keyspace = topology.keyspaces.add(name='test_keyspace')
    keyspace.replica_count = 2
    keyspace.rdonly_count = 2
    keyspace.shards.add(name='-80')
    keyspace.shards.add(name='80-')
    keyspace2 = topology.keyspaces.add(name='test_keyspace2')
    keyspace2.shards.add(name='0')
    keyspace2.replica_count = 2
    keyspace2.rdonly_count = 1

    cls.driver = environment.create_webdriver()

    port = environment.reserve_ports(1)
    vttest_environment.base_port = port

    environment.reset_mysql_flavor()

    cls.db = local_database.LocalDatabase(
        topology,
        os.path.join(environment.vttop, 'test/vttest_schema'),
        False, None,
        web_dir=os.path.join(environment.vttop, 'web/vtctld'),
        default_schema_dir=os.path.join(
            environment.vttop, 'test/vttest_schema/default'),
        web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
    cls.db.setup()
    cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
    utils.pause('Paused test after vtcombo was started.\n'
                'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)

    def test_reparent_cross_cell(self, shard_id="0"):
        utils.run_vtctl("CreateKeyspace test_keyspace")

        # create the database so vttablets start, as they are serving
        tablet_62344.create_db("vt_test_keyspace")
        tablet_62044.create_db("vt_test_keyspace")
        tablet_41983.create_db("vt_test_keyspace")
        tablet_31981.create_db("vt_test_keyspace")

        # Start up a master mysql and vttablet
        tablet_62344.init_tablet("master", "test_keyspace", shard_id, start=True)
        if environment.topo_server_implementation == "zookeeper":
            shard = utils.run_vtctl_json(["GetShard", "test_keyspace/" + shard_id])
            self.assertEqual(shard["Cells"], ["test_nj"], "wrong list of cell in Shard: %s" % str(shard["Cells"]))

        # Create a few slaves for testing reparenting.
        tablet_62044.init_tablet("replica", "test_keyspace", shard_id, start=True, wait_for_start=False)
        tablet_41983.init_tablet("replica", "test_keyspace", shard_id, start=True, wait_for_start=False)
        tablet_31981.init_tablet("replica", "test_keyspace", shard_id, start=True, wait_for_start=False)
        for t in [tablet_62044, tablet_41983, tablet_31981]:
            t.wait_for_vttablet_state("SERVING")
        if environment.topo_server_implementation == "zookeeper":
            shard = utils.run_vtctl_json(["GetShard", "test_keyspace/" + shard_id])
            self.assertEqual(
                shard["Cells"], ["test_nj", "test_ny"], "wrong list of cell in Shard: %s" % str(shard["Cells"])
            )

        # Recompute the shard layout node - until you do that, it might not be valid.
        utils.run_vtctl("RebuildShardGraph test_keyspace/" + shard_id)
        utils.validate_topology()

        # Force the slaves to reparent assuming that all the datasets are identical.
        for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
            t.reset_replication()
        utils.pause("force ReparentShard?")
        utils.run_vtctl("ReparentShard -force test_keyspace/%s %s" % (shard_id, tablet_62344.tablet_alias))
        utils.validate_topology(ping_tablets=True)

        self._check_db_addr(shard_id, "master", tablet_62344.port)

        # Verify MasterCell is properly set
        srvShard = utils.run_vtctl_json(["GetSrvShard", "test_nj", "test_keyspace/%s" % (shard_id)])
        self.assertEqual(srvShard["MasterCell"], "test_nj")
        srvShard = utils.run_vtctl_json(["GetSrvShard", "test_ny", "test_keyspace/%s" % (shard_id)])
        self.assertEqual(srvShard["MasterCell"], "test_nj")

        # Perform a graceful reparent operation to another cell.
        utils.pause("graceful ReparentShard?")
        utils.run_vtctl("ReparentShard test_keyspace/%s %s" % (shard_id, tablet_31981.tablet_alias), auto_log=True)
        utils.validate_topology()

        self._check_db_addr(shard_id, "master", tablet_31981.port, cell="test_ny")

        # Verify MasterCell is set to new cell.
        srvShard = utils.run_vtctl_json(["GetSrvShard", "test_nj", "test_keyspace/%s" % (shard_id)])
        self.assertEqual(srvShard["MasterCell"], "test_ny")
        srvShard = utils.run_vtctl_json(["GetSrvShard", "test_ny", "test_keyspace/%s" % (shard_id)])
        self.assertEqual(srvShard["MasterCell"], "test_ny")

        tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983, tablet_31981])

def run_test_mysqlctl_split():
  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  tablet_62344.init_tablet('master', 'test_keyspace', '0')
  utils.run_vtctl('RebuildShardGraph test_keyspace/0')
  utils.validate_topology()

  tablet_62344.populate('vt_test_keyspace', create_vt_insert_test,
                        populate_vt_insert_test)

  tablet_62344.start_vttablet()

  err = tablet_62344.mysqlctl('-port %u -mysql-port %u partialsnapshot -end=0000000000000003 vt_test_keyspace id' % (tablet_62344.port, tablet_62344.mysql_port)).wait()
  if err != 0:
    raise utils.TestError('mysqlctl partialsnapshot failed')


  utils.pause("partialsnapshot finished")

  tablet_62044.mquery('', 'stop slave')
  tablet_62044.create_db('vt_test_keyspace')
  call(["touch", "/tmp/vtSimulateFetchFailures"])
  err = tablet_62044.mysqlctl('-port %u -mysql-port %u partialrestore %s/snapshot/vt_0000062344/data/vt_test_keyspace-,0000000000000003/partial_snapshot_manifest.json' % (tablet_62044.port, tablet_62044.mysql_port, utils.vtdataroot)).wait()
  if err != 0:
    raise utils.TestError('mysqlctl partialrestore failed')

  tablet_62044.assert_table_count('vt_test_keyspace', 'vt_insert_test', 2)

  # change/add two values on the master, one in range, one out of range, make
  # sure the right one propagate and not the other
  utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias)
  tablet_62344.mquery('vt_test_keyspace', "insert into vt_insert_test (id, msg) values (5, 'test should not propagate')", write=True)
  tablet_62344.mquery('vt_test_keyspace', "update vt_insert_test set msg='test should propagate' where id=2", write=True)

  utils.pause("look at db now!")

  # wait until value that should have been changed is here
  timeout = 10
  while timeout > 0:
    result = tablet_62044.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=2')
    if result[0][0] == "test should propagate":
      break
    timeout -= 1
    time.sleep(1)
  if timeout == 0:
    raise utils.TestError("expected propagation to happen", result)

  # test value that should not propagate
  # this part is disabled now, as the replication pruning is only enabled
  # for row-based replication, but the mysql server is statement based.
  # will re-enable once we get statement-based pruning patch into mysql.
#  tablet_62044.assert_table_count('vt_test_keyspace', 'vt_insert_test', 0, 'where id=5')

  tablet_62344.kill_vttablet()

def run_test_reparent_lag_slave(shard_id="0"):
    utils.zk_wipe()

    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # create the database so vttablets start, as they are serving
    tablet_62344.create_db("vt_test_keyspace")
    tablet_62044.create_db("vt_test_keyspace")
    tablet_41983.create_db("vt_test_keyspace")
    tablet_31981.create_db("vt_test_keyspace")

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet("master", "test_keyspace", shard_id, start=True)

    # Create a few slaves for testing reparenting.
    tablet_62044.init_tablet("replica", "test_keyspace", shard_id, start=True)
    tablet_31981.init_tablet("replica", "test_keyspace", shard_id, start=True)
    tablet_41983.init_tablet("lag", "test_keyspace", shard_id, start=True)

    # Recompute the shard layout node - until you do that, it might not be valid.
    utils.run_vtctl("RebuildShardGraph test_keyspace/" + shard_id)
    utils.validate_topology()

    # Force the slaves to reparent assuming that all the datasets are identical.
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
        t.reset_replication()
    utils.run_vtctl("ReparentShard -force test_keyspace/%s %s" % (shard_id, tablet_62344.tablet_alias))
    utils.validate_topology(ping_tablets=True)

    tablet_62344.create_db("vt_test_keyspace")
    tablet_62344.mquery("vt_test_keyspace", create_vt_insert_test)

    tablet_41983.mquery("", "stop slave")
    for q in populate_vt_insert_test:
        tablet_62344.mquery("vt_test_keyspace", q, write=True)

    # Perform a graceful reparent operation.
    utils.run_vtctl("ReparentShard test_keyspace/%s %s" % (shard_id, tablet_62044.tablet_alias))

    tablet_41983.mquery("", "start slave")
    time.sleep(1)

    utils.pause("check orphan")

    utils.run_vtctl("ReparentTablet %s" % tablet_41983.tablet_alias)

    result = tablet_41983.mquery("vt_test_keyspace", "select msg from vt_insert_test where id=1")
    if len(result) != 1:
        raise utils.TestError("expected 1 row from vt_insert_test", result)

    utils.pause("check lag reparent")

    tablet_62344.kill_vttablet()
    tablet_62044.kill_vttablet()
    tablet_41983.kill_vttablet()
    tablet_31981.kill_vttablet()

  def test_reparent_lag_slave(self, shard_id='0'):
    utils.run_vtctl('CreateKeyspace test_keyspace')

    # create the database so vttablets start, as they are serving
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62044.create_db('vt_test_keyspace')
    tablet_41983.create_db('vt_test_keyspace')
    tablet_31981.create_db('vt_test_keyspace')

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True, wait_for_start=False)

    # Create a few slaves for testing reparenting.
    tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False)
    tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False)
    tablet_41983.init_tablet('lag', 'test_keyspace', shard_id, start=True, wait_for_start=False)

    # wait for all tablets to start
    for t in [tablet_62344, tablet_62044, tablet_31981]:
      t.wait_for_vttablet_state("SERVING")
    tablet_41983.wait_for_vttablet_state("NOT_SERVING")

    # Recompute the shard layout node - until you do that, it might not be valid.
    utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id)
    utils.validate_topology()

    # Force the slaves to reparent assuming that all the datasets are identical.
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
      t.reset_replication()
    utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' % (shard_id, tablet_62344.tablet_alias))
    utils.validate_topology(ping_tablets=True)

    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.mquery('vt_test_keyspace', self._create_vt_insert_test)

    tablet_41983.mquery('', 'stop slave')
    for q in self._populate_vt_insert_test:
      tablet_62344.mquery('vt_test_keyspace', q, write=True)

    # Perform a graceful reparent operation.
    utils.run_vtctl('ReparentShard test_keyspace/%s %s' % (shard_id, tablet_62044.tablet_alias))

    tablet_41983.mquery('', 'start slave')
    time.sleep(1)

    utils.pause("check orphan")

    utils.run_vtctl('ReparentTablet %s' % tablet_41983.tablet_alias)

    result = tablet_41983.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=1')
    if len(result) != 1:
      self.fail('expected 1 row from vt_insert_test: %s' % str(result))

    utils.pause("check lag reparent")

    tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983, tablet_31981])

  def test_reparent_cross_cell(self, shard_id='0'):
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    # create the database so vttablets start, as they are serving
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62044.create_db('vt_test_keyspace')
    tablet_41983.create_db('vt_test_keyspace')
    tablet_31981.create_db('vt_test_keyspace')

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True,
                             wait_for_start=False)
    shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/' + shard_id])
    self.assertEqual(shard['cells'], ['test_nj'],
                     'wrong list of cell in Shard: %s' % str(shard['cells']))

    # Create a few slaves for testing reparenting. Won't be healthy
    # as replication is not running.
    tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True,
                             wait_for_start=False)
    tablet_41983.init_tablet('replica', 'test_keyspace', shard_id, start=True,
                             wait_for_start=False)
    tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True,
                             wait_for_start=False)
    tablet_62344.wait_for_vttablet_state('SERVING')
    for t in [tablet_62044, tablet_41983, tablet_31981]:
      t.wait_for_vttablet_state('NOT_SERVING')
    shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/' + shard_id])
    self.assertEqual(
        shard['cells'], ['test_nj', 'test_ny'],
        'wrong list of cell in Shard: %s' % str(shard['cells']))

    utils.validate_topology()

    # Force the slaves to reparent assuming that all the datasets are
    # identical.
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
      t.reset_replication()
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/' + shard_id,
                     tablet_62344.tablet_alias], auto_log=True)
    utils.validate_topology(ping_tablets=True)

    self._check_master_tablet(tablet_62344)

    # Perform a graceful reparent operation to another cell.
    utils.pause('test_reparent_cross_cell PlannedReparentShard')
    utils.run_vtctl(['PlannedReparentShard', 'test_keyspace/' + shard_id,
                     tablet_31981.tablet_alias], auto_log=True)
    utils.validate_topology()

    self._check_master_tablet(tablet_31981)

    tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983,
                         tablet_31981])

    def setUpClass(cls):
        """Set up two keyspaces: one unsharded, one with two shards."""
        topology = vttest_pb2.VTTestTopology()
        topology.cells.append("test")
        topology.cells.append("test2")
        keyspace = topology.keyspaces.add(name="test_keyspace")
        keyspace.replica_count = 2
        keyspace.rdonly_count = 2
        keyspace.shards.add(name="-80")
        keyspace.shards.add(name="80-")
        keyspace2 = topology.keyspaces.add(name="test_keyspace2")
        keyspace2.shards.add(name="0")
        keyspace2.replica_count = 2
        keyspace2.rdonly_count = 1

        if os.environ.get("CI") == "true" and os.environ.get("TRAVIS") == "true":
            username = os.environ["SAUCE_USERNAME"]
            access_key = os.environ["SAUCE_ACCESS_KEY"]
            capabilities = {}
            capabilities["tunnel-identifier"] = os.environ["TRAVIS_JOB_NUMBER"]
            capabilities["build"] = os.environ["TRAVIS_BUILD_NUMBER"]
            capabilities["platform"] = "Linux"
            capabilities["browserName"] = "chrome"
            hub_url = "%s:%[email protected]:4445" % (username, access_key)
            cls.driver = webdriver.Remote(
                desired_capabilities=capabilities, command_executor="http://%s/wd/hub" % hub_url
            )
        else:
            os.environ["webdriver.chrome.driver"] = os.path.join(environment.vtroot, "dist")
            # Only testing against Chrome for now
            cls.driver = webdriver.Chrome()
            cls.driver.set_window_position(0, 0)
            cls.driver.set_window_size(1920, 1280)

        port = environment.reserve_ports(1)
        vttest_environment.base_port = port
        mysql_flavor.set_mysql_flavor(None)

        cls.db = local_database.LocalDatabase(
            topology,
            os.path.join(environment.vttop, "test/vttest_schema"),
            False,
            None,
            web_dir=os.path.join(environment.vttop, "web/vtctld"),
            default_schema_dir=os.path.join(environment.vttop, "test/vttest_schema/default"),
            web_dir2=os.path.join(environment.vttop, "web/vtctld2/app"),
        )
        cls.db.setup()
        cls.vtctld_addr = "http://localhost:%d" % cls.db.config()["port"]
        utils.pause(
            "Paused test after vtcombo was started.\n" "For manual testing, connect to vtctld: %s" % cls.vtctld_addr
        )

def _run_test_vtctl_clone(server_mode):
    if server_mode:
        clone_flags = "-server-mode"
    else:
        clone_flags = ""
    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force snapshot_test")

    tablet_62344.init_tablet("master", "snapshot_test", "0")
    utils.run_vtctl("RebuildShardGraph snapshot_test/0")
    utils.validate_topology()

    tablet_62344.populate("vt_snapshot_test", create_vt_insert_test, populate_vt_insert_test)
    tablet_62344.start_vttablet()

    tablet_62044.create_db("vt_snapshot_test")
    tablet_62044.init_tablet("idle", start=True)

    # small test to make sure the directory validation works
    snapshot_dir = os.path.join(utils.vtdataroot, "snapshot")
    utils.run("rm -rf %s" % snapshot_dir)
    utils.run("mkdir -p %s" % snapshot_dir)
    utils.run("chmod -w %s" % snapshot_dir)
    out, err = utils.run(
        utils.vtroot
        + "/bin/vtctl --alsologtostderr Clone -force %s %s %s"
        % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias),
        trap_output=True,
        raise_on_error=False,
    )
    if "Cannot validate snapshot directory" not in err:
        raise utils.TestError("expected validation error", err)
    if "Un-reserved test_nj-0000062044" not in err:
        raise utils.TestError("expected Un-reserved", err)
    utils.debug("Failed Clone output: " + err)
    utils.run("chmod +w %s" % snapshot_dir)

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    utils.run_vtctl(
        "Clone -force %s %s %s" % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias), auto_log=True
    )

    utils.pause("look at logs!")
    tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)
    tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

    utils.validate_topology()

    tablet_62344.kill_vttablet()
    tablet_62044.kill_vttablet()

def _run_test_mysqlctl_clone(server_mode):
    if server_mode:
        snapshot_cmd = "snapshotsourcestart -concurrency=8"
        restore_flags = "-dont-wait-for-slave-start"
    else:
        snapshot_cmd = "snapshot -concurrency=5"
        restore_flags = ""

    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force snapshot_test")

    tablet_62344.init_tablet("master", "snapshot_test", "0")
    utils.run_vtctl("RebuildShardGraph snapshot_test/0")
    utils.validate_topology()

    tablet_62344.populate("vt_snapshot_test", create_vt_insert_test, populate_vt_insert_test)

    tablet_62344.start_vttablet()

    err = tablet_62344.mysqlctl(
        "-port %u -mysql-port %u %s vt_snapshot_test" % (tablet_62344.port, tablet_62344.mysql_port, snapshot_cmd)
    ).wait()
    if err != 0:
        raise utils.TestError("mysqlctl %s failed" % snapshot_cmd)

    utils.pause("%s finished" % snapshot_cmd)

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    err = tablet_62044.mysqlctl(
        "-port %u -mysql-port %u restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s/snapshot/vt_0000062344/snapshot_manifest.json"
        % (tablet_62044.port, tablet_62044.mysql_port, restore_flags, utils.vtdataroot)
    ).wait()
    if err != 0:
        raise utils.TestError("mysqlctl restore failed")

    tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

    if server_mode:
        err = tablet_62344.mysqlctl(
            "-port %u -mysql-port %u snapshotsourceend -read-write vt_snapshot_test"
            % (tablet_62344.port, tablet_62344.mysql_port)
        ).wait()
        if err != 0:
            raise utils.TestError("mysqlctl snapshotsourceend failed")

        # see if server restarted properly
        tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

    tablet_62344.kill_vttablet()

def cache(url, path):
	print 'Downloading: %s' % urlparse.urlsplit(url).path
	page = requests.get(url, proxies=PROX)
	if page.status_code == 404:
		return {'url': url, 'uid': 'Error: 404', 'time': gmt_now()}, []
	assert page.status_code == 200, 'Error: Status code error in page download, received %s @ %s' % (page.status_code, urlparse.urlsplit(url).path)
	tree = html.fromstring(page.content)

	links = list_links([str(x) for x in tree.xpath(r'//*[@href]/@href')], url=url)

	uid = dump(html.tostring(tree), path, verbose=True)

	pause()
	return {'url': url, 'uid': uid, 'time': gmt_now()}, links

  def test_vtgate(self):
    # do a few vtgate topology queries to prime the cache
    vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), "test_nj", 30.0)
    vtgate_client.dial()
    vtgate_client.get_srv_keyspace_names("test_nj")
    vtgate_client.get_srv_keyspace("test_nj", "test_keyspace")
    vtgate_client.get_end_points("test_nj", "test_keyspace", "-80", "master")
    vtgate_client.close()

    status = utils.vtgate.get_status()
    self.assertIn('</html>', status) # end of page
    self.assertIn('/serving_graph/test_nj">test_nj', status) # vtctld link

    utils.pause("You can now run a browser and connect to http://%s:%d%s to manually check vtgate status page" % (socket.getfqdn(), utils.vtgate.port, environment.status_url))

def _run_test_vtctl_clone(server_mode):
  if server_mode:
    clone_flags = '-server-mode'
  else:
    clone_flags = ''
  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force snapshot_test')

  tablet_62344.init_tablet('master', 'snapshot_test', '0')
  utils.run_vtctl('RebuildShardGraph snapshot_test/0')
  utils.validate_topology()

  tablet_62344.populate('vt_snapshot_test', create_vt_insert_test,
                        populate_vt_insert_test)
  tablet_62344.start_vttablet()

  tablet_62044.create_db('vt_snapshot_test')
  tablet_62044.init_tablet('idle', start=True)

  # small test to make sure the directory validation works
  snapshot_dir = os.path.join(utils.vtdataroot, 'snapshot')
  utils.run("rm -rf %s" % snapshot_dir)
  utils.run("mkdir -p %s" % snapshot_dir)
  utils.run("chmod -w %s" % snapshot_dir)
  out, err = utils.run(utils.vtroot+'/bin/vtctl -logfile=/dev/null Clone -force %s %s %s' %
                       (clone_flags, tablet_62344.tablet_alias,
                        tablet_62044.tablet_alias),
                       trap_output=True, raise_on_error=False)
  if "Cannot validate snapshot directory" not in err:
    raise utils.TestError("expected validation error", err)
  if "Un-reserved test_nj-0000062044" not in err:
    raise utils.TestError("expected Un-reserved", err)
  utils.debug("Failed Clone output: " + err)
  utils.run("chmod +w %s" % snapshot_dir)

  call(["touch", "/tmp/vtSimulateFetchFailures"])
  utils.run_vtctl('Clone -force %s %s %s' %
                  (clone_flags, tablet_62344.tablet_alias,
                   tablet_62044.tablet_alias))

  utils.pause("look at logs!")
  tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)
  tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)

  utils.validate_topology()

  tablet_62344.kill_vttablet()
  tablet_62044.kill_vttablet()

  def _test_mysqlctl_clone(server_mode):
    if server_mode:
      snapshot_cmd = ['snapshotsourcestart', '-concurrency=8']
      restore_flags = ['-dont_wait_for_slave_start']
    else:
      snapshot_cmd = ['snapshot', '-concurrency=5']
      restore_flags = []

    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'snapshot_test'])

    tablet_62344.init_tablet('master', 'snapshot_test', '0')
    utils.run_vtctl(['RebuildShardGraph', 'snapshot_test/0'])
    utils.validate_topology()

    tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test,
                          self._populate_vt_insert_test)

    tablet_62344.start_vttablet()

    err = tablet_62344.mysqlctl(snapshot_cmd + ['vt_snapshot_test'],
                                with_ports=True).wait()
    if err != 0:
      self.fail('mysqlctl %s failed' % str(snapshot_cmd))

    utils.pause("%s finished" % str(snapshot_cmd))

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    err = tablet_31981.mysqlctl(['restore',
                                 '-fetch_concurrency=2',
                                 '-fetch_retry_count=4'] +
                                restore_flags +
                                [environment.vtdataroot + '/snapshot/vt_0000062344/snapshot_manifest.json'],
                                with_ports=True).wait()
    if err != 0:
      self.fail('mysqlctl restore failed')

    tablet_31981.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)

    if server_mode:
      err = tablet_62344.mysqlctl(['snapshotsourceend',
                                   '-read_write',
                                   'vt_snapshot_test'], with_ports=True).wait()
      if err != 0:
        self.fail('mysqlctl snapshotsourceend failed')

      # see if server restarted properly
      tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)

    tablet_62344.kill_vttablet()

  def _test_vtctl_clone(self, server_mode):
    if server_mode:
      clone_flags = ['-server-mode']
    else:
      clone_flags = []

    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'snapshot_test'])

    tablet_62344.init_tablet('master', 'snapshot_test', '0')
    utils.run_vtctl(['RebuildShardGraph', 'snapshot_test/0'])
    utils.validate_topology()

    tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test,
                          self._populate_vt_insert_test)
    tablet_62344.start_vttablet()

    tablet_31981.create_db('vt_snapshot_test')
    tablet_31981.init_tablet('idle', start=True)

    # small test to make sure the directory validation works
    snapshot_dir = os.path.join(environment.vtdataroot, 'snapshot')
    utils.run("rm -rf %s" % snapshot_dir)
    utils.run("mkdir -p %s" % snapshot_dir)
    utils.run("chmod -w %s" % snapshot_dir)
    out, err = utils.run_vtctl(['Clone', '-force'] + clone_flags +
                               [tablet_62344.tablet_alias,
                                tablet_31981.tablet_alias],
                               log_level='INFO', expect_fail=True)
    if "Cannot validate snapshot directory" not in err:
      self.fail("expected validation error: %s" % err)
    if "Un-reserved test_ny-0000031981" not in err:
      self.fail("expected Un-reserved: %s" % err)
    logging.debug("Failed Clone output: " + err)
    utils.run("chmod +w %s" % snapshot_dir)

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    utils.run_vtctl(['Clone', '-force'] + clone_flags +
                    [tablet_62344.tablet_alias, tablet_31981.tablet_alias],
                    auto_log=True)
    self._check_shard()

    utils.pause("look at logs!")
    tablet_31981.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)
    tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)

    utils.validate_topology()

    tablet.kill_tablets([tablet_62344, tablet_31981])

    def _test_vtctl_clone(self, server_mode):
        if server_mode:
            clone_flags = ["-server-mode"]
        else:
            clone_flags = []

        # Start up a master mysql and vttablet
        utils.run_vtctl(["CreateKeyspace", "snapshot_test"])

        tablet_62344.init_tablet("master", "snapshot_test", "0")
        utils.run_vtctl(["RebuildShardGraph", "snapshot_test/0"])
        utils.validate_topology()

        tablet_62344.populate("vt_snapshot_test", self._create_vt_insert_test, self._populate_vt_insert_test)
        tablet_62344.start_vttablet()

        tablet_62044.create_db("vt_snapshot_test")
        tablet_62044.init_tablet("idle", start=True)

        # small test to make sure the directory validation works
        snapshot_dir = os.path.join(environment.vtdataroot, "snapshot")
        utils.run("rm -rf %s" % snapshot_dir)
        utils.run("mkdir -p %s" % snapshot_dir)
        utils.run("chmod -w %s" % snapshot_dir)
        out, err = utils.run_vtctl(
            ["Clone", "-force"] + clone_flags + [tablet_62344.tablet_alias, tablet_62044.tablet_alias],
            log_level="INFO",
            expect_fail=True,
        )
        if "Cannot validate snapshot directory" not in err:
            self.fail("expected validation error: %s" % err)
        if "Un-reserved test_nj-0000062044" not in err:
            self.fail("expected Un-reserved: %s" % err)
        logging.debug("Failed Clone output: " + err)
        utils.run("chmod +w %s" % snapshot_dir)

        call(["touch", "/tmp/vtSimulateFetchFailures"])
        utils.run_vtctl(
            ["Clone", "-force"] + clone_flags + [tablet_62344.tablet_alias, tablet_62044.tablet_alias], auto_log=True
        )

        utils.pause("look at logs!")
        tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)
        tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

        utils.validate_topology()

        tablet.kill_tablets([tablet_62344, tablet_62044])