Python utils.vtgate_start函数代码示例

 def setUp(self):
   environment.topo_server_wipe()
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
   if environment.topo_server_implementation == 'zookeeper':
     self.zkocc_server = utils.zkocc_start()
     self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
     self.topo = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base, 'test_nj', 30)
     self.topo.dial()

 def setUp(self):
   utils.zk_wipe()
   self.zkocc_server = utils.zkocc_start()
   # the default topo implementation for vtgate is zookeeper
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
   self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
   self.topo = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base, 'test_nj', 30)
   self.topo.dial()

  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildShardGraph test_keyspace/0', auto_log=True)
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start()
    qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
    time.sleep(10)
    utils.kill_sub_process(qpser)

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    # a typical workstation will do 38-40k QPS, check we have more than 15k
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < 150000:
      self.fail('QPS is too low: %u < 15000' % (rpcCalls / 10))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)

 def set_up(self):
   try:
     environment.topo_server().setup()
     utils.wait_procs([t.init_mysql() for t in self.tablets])
     utils.run_vtctl(['CreateKeyspace', self.keyspace])
     utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
     for t in self.tablets:
       t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     for t in self.tablets:
       t.create_db('vt_' + self.keyspace)
       t.start_vttablet(
         wait_for_state=None,
         extra_args=['-queryserver-config-schema-reload-time', '1'],
       )
     for t in self.tablets:
       t.wait_for_vttablet_state('SERVING')
     for t in self.tablets:
       if t.type == "master":
         utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     if self.schema:
       utils.run_vtctl(['ApplySchemaKeyspace', '-simple', '-sql', self.schema, self.keyspace])
     if self.vschema:
       if self.vschema[0] == '{':
         utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
       else:
         utils.run_vtctl(['ApplyVSchema', "-vschema_file", self.vschema])
     self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='500s', vtport=self.vtgate_port)
     vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
     topology.read_topology(vtgate_client)
   except:
     self.shutdown()
     raise

 def set_up(self):
   try:
     environment.topo_server_setup()
     utils.wait_procs([t.init_mysql() for t in self.tablets])
     utils.run_vtctl(['CreateKeyspace', self.keyspace])
     utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
     for t in self.tablets:
       t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     for t in self.tablets:
       t.create_db('vt_' + self.keyspace)
       t.mquery(t.dbname, create_table)
       t.start_vttablet(wait_for_state=None)
     for t in self.tablets:
       t.wait_for_vttablet_state('SERVING')
     for t in self.tablets:
       if t.type == "master":
         utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     self.vtgate_server, self.vtgate_port = utils.vtgate_start()
     vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
     topology.read_topology(vtgate_client)
   except:
     self.shutdown()
     raise

def setup_tablets():
  global vtgate_server
  global vtgate_port

  setup_sharded_keyspace()
  setup_unsharded_keyspace()
  vtgate_server, vtgate_port = utils.vtgate_start()

def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', USER_KEYSACE])
  utils.run_vtctl(['CreateKeyspace', LOOKUP_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', USER_KEYSACE,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=USER_KEYSACE, shard='-80')
  shard_1_master.init_tablet('master', keyspace=USER_KEYSACE, shard='80-')
  lookup_master.init_tablet('master', keyspace=LOOKUP_KEYSPACE, shard='0')

  for t in [shard_0_master, shard_1_master]:
    t.create_db('vt_user_keyspace')
    t.mquery('vt_user_keyspace', create_vt_user)
    t.mquery('vt_user_keyspace', create_vt_user2)
    t.mquery('vt_user_keyspace', create_vt_user_extra)
    t.mquery('vt_user_keyspace', create_vt_music)
    t.mquery('vt_user_keyspace', create_vt_music_extra)
    t.start_vttablet(wait_for_state='SERVING')
    utils.run_vtctl(['SetReadWrite', t.tablet_alias])
  lookup_master.create_db('vt_lookup_keyspace')
  lookup_master.mquery('vt_lookup_keyspace', create_vt_user_idx)
  lookup_master.mquery('vt_lookup_keyspace', create_music_user_map)
  lookup_master.mquery('vt_lookup_keyspace', create_name_user2_map)
  lookup_master.start_vttablet(wait_for_state='SERVING')
  utils.run_vtctl(['SetReadWrite', lookup_master.tablet_alias])

  utils.run_vtctl(['RebuildKeyspaceGraph', USER_KEYSACE], auto_log=True)
  utils.run_vtctl(['RebuildKeyspaceGraph', LOOKUP_KEYSPACE], auto_log=True)

  vtgate_server, vtgate_port = utils.vtgate_start(schema=schema)

  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start(
        extra_args=['-cpu_profile', os.path.join(environment.tmproot,
                                                 'vtgate.pprof')])
    qpser = utils.run_bg(environment.binary_args('zkclient2') + [
        '-server', 'localhost:%u' % vtgate_port,
        '-mode', 'qps',
        '-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
        'test_nj', 'test_keyspace'])
    qpser.wait()

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < MIN_QPS * 10:
      self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)

def setUpModule():
  global vtgate_server
  global vtgate_port
  global vtgate_socket_file
  global master_start_position

  try:
    environment.topo_server_setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()
                   ]
    utils.wait_procs(setup_procs)

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    master_tablet.create_db('vt_test_keyspace')
    master_tablet.create_db('other_database')
    replica_tablet.create_db('vt_test_keyspace')
    replica_tablet.create_db('other_database')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

    vtgate_socket_file = environment.tmproot + '/vtgate.sock'
    vtgate_server, vtgate_port = utils.vtgate_start(socket_file=vtgate_socket_file)

    master_tablet.start_vttablet()
    replica_tablet.start_vttablet()
    utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
    utils.check_db_read_write(master_tablet.tablet_uid)

    for t in [master_tablet, replica_tablet]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)

    # reset counter so tests don't assert
    tablet.Tablet.tablets_running = 0

    master_start_position = _get_master_current_position()
    master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
    master_tablet.mquery('vt_test_keyspace', _create_vt_a)
    master_tablet.mquery('vt_test_keyspace', _create_vt_b)

    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

  except:
    tearDownModule()
    raise

  def setUpClass(klass):
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl(['CreateKeyspace',
                     '--served-from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
                     'redirected_keyspace'])

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('spare',   'test_keyspace', '-80')
    shard_0_spare.init_tablet(  'spare',   'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
    idle.init_tablet('idle')
    scrap.init_tablet('idle')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
                    auto_log=True)

    # start running all the tablets
    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None,
                       extra_args=utils.vtctld.process_args())
    shard_0_replica.create_db('vt_test_keyspace')
    shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(),
                                   target_tablet_type='replica',
                                   wait_for_state=None)

    for t in scrap, idle, shard_0_spare:
      t.start_vttablet(wait_for_state=None,
                       extra_args=utils.vtctld.process_args())

    # wait for the right states
    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.wait_for_vttablet_state('SERVING')
    for t in [scrap, idle, shard_0_replica, shard_0_spare]:
      t.wait_for_vttablet_state('NOT_SERVING')

    scrap.scrap()

    for t in [shard_0_master, shard_0_replica, shard_0_spare,
              shard_1_master, shard_1_replica, idle, scrap]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-',
                     shard_1_master.tablet_alias], auto_log=True)
    shard_0_replica.wait_for_vttablet_state('SERVING')

    # run checks now before we start the tablets
    utils.validate_topology()

    # start a vtgate server too
    global vtgate_server, vtgate_port
    vtgate_server, vtgate_port = utils.vtgate_start(
        cache_ttl='0s', extra_args=utils.vtctld.process_args())

  def test_vtgate(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace -force test_keyspace')
    utils.run_vtctl('CreateShard -force test_keyspace/0')
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
    utils.validate_topology()

    # if these statements don't run before the tablet it will wedge waiting for the
    # db to become accessible. this is more a bug than a feature.
    tablet_62344.mquery("", ["set global read_only = off"])
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()
    gate_proc = utils.vtgate_start()

    try:
      conn = vtgate.connect("localhost:%s"%(utils.vtgate_port_base), "master", "test_keyspace", "0", 2.0)

      # _execute
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))

      # _stream_execute
      (result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {})
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
      count = 0
      while 1:
        r = conn._stream_next()
        if not r:
          break
        count += 1
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-rollback
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.rollback()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-commit
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.commit()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 5, "want 5, got %d" % (count))

      # close
      conn.close()
    finally:
      utils.vtgate_kill(gate_proc)
      tablet_62344.kill_vttablet()

  def setUp(self):
    self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
    self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
                                               "test_nj", 30.0)
    self.vtgate_addrs = None
    if vtgate_protocol == VTGATE_PROTOCOL_V1BSON:
      global vtgate_addrs
      self.vtgate_addrs = {"_vt": ["localhost:%s"%(self.vtgate_port),]}

    self.insert_index = 0

  def setUp(self):
    self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
    self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
                                               "test_nj", 30.0)
    self.vtgate_addrs = None
    if client_type == VTGATE:
      self.vtgate_addrs = {"vt": ["localhost:%s"%(self.vtgate_port),]}

    self.insert_index = 0
    # Lowering the keyspace refresh throttle so things are testable.
    self.throttle_sleep_interval = 0.1
    topology.set_keyspace_fetch_throttle(0.01)

 def setUp(self):
     global vtgate_server, vtgate_port
     self.shard_index = 0
     self.replica_tablet = shard_0_replica
     self.replica_tablet.kill_vttablet()
     self.replica_tablet.start_vttablet(auth=True)
     utils.vtgate_kill(vtgate_server)
     vtgate_server, vtgate_port = utils.vtgate_start(auth=True)
     credentials_file_name = os.path.join(environment.vttop, "test", "test_data", "authcredentials_test.json")
     credentials_file = open(credentials_file_name, "r")
     credentials = json.load(credentials_file)
     self.user = str(credentials.keys()[0])
     self.password = str(credentials[self.user][0])
     self.secondary_password = str(credentials[self.user][1])

def setUpModule():
  global vtgate_server, vtgate_port
  try:
    environment.topo_server().setup()
    setup_topology()

    # start mysql instance external to the test
    global __tablets
    setup_procs = []
    for tablet in __tablets:
      setup_procs.append(tablet.init_mysql())
    utils.wait_procs(setup_procs)
    create_db()
    start_tablets()
    vtgate_server, vtgate_port = utils.vtgate_start()
  except:
    tearDownModule()
    raise