Java RemoteInterpreterUtils类代码示例

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
@Override
public void open() throws InterpreterException {
  try {
    if (ipythonClient != null) {
      // IPythonInterpreter might already been opened by PythonInterpreter
      return;
    }
    pythonExecutable = getProperty("zeppelin.python", "python");
    LOGGER.info("Python Exec: " + pythonExecutable);

    ipythonLaunchTimeout = Long.parseLong(
        getProperty("zeppelin.ipython.launch.timeout", "30000"));
    this.zeppelinContext = new PythonZeppelinContext(
        getInterpreterGroup().getInterpreterHookRegistry(),
        Integer.parseInt(getProperty("zeppelin.python.maxResult", "1000")));
    int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
    int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
    LOGGER.info("Launching IPython Kernel at port: " + ipythonPort);
    LOGGER.info("Launching JVM Gateway at port: " + jvmGatewayPort);
    ipythonClient = new IPythonClient("127.0.0.1", ipythonPort);
    launchIPythonKernel(ipythonPort);
    setupJVMGateway(jvmGatewayPort);
  } catch (Exception e) {
    throw new RuntimeException("Fail to open IPythonInterpreter", e);
  }
}
 

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
@Test
public void testStartStop() throws InterruptedException, IOException, TException {
  RemoteInterpreterServer server = new RemoteInterpreterServer(
      RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces());
  assertEquals(false, server.isRunning());

  server.start();
  long startTime = System.currentTimeMillis();
  boolean running = false;

  while (System.currentTimeMillis() - startTime < 10 * 1000) {
    if (server.isRunning()) {
      running = true;
      break;
    } else {
      Thread.sleep(200);
    }
  }

  assertEquals(true, running);
  assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", server.getPort()));

  server.shutdown();

  while (System.currentTimeMillis() - startTime < 10 * 1000) {
    if (server.isRunning()) {
      Thread.sleep(200);
    } else {
      running = false;
      break;
    }
  }
  assertEquals(false, running);
}
 

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
protected Map<String, String> buildEnvFromProperties(InterpreterLaunchContext context) {
  Map<String, String> env = new HashMap<>();
  for (Object key : context.getProperties().keySet()) {
    if (RemoteInterpreterUtils.isEnvString((String) key)) {
      env.put((String) key, context.getProperties().getProperty((String) key));
    }
  }
  return env;
}
 

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
@Test
public void testStartStop() throws InterruptedException, IOException, TException {
  RemoteInterpreterServer server = new RemoteInterpreterServer("localhost",
      RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", true);
  assertEquals(false, server.isRunning());

  server.start();
  long startTime = System.currentTimeMillis();
  boolean running = false;

  while (System.currentTimeMillis() - startTime < 10 * 1000) {
    if (server.isRunning()) {
      running = true;
      break;
    } else {
      Thread.sleep(200);
    }
  }

  assertEquals(true, running);
  assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", server.getPort()));

  server.shutdown();

  while (System.currentTimeMillis() - startTime < 10 * 1000) {
    if (server.isRunning()) {
      Thread.sleep(200);
    } else {
      running = false;
      break;
    }
  }
  assertEquals(false, running);
}
 

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
@Test
public void testFindRandomAvailablePortOnAllLocalInterfaces() throws IOException {
  assertTrue(RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces() > 0);
}
 

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
@Override
protected Map<String, String> buildEnvFromProperties(InterpreterLaunchContext context) {
  Map<String, String> env = new HashMap<String, String>();
  Properties sparkProperties = new Properties();
  String sparkMaster = getSparkMaster(properties);
  for (String key : properties.stringPropertyNames()) {
    if (RemoteInterpreterUtils.isEnvString(key)) {
      env.put(key, properties.getProperty(key));
    }
    if (isSparkConf(key, properties.getProperty(key))) {
      sparkProperties.setProperty(key, toShellFormat(properties.getProperty(key)));
    }
  }

  setupPropertiesForPySpark(sparkProperties);
  setupPropertiesForSparkR(sparkProperties);
  if (isYarnMode() && getDeployMode().equals("cluster")) {
    env.put("ZEPPELIN_SPARK_YARN_CLUSTER", "true");
  }

  StringBuilder sparkConfBuilder = new StringBuilder();
  if (sparkMaster != null) {
    sparkConfBuilder.append(" --master " + sparkMaster);
  }
  if (isYarnMode() && getDeployMode().equals("cluster")) {
    sparkConfBuilder.append(" --files " + zConf.getConfDir() + "/log4j_yarn_cluster.properties");
  }
  for (String name : sparkProperties.stringPropertyNames()) {
    sparkConfBuilder.append(" --conf " + name + "=" + sparkProperties.getProperty(name));
  }
  String useProxyUserEnv = System.getenv("ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER");
  if (context.getOption().isUserImpersonate() && (StringUtils.isBlank(useProxyUserEnv) ||
      !useProxyUserEnv.equals("false"))) {
    sparkConfBuilder.append(" --proxy-user " + context.getUserName());
  }

  env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());

  // set these env in the order of
  // 1. interpreter-setting
  // 2. zeppelin-env.sh
  // It is encouraged to set env in interpreter setting, but just for backward compatability,
  // we also fallback to zeppelin-env.sh if it is not specified in interpreter setting.
  for (String envName : new String[]{"SPARK_HOME", "SPARK_CONF_DIR", "HADOOP_CONF_DIR"})  {
    String envValue = getEnv(envName);
    if (envValue != null) {
      env.put(envName, envValue);
    }
  }
  LOGGER.debug("buildEnvFromProperties: " + env);
  return env;

}
 

import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; //导入依赖的package包/类
@Test
public void testStartStopWithQueuedEvents() throws InterruptedException, IOException, TException {
  RemoteInterpreterServer server = new RemoteInterpreterServer("localhost",
      RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(), ":", true);
  assertEquals(false, server.isRunning());

  server.start();
  long startTime = System.currentTimeMillis();
  boolean running = false;

  while (System.currentTimeMillis() - startTime < 10 * 1000) {
    if (server.isRunning()) {
      running = true;
      break;
    } else {
      Thread.sleep(200);
    }
  }

  assertEquals(true, running);
  assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", server.getPort()));

  //just send an event on the client queue
  server.eventClient.onAppStatusUpdate("","","","");

  ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();

  Runnable task = new ShutdownRun(server);

  executor.schedule(task, 0, TimeUnit.MILLISECONDS);

  while (System.currentTimeMillis() - startTime < 10 * 1000) {
    if (server.isRunning()) {
      Thread.sleep(200);
    } else {
      running = false;
      break;
    }
  }

  executor.shutdown();

  //cleanup environment for next tests
  server.shutdown();

  assertEquals(false, running);
}