C++ commandinfo::URI类代码示例

TEST_F(MesosContainerizerProcessTest, MultipleURIs)
{
  CommandInfo commandInfo;
  CommandInfo::URI uri;
  uri.set_value("hdfs:///uri1");
  uri.set_executable(false);
  commandInfo.add_uris()->MergeFrom(uri);
  uri.set_value("hdfs:///uri2");
  uri.set_executable(true);
  commandInfo.add_uris()->MergeFrom(uri);

  string directory = "/tmp/directory";
  Option<string> user("user");

  Flags flags;
  flags.frameworks_home = "/tmp/frameworks";
  flags.hadoop_home = "/tmp/hadoop";

  map<string, string> environment =
    fetcherEnvironment(commandInfo, directory, user, flags);

  EXPECT_EQ(5u, environment.size());
  EXPECT_EQ(
      "hdfs:///uri1+0X hdfs:///uri2+1X", environment["MESOS_EXECUTOR_URIS"]);
  EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
  EXPECT_EQ(user.get(), environment["MESOS_USER"]);
  EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
  EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}

TEST_F(FetcherTest, FileLocalhostURI)
{
  string fromDir = path::join(os::getcwd(), "from");
  ASSERT_SOME(os::mkdir(fromDir));
  string testFile = path::join(fromDir, "test");
  EXPECT_FALSE(os::write(testFile, "data").isError());

  string localFile = path::join(os::getcwd(), "test");
  EXPECT_FALSE(os::exists(localFile));

  slave::Flags flags;
  flags.frameworks_home = "/tmp/frameworks";

  CommandInfo commandInfo;
  CommandInfo::URI* uri = commandInfo.add_uris();
  uri->set_value(path::join("file://localhost", testFile));

  map<string, string> env =
    fetcher::environment(commandInfo, os::getcwd(), None(), flags);

  Try<Subprocess> fetcherProcess =
    process::subprocess(
      path::join(mesos::internal::tests::flags.build_dir, "src/mesos-fetcher"),
      env);

  ASSERT_SOME(fetcherProcess);
  Future<Option<int>> status = fetcherProcess.get().status();

  AWAIT_READY(status);
  ASSERT_SOME(status.get());

  EXPECT_EQ(0, status.get().get());
  EXPECT_TRUE(os::exists(localFile));
}

TEST_F(FetcherEnvironmentTest, MultipleURIs)
{
  CommandInfo commandInfo;
  CommandInfo::URI uri;
  uri.set_value("hdfs:///uri1");
  uri.set_executable(false);
  commandInfo.add_uris()->MergeFrom(uri);
  uri.set_value("hdfs:///uri2");
  uri.set_executable(true);
  commandInfo.add_uris()->MergeFrom(uri);

  string directory = "/tmp/directory";
  Option<string> user("user");

  slave::Flags flags;
  flags.frameworks_home = "/tmp/frameworks";
  flags.hadoop_home = "/tmp/hadoop";

  map<string, string> environment =
    fetcher::environment(commandInfo, directory, user, flags);

  EXPECT_EQ(5u, environment.size());
  EXPECT_EQ(stringify(JSON::Protobuf(commandInfo)),
            environment["MESOS_COMMAND_INFO"]);
  EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
  EXPECT_EQ(user.get(), environment["MESOS_USER"]);
  EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
  EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}

inline std::size_t hash_value(const CommandInfo::URI& uri)
{
  size_t seed = 0;

  if (uri.extract()) {
    seed += 11;
  }

  if (uri.executable()) {
    seed += 2003;
  }

  boost::hash_combine(seed, uri.value());
  return seed;
}

// Returns the resulting file or in case of extraction the destination
// directory (for logging).
static Try<string> fetchBypassingCache(
    const CommandInfo::URI& uri,
    const string& sandboxDirectory,
    const Option<string>& frameworksHome)
{
  LOG(INFO) << "Fetching directly into the sandbox directory";

  // TODO(mrbrowning): Factor out duplicated processing of "output_file" field
  // here and in fetchFromCache into a separate helper function.
  if (uri.has_output_file()) {
    string dirname = Path(uri.output_file()).dirname();
    if (dirname != ".") {
      Try<Nothing> result =
        os::mkdir(path::join(sandboxDirectory, dirname), true);

      if (result.isError()) {
        return Error(
            "Unable to create subdirectory " + dirname + " in sandbox");
      }
    }
  }

  Try<string> outputFile = uri.has_output_file()
    ? uri.output_file()
    : Fetcher::basename(uri.value());

  if (outputFile.isError()) {
    return Error(outputFile.error());
  }

  string path = path::join(sandboxDirectory, outputFile.get());

  Try<string> downloaded = download(uri.value(), path, frameworksHome);
  if (downloaded.isError()) {
    return Error(downloaded.error());
  }

  if (uri.executable()) {
    return chmodExecutable(downloaded.get());
  } else if (uri.extract()) {
    Try<bool> extracted = extract(path, sandboxDirectory);
    if (extracted.isError()) {
      return Error(extracted.error());
    } else if (!extracted.get()) {
      LOG(WARNING) << "Copying instead of extracting resource from URI with "
                   << "'extract' flag, because it does not seem to be an "
                   << "archive: " << uri.value();
    }
  }

  return downloaded;
}

inline bool operator == (
    const CommandInfo::URI& left,
    const CommandInfo::URI& right)
{
  return left.has_executable() == right.has_executable() &&
    (!left.has_executable() || (left.executable() == right.executable())) &&
    left.value() == right.value();
}

// Returns the resulting file or in case of extraction the destination
// directory (for logging).
static Try<string> fetchBypassingCache(
    const CommandInfo::URI& uri,
    const string& sandboxDirectory,
    const Option<string>& frameworksHome)
{
  LOG(INFO) << "Fetching directly into the sandbox directory";

  Try<string> basename = Fetcher::basename(uri.value());
  if (basename.isError()) {
    return Error("Failed to determine the basename of the URI '" +
                 uri.value() + "' with error: " + basename.error());
  }

  string path = path::join(sandboxDirectory, basename.get());

  Try<string> downloaded = download(uri.value(), path, frameworksHome);
  if (downloaded.isError()) {
    return Error(downloaded.error());
  }

  if (uri.executable()) {
    return chmodExecutable(downloaded.get());
  } else if (uri.extract()) {
    Try<bool> extracted = extract(path, sandboxDirectory);
    if (extracted.isError()) {
      return Error(extracted.error());
    } else if (!extracted.get()) {
      LOG(WARNING) << "Copying instead of extracting resource from URI with "
                   << "'extract' flag, because it does not seem to be an "
                   << "archive: " << uri.value();
    }
  }

  return downloaded;
}

TEST_F(MesosContainerizerProcessTest, NoUser)
{
  CommandInfo commandInfo;
  CommandInfo::URI uri;
  uri.set_value("hdfs:///uri");
  uri.set_executable(false);
  commandInfo.add_uris()->MergeFrom(uri);

  string directory = "/tmp/directory";

  Flags flags;
  flags.frameworks_home = "/tmp/frameworks";
  flags.hadoop_home = "/tmp/hadoop";

  map<string, string> environment =
    fetcherEnvironment(commandInfo, directory, None(), flags);

  EXPECT_EQ(4u, environment.size());
  EXPECT_EQ("hdfs:///uri+0X", environment["MESOS_EXECUTOR_URIS"]);
  EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
  EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
  EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}

TEST_F(FetcherEnvironmentTest, NoHadoop)
{
  CommandInfo commandInfo;
  CommandInfo::URI* uri = commandInfo.add_uris();
  uri->set_value("hdfs:///uri");
  uri->set_executable(false);

  string directory = "/tmp/directory";
  Option<string> user = "user";

  slave::Flags flags;
  flags.frameworks_home = "/tmp/frameworks";

  map<string, string> environment =
    fetcher::environment(commandInfo, directory, user, flags);

  EXPECT_EQ(4u, environment.size());
  EXPECT_EQ(stringify(JSON::Protobuf(commandInfo)),
            environment["MESOS_COMMAND_INFO"]);
  EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
  EXPECT_EQ(user.get(), environment["MESOS_USER"]);
  EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
}

TEST_F(FetcherTest, OSNetUriTest)
{
  HttpProcess process;

  spawn(process);

  string url = "http://" + net::getHostname(process.self().node.ip).get() +
                ":" + stringify(process.self().node.port) + "/help";

  string localFile = path::join(os::getcwd(), "help");
  EXPECT_FALSE(os::exists(localFile));

  slave::Flags flags;
  flags.frameworks_home = "/tmp/frameworks";

  CommandInfo commandInfo;
  CommandInfo::URI* uri = commandInfo.add_uris();
  uri->set_value(url);

  map<string, string> env =
    fetcher::environment(commandInfo, os::getcwd(), None(), flags);

  Try<Subprocess> fetcherProcess =
    process::subprocess(
      path::join(mesos::internal::tests::flags.build_dir, "src/mesos-fetcher"),
      env);

  ASSERT_SOME(fetcherProcess);
  Future<Option<int>> status = fetcherProcess.get().status();

  AWAIT_READY(status);
  ASSERT_SOME(status.get());

  EXPECT_EQ(0, status.get().get());
  EXPECT_TRUE(os::exists(localFile));
}

bool operator == (const CommandInfo::URI& left, const CommandInfo::URI& right)
{
    return left.value() == right.value() &&
           left.executable() == right.executable() &&
           left.extract() == right.extract();
}

int main(int argc, char* argv[])
{
  GOOGLE_PROTOBUF_VERIFY_VERSION;

  CommandInfo commandInfo;
  // Construct URIs from the encoded environment string.
  const std::string& uris = os::getenv("MESOS_EXECUTOR_URIS");
  foreach (const std::string& token, strings::tokenize(uris, " ")) {
    // Delimiter between URI, execute permission and extract options
    // Expected format: {URI}+[01][XN]
    //  {URI} - The actual URI for the asset to fetch
    //  [01]  - 1 if the execute permission should be set else 0
    //  [XN]  - X if we should extract the URI (if it's compressed) else N
    size_t pos = token.rfind("+");
    CHECK(pos != std::string::npos)
      << "Invalid executor uri token in env " << token;

    CommandInfo::URI uri;
    uri.set_value(token.substr(0, pos));
    uri.set_executable(token.substr(pos + 1, 1) == "1");
    uri.set_extract(token.substr(pos + 2, 1) == "X");

    commandInfo.add_uris()->MergeFrom(uri);
  }

  CHECK(os::hasenv("MESOS_WORK_DIRECTORY"))
    << "Missing MESOS_WORK_DIRECTORY environment variable";
  std::string directory = os::getenv("MESOS_WORK_DIRECTORY");

  // We cannot use Some in the ternary expression because the compiler needs to
  // be able to infer the type, thus the explicit Option<string>.
  // TODO(idownes): Add an os::hasenv that returns an Option<string>.
  Option<std::string> user = os::hasenv("MESOS_USER")
    ? Option<std::string>(os::getenv("MESOS_USER")) // Explicit so it compiles.
    : None();

  // Fetch each URI to a local file, chmod, then chown if a user is provided.
  foreach (const CommandInfo::URI& uri, commandInfo.uris()) {
    // Fetch the URI to a local file.
    Try<string> fetched = fetch(uri.value(), directory);
    if (fetched.isError()) {
      EXIT(1) << "Failed to fetch: " << uri.value();
    }

    // Chmod the fetched URI if it's executable, else assume it's an archive
    // that should be extracted.
    if (uri.executable()) {
      Try<Nothing> chmod = os::chmod(
          fetched.get(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
      if (chmod.isError()) {
        EXIT(1) << "Failed to chmod " << fetched.get() << ": " << chmod.error();
      }
    } else if (uri.extract()) {
      //TODO(idownes): Consider removing the archive once extracted.
      // Try to extract the file if it's recognized as an archive.
      Try<bool> extracted = extract(fetched.get(), directory);
      if (extracted.isError()) {
        EXIT(1) << "Failed to extract "
                << fetched.get() << ":" << extracted.error();
      }
    } else {
      LOG(INFO) << "Skipped extracting path '" << fetched.get() << "'";
    }

    // Recursively chown the directory if a user is provided.
    if (user.isSome()) {
      Try<Nothing> chowned = os::chown(user.get(), directory);
      if (chowned.isError()) {
        EXIT(1) << "Failed to chown " << directory << ": " << chowned.error();
      }
    }
  }

  return 0;
}