Java URL类代码示例

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
public static ResourceLocalizationSpec newResourceLocalizationSpec(
    LocalResource rsrc, Path path) {
  URL local = ConverterUtils.getYarnUrlFromPath(path);
  ResourceLocalizationSpec resourceLocalizationSpec =
      Records.newRecord(ResourceLocalizationSpec.class);
  resourceLocalizationSpec.setDestinationDirectory(local);
  resourceLocalizationSpec.setResource(rsrc);
  return resourceLocalizationSpec;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
static LocalResourceStatus createLocalResourceStatus() {
  LocalResourceStatus ret =
    recordFactory.newRecordInstance(LocalResourceStatus.class);
  assertTrue(ret instanceof LocalResourceStatusPBImpl);
  ret.setResource(createResource());
  ret.setLocalPath(
      URL.fromPath(
        new Path("file:///local/foo/bar")));
  ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
  ret.setLocalSize(4443L);
  Exception e = new Exception("Dingos.");
  e.setStackTrace(new StackTraceElement[] {
      new StackTraceElement("foo", "bar", "baz", 10),
      new StackTraceElement("sbb", "one", "onm", 10) });
  ret.setException(SerializedException.newInstance(e));
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(
      URL.fromPath((new Path("/tmp" + System.currentTimeMillis()))));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
  try {
    FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

    if (null == resourceFileStatus) {
      throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
    }

    localResource.setResource(resourceUrl);
    log.info("setLocalizerResource for {}", resourceUrl);
    localResource.setSize(resourceFileStatus.getLen());
    localResource.setTimestamp(resourceFileStatus.getModificationTime());
    localResource.setType(resourceType);
    localResource.setVisibility(resourceVisibility);
    return localResource;
  } catch (IOException ioe) {
    log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
    throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
  }

}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(URL.fromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(URL.fromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(URL.fromPath(p));
  return resourceLocalizationSpec;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
private static void addToLocalResources(FileSystem fs, String key, Path dst,
    Map<String, LocalResource> localResources) throws IOException {
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource resource =
      LocalResource.newInstance(
          URL.fromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(key, resource);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
@Override
public synchronized void setResource(URL resource) {
  maybeInitBuilder();
  if (resource == null) 
    builder.clearResource();
  this.url = resource;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
  Path expectedPath = new Path("hdfs://foo.com");
  URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
  Path actualPath = ConverterUtils.getPathFromYarnURL(url);
  assertEquals(expectedPath, actualPath);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入依赖的package包/类
@Test
public void testConvertUrlWithUserinfo() throws URISyntaxException {
  Path expectedPath = new Path("foo://username:[email protected]:8042");
  URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
  Path actualPath = ConverterUtils.getPathFromYarnURL(url);
  assertEquals(expectedPath, actualPath);
}