Java FileStatus.getGroup方法代码示例

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path path) throws IOException {
    DatasetJsonRecord record = null;
    if (!fs.exists(path))
        LOG.error(" File Path: " + path.toUri().getPath() + " is not exist in HDFS");
    else {
        try {
            LOG.info("xmlfileanalyzer start parse xml schema, path is {}" , path.toUri().getPath());
            startParseXML(path);
            FileStatus status = fs.getFileStatus(path);
            // replace "\" to  "\\"
            String schemaString = getXMLSchema().replace("\\","\\"+"\\");
            LOG.info("xml file schemaString is {} " , schemaString);
            String storage = STORAGE_TYPE;
            String abstractPath = path.toUri().getPath();
            String codec = "xml.codec";
            record = new DatasetJsonRecord(schemaString, abstractPath, status.getModificationTime(), status.getOwner(), status.getGroup(),
                    status.getPermission().toString(), codec, storage, "");
        } catch (Exception e) {
            LOG.error("path : {} content " + " is not XML File format content  ",path.toUri().getPath());
            LOG.info(e.getStackTrace().toString());
        }

    }
    return record;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path path) throws IOException {
    DatasetJsonRecord record = null;
    if (!fs.exists(path))
        LOG.error("file path : {} not in hdfs", path);
    else {
        try {
            ParquetMetadata readFooter = ParquetFileReader.readFooter(fs.getConf(), path, ParquetMetadataConverter.NO_FILTER);
            Map<String, String> schema = readFooter.getFileMetaData().getKeyValueMetaData();
            String allFields = schema.get("org.apache.spark.sql.parquet.row.metadata");
            FileStatus status = fs.getFileStatus(path);
            String storage = STORAGE_TYPE;
            String abstractPath = path.toUri().getPath();
            String codec = "parquet.codec";
            record = new DatasetJsonRecord(allFields, abstractPath, status.getModificationTime(), status.getOwner(), status.getGroup(),
                    status.getPermission().toString(), codec, storage, "");
            LOG.info("parquetfileanalyzer parse path :{},schema is {}", path.toUri().getPath(), record.toCsvString());

        } catch (Exception e) {
            LOG.error("path : {} content " + " is not Parquet File format content  ", path.toUri().getPath());
            LOG.info(e.getStackTrace().toString());
        }
    }
    return record;

}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path targetFilePath)
        throws IOException {
    DatasetJsonRecord datasetJsonRecord = null;
    try {
        Reader orcReader = OrcFile.createReader(fs, targetFilePath);
        String codec = String.valueOf(orcReader.getCompression());
        String schemaString = orcReader.getObjectInspector().getTypeName();
        String storage = STORAGE_TYPE;
        String abstractPath = targetFilePath.toUri().getPath();
        FileStatus fstat = fs.getFileStatus(targetFilePath);
        datasetJsonRecord =
                new DatasetJsonRecord(schemaString, abstractPath, fstat.getModificationTime(), fstat.getOwner(), fstat.getGroup(),
                        fstat.getPermission().toString(), codec, storage, "");
    } catch (Exception e) {
        LOG.error("path : {} content " + " is not ORC File format content  ",targetFilePath.toUri().getPath());
        LOG.info(e.getStackTrace().toString());
    }

    return datasetJsonRecord;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@BeforeClass
public static void makeTestFile() throws Exception {
  Configuration conf = new Configuration();
  fs = FileSystem.getLocal(conf).getRaw();
  testFilePathIs =
      new File((new Path("target", TestSecureIOUtils.class.getSimpleName()
          + "1")).toUri().getRawPath());
  testFilePathRaf =
      new File((new Path("target", TestSecureIOUtils.class.getSimpleName()
          + "2")).toUri().getRawPath());
  testFilePathFadis =
      new File((new Path("target", TestSecureIOUtils.class.getSimpleName()
          + "3")).toUri().getRawPath());
  for (File f : new File[] { testFilePathIs, testFilePathRaf,
      testFilePathFadis }) {
    FileOutputStream fos = new FileOutputStream(f);
    fos.write("hello".getBytes("UTF-8"));
    fos.close();
  }

  FileStatus stat = fs.getFileStatus(
      new Path(testFilePathIs.toString()));
  // RealOwner and RealGroup would be same for all three files.
  realOwner = stat.getOwner();
  realGroup = stat.getGroup();
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
private static void updateDestStatus(FileStatus src, FileStatus dst,
    EnumSet<FileAttribute> preseved, FileSystem destFileSys
    ) throws IOException {
  String owner = null;
  String group = null;
  if (preseved.contains(FileAttribute.USER)
      && !src.getOwner().equals(dst.getOwner())) {
    owner = src.getOwner();
  }
  if (preseved.contains(FileAttribute.GROUP)
      && !src.getGroup().equals(dst.getGroup())) {
    group = src.getGroup();
  }
  if (owner != null || group != null) {
    destFileSys.setOwner(dst.getPath(), owner, group);
  }
  if (preseved.contains(FileAttribute.PERMISSION)
      && !src.getPermission().equals(dst.getPermission())) {
    destFileSys.setPermission(dst.getPath(), src.getPermission());
  }
  if (preseved.contains(FileAttribute.TIMES)) {
    destFileSys.setTimes(dst.getPath(), src.getModificationTime(), src.getAccessTime());
  }
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path path) throws IOException {
    DatasetJsonRecord record = null;
    if (!fs.exists(path))
        LOG.error("file path : {} not in hdfs", path);
    else {
        try {
            RCFile.Reader reader = new RCFile.Reader(fs, path, fs.getConf());
            Map<Text, Text> meta = reader.getMetadata().getMetadata();
            /** rcfile column number */
            int columnNumber = Integer.parseInt(meta.get(new Text(COLUMN_NUMBER_KEY)).toString());
            FileStatus status = fs.getFileStatus(path);
            String schemaString = getRCFileSchema(columnNumber);
            String storage = STORAGE_TYPE;
            String abstractPath = path.toUri().getPath();
            String codec = "rc.codec";
            record = new DatasetJsonRecord(schemaString, abstractPath, status.getModificationTime(), status.getOwner(), status.getGroup(),
                    status.getPermission().toString(), codec, storage, "");
            LOG.info("rc file : {} schema is {}", path.toUri().getPath(), schemaString);
        } catch (Exception e) {
            LOG.error("path : {} content " + " is not RC File format content  ", path.toUri().getPath());
            LOG.info(e.getStackTrace().toString());
        }
    }

    return record;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path targetFilePath)
        throws IOException {
    LOG.info("avro file path : " + targetFilePath.toUri().getPath());
    try {
        SeekableInput sin = new FsInput(targetFilePath, fs.getConf());
        DataFileReader<GenericRecord> reader =
                new DataFileReader<GenericRecord>(sin, new GenericDatumReader<GenericRecord>());
        String codec = reader.getMetaString("avro.codec");
        long record_count = reader.getBlockCount();

        String schemaString = reader.getSchema().toString();
        String storage = STORAGE_TYPE;
        String abstractPath = targetFilePath.toUri().getPath();
        System.out.println("the schema string is: " + schemaString);
        System.out.println("the abstract path is: " + abstractPath);
      
        FileStatus fstat = fs.getFileStatus(targetFilePath);
        DatasetJsonRecord datasetJsonRecord =
                new DatasetJsonRecord(schemaString, abstractPath, fstat.getModificationTime(), fstat.getOwner(), fstat.getGroup(),
                        fstat.getPermission().toString(), codec, storage, "");
        reader.close();
        sin.close();
        LOG.info("Avro file datasetjsonrecorc get success, it is : " + datasetJsonRecord);
        return datasetJsonRecord;
    } catch (Exception e) {
        LOG.info("AvroAnalyzer get datasetjson failure, and exception is " + e.getMessage());
        return null;
    }

}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path targetFilePath)
        throws IOException {
    String filePath = targetFilePath.toUri().getPath();
    System.out.println("[getSchema] HiveExportFile path : " + filePath);
    // give it a try.
    if (!filePath.contains("000000_0")) return null;
    // if (!filePath.equalsIgnoreCase("/project/T405/out/000000_0")) return null;

    InputStream inputStream = fs.open(targetFilePath);
    BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
    String str;
    int columnNum = 0;
    while((str = bufferedReader.readLine()) != null) {
        columnNum = str.split(delemiter).length;
        System.out.println(String.format("the first column string is: %s", str));
        break;
    }
    // debug.
    System.out.println("the number of column is: " + columnNum);

    inputStream.close();
    bufferedReader.close();
    // if the number of column is zero, file format unmatched.
    if (columnNum == 1) return null;

    String codec = "plain.codec";
    String schemaString = "{\"fields\": [{\"name\": \"name\", \"type\": \"string\"}, {\"name\": \"age\", \"type\": \"int\"}], \"name\": \"Result\", \"namespace\": \"com.tencent.thomas\", \"type\": \"record\"}";
    String storage = STORAGE_TYPE;
    String abstractPath = targetFilePath.toUri().getPath();

    System.out.println("current file is: " + filePath);
    FileStatus fstat = fs.getFileStatus(targetFilePath);
    DatasetJsonRecord datasetJsonRecord =
            new DatasetJsonRecord(schemaString, abstractPath, fstat.getModificationTime(), fstat.getOwner(), fstat.getGroup(),
                    fstat.getPermission().toString(), codec, storage, "");
    return datasetJsonRecord;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path path) throws IOException {
    DatasetJsonRecord record = null;
    if (!fs.exists(path))
        LOG.error("sequencefileanalyzer file : " + path.toUri().getPath() + " is not exist on hdfs");
    else {
        try {
            LOG.info("sequencefileanalyzer start parse schema for  file path : {}", path.toUri().getPath());
            SequenceFile.Reader reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(path));
            String keyName = "Key";
            String keyType = getWritableType(reader.getKeyClassName());
            String valueName = "Value";
            String valueType = getWritableType(reader.getValueClassName());
            FileStatus status = fs.getFileStatus(path);
            String storage = STORAGE_TYPE;
            String abstractPath = path.toUri().getPath();
            String codec = "sequence.codec";
            String schemaString = "{\"fields\": [{\"name\": \"" + keyName + "\", \"type\": \"" + keyType + "\"}, {\"name\": \"" + valueName + "\", \"type\": \"" + valueType + "\"}], \"name\": \"Result\", \"namespace\": \"com.tencent.lake\", \"type\": \"record\"}";

            record = new DatasetJsonRecord(schemaString, abstractPath, status.getModificationTime(), status.getOwner(), status.getGroup(),
                    status.getPermission().toString(), codec, storage, "");
            LOG.info("sequencefileanalyzer parse path :{},schema is {}", path.toUri().getPath(), record.toCsvString());

        } catch (Exception e) {
            LOG.error("path : {} content " + " is not Sequence File format content  ",path.toUri().getPath());
            LOG.info(e.getStackTrace().toString());
        }

    }
    return record;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
public DatasetJsonRecord getSchema(Path targetFilePath) throws IOException {
    DatasetJsonRecord datasetJsonRecord = null;
    try {
        StringBuilder JsonObjectList = new StringBuilder();
        List lsList = this.getLineToData(targetFilePath, 1);
        String[] lsString = (String[]) lsList.get(0);
        for (String realName : lsString) {
            if (realName.indexOf("\"")>=0){
                JsonObjectList.append("{\"name\": " + realName + ", \"type\": \"string\"},");
            }else {
                JsonObjectList.append("{\"name\": \"" + realName + "\", \"type\": \"string\"},");
            }
        }
        JsonObjectList.deleteCharAt(JsonObjectList.length() - 1);
        String schemaString = "{\"fields\":[" + JsonObjectList + "],\"name\": \"Result\", \"namespace\": \"com.tencent.thomas\", \"type\": \"record\"}";
        String codec = "csv.codec";
        String storage = STORAGE_TYPE;
        String abstractPath = targetFilePath.toUri().getPath();
        FileStatus fstat = fs.getFileLinkStatus(targetFilePath);
        datasetJsonRecord =
                new DatasetJsonRecord(schemaString, abstractPath, fstat.getModificationTime(), fstat.getOwner(), fstat.getGroup(),
                        fstat.getPermission().toString(), codec, storage, "");
        LOG.info("csv schma get success , it is {}", datasetJsonRecord.toCsvString());
    } catch (Exception e) {
        LOG.error("path : {} content " + " is not CSV File format content  ",targetFilePath.toUri().getPath());
        LOG.info(e.getStackTrace().toString());
    }
    return datasetJsonRecord;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
@Override
public DatasetJsonRecord getSchema(Path targetFilePath) throws IOException {
    StringBuilder JsonObjectList = new StringBuilder();
    DatasetJsonRecord datasetJsonRecord = null;
    try {
        for (String realName : this.json2Array(getJsonObject(targetFilePath), "schema")) {
            if (realName.charAt(0) == '$') {
                JsonObjectList.append("{\"name\": \"" + realName.substring(1, realName.length()) + "\", \"type\": \"int\"},");
            } else {
                JsonObjectList.append("{\"name\": \"" + realName + "\", \"type\": \"string\"},");
            }
        }
        JsonObjectList.deleteCharAt(JsonObjectList.length() - 1);
        String schemaString = "{\"fields\":[" + JsonObjectList + "],\"name\": \"Result\", \"namespace\": \"com.tencent.thomas\", \"type\": \"record\"}";
        String codec = "json.codec";
        String storage = STORAGE_TYPE;
        String abstractPath = targetFilePath.toUri().getPath();
        FileStatus fstat = fs.getFileLinkStatus(targetFilePath);

        datasetJsonRecord =
                new DatasetJsonRecord(schemaString, abstractPath, fstat.getModificationTime(), fstat.getOwner(), fstat.getGroup(),
                        fstat.getPermission().toString(), codec, storage, "");
    } catch (Exception e) {
        LOG.error("path : {} content " + " is not JSON File format content  ",targetFilePath.toUri().getPath());
        LOG.info(e.getStackTrace().toString());
    }

    return datasetJsonRecord;
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
/**
 * Converts a Hadoop {@link FileStatus} instance into a protobuf
 * {@link DFSProtos.FileStatus}
 *
 * @param status
 *          the Hadoop status instance to convert
 * @return a protobuf status instance
 * @throws IOException
 */
static DFS.FileStatus toProtoFileStatus(FileStatus status) throws IOException {
  DFS.FileStatus.Builder builder = DFS.FileStatus.newBuilder();

  builder
    .setLength(status.getLen())
    .setIsDirectory(status.isDirectory())
    .setBlockReplication(status.getReplication())
    .setBlockSize(status.getBlockSize())
    .setModificationTime(status.getModificationTime())
    .setAccessTime(status.getAccessTime());

  // Handling potential null values
  if (status.getPath() != null) {
    builder = builder.setPath(status.getPath().toUri().getPath());
  }
  if (status.getPermission() != null) {
    builder = builder.setPermission(status.getPermission().toExtendedShort());
  }
  if (status.getOwner() != null) {
    builder = builder.setOwner(status.getOwner());
  }
  if (status.getGroup() != null) {
    builder = builder.setGroup(status.getGroup());
  }
  if (status.isSymlink()) {
    builder = builder.setSymlink(status.getSymlink().toString());
  }

  return builder.build();
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
private static FileStatus transform(FileStatus input, String bucket) {
  String relativePath = removeLeadingSlash(Path.getPathWithoutSchemeAndAuthority(input.getPath()).toString());
  Path bucketPath  = new Path(Path.SEPARATOR + bucket);
  Path fullPath = Strings.isEmpty(relativePath) ? bucketPath : new Path(bucketPath, relativePath);
  return new FileStatus(input.getLen(),
          input.isDirectory(),
          input.getReplication(),
          input.getBlockSize(),
          input.getModificationTime(),
          input.getAccessTime(),
          input.getPermission(),
          input.getOwner(),
          input.getGroup(),
          fullPath);
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
public IndexDirectory(FileStatus fileStatus) {
    this.path = fileStatus.getPath().toString().replaceFirst("hdfs:\\/\\/.+:\\d{4,6}",""); // don't ask
    this.type = IndexEntry.TYPE_DIRECTORY;
    this.time = fileStatus.getModificationTime();
    this.rights = new Short(fileStatus.getPermission().toShort()).toString();
    this.user = fileStatus.getOwner();
    this.group = fileStatus.getGroup();
    this.name = fileStatus.getPath().getName();
}
 

import org.apache.hadoop.fs.FileStatus; //导入方法依赖的package包/类
public IndexFile(FileStatus fileStatus, long offset, String part) {
    String qualifiedPath = getPathRelativeToRootDirectory(fileStatus);
    this.path = qualifiedPath;
    this.type = IndexEntry.TYPE_FILE;
    this.part = part;
    this.offset = offset;
    this.length = fileStatus.getLen();
    this.time = fileStatus.getModificationTime();
    this.rights = new Short(fileStatus.getPermission().toShort()).toString();
    this.user = fileStatus.getOwner();
    this.group = fileStatus.getGroup();
    this.name = fileStatus.getPath().getName();
}