Hdfs FileSystem Client

Dlimeng

发布于 2023-06-30 14:35:25

2790

一.示例

Java抽象类org.apache.hadoop.fs.FileSystem定义了hadoop的一个文件系统接口。Hadoop中关于文件操作类基本上全部是在"org.apache.hadoop.fs"包中，这些API能够支持的操作包含：打开文件，读写文件，删除文件等。

Hadoop类库中最终面向用户提供的接口类是FileSystem，该类是个抽象类，只能通过来类的get方法得到具体类。

public class HDFSFileSystem {
    public static void main(String[] args) {
        String res = args[0];
        long start,end;
        start = System.currentTimeMillis();
        FileSystem fs = init();
        //String res="/tmp/linkis/hdfs/20201215/6e8fa663-ffa8-4d50-9510-f8cf15518dc0.json";
        try {
            FSDataInputStream in = fs.open(new Path(res));
            IOUtils.copyBytes(in, System.out, 4096, false);

        } catch (IOException e) {
            e.printStackTrace();
        }

        end = System.currentTimeMillis();
        System.out.println("\n start time:" + start+ "; end time:" + end+ "; Run Time:" + (end - start) + "(ms)");
    }

    public static FileSystem init(){
        Configuration conf = new Configuration();
        String hadoopConfDir="/etc/hadoop/conf";
        conf.addResource(new Path(Paths.get(hadoopConfDir,"core-site.xml").toAbsolutePath().toFile().getAbsolutePath()));
        conf.addResource(new Path(Paths.get(hadoopConfDir, "hdfs-site.xml").toAbsolutePath().toFile().getAbsolutePath()));
        conf.addResource(new Path(Paths.get(hadoopConfDir, "yarn-site.xml").toAbsolutePath().toFile().getAbsolutePath()));
        conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
        try {
            FileSystem fs = UserGroupInformation.createRemoteUser("hdfs").doAs(new PrivilegedExceptionAction<FileSystem>() {
                @Override
                public FileSystem run() throws Exception {
                    return FileSystem.get(conf);
                }
            });
            return fs;
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return null;
    }

}