package com.cntaiping.tpa.storage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.net.URI;
public class GetFileDemo {
public static void main(String[] args) throws IOException{
String hdfsPath="hdfs://192.168.1.180:8020/a/b.txt";
String localPath="D:/copy.txt";
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
Path hdfs_path = new Path(hdfsPath);
Path local_path = new Path(localPath);
fs.copyToLocalFile(hdfs_path, local_path);
fs.close();
}
}
(null) entry in command string: null chmod 0644
log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.io.IOException: (null) entry in command string: null chmod 0644 D:\copy.txt
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:770)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:866)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:849)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:733)
at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:225)
at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:209)
at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:307)
at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:296)
at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:328)
at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.<init>(ChecksumFileSystem.java:398)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:461)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:440)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:911)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:892)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:789)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:365)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:338)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:289)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:2034)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:2003)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:1979)
at com.cntaiping.tpa.storage.GetFileDemo.main(GetFileDemo.java:16)
fs.copyToLocalFile(false,hdfs_path, local_path,true);
说明: 第一个false参数表示不删除源文件,第4个true参数表示使用本地原文件系统,因为这个Demo程序是在Windows系统下运行的。
查看官方API http://hadoop.apache.org/docs/r2.7.6/api/
public void copyToLocalFile(boolean delSrc,
Path src,
Path dst,
boolean useRawLocalFileSystem)
throws IOException
The src file is under FS, and the dst is on the local disk. Copy it from FS control to the local dst name. delSrc indicates if the src will be removed or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem as local file system or not. RawLocalFileSystem is non crc file system.So, It will not create any crc files at local.
Parameters:
delSrc - whether to delete the src
src - path
dst - path
useRawLocalFileSystem - whether to use RawLocalFileSystem as local file system or not.
Throws:
IOException - - if any IO error