step1:增加依赖
pom.xml
...
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.2.0</version>
<exclusions>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.2.0</version>
</dependency>
...
step2: 拷贝配置文件
‘hdfs-site.xml’和‘core-site.xml’
step3:测试代码
package cjkjcn.demo.hadoop.hdfs;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.LinkedList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
*
* @author junhong
*
* 2017年5月18日
*/
public class HDFSDao {
private static Configuration conf = new Configuration();
private FileSystem hdfs;
final static String ROOT_PATH = "/user";
public HDFSDao() {
conf.addResource("hdfs-site.xml");
conf.addResource("core-site.xml");
try {
hdfs = FileSystem.get(conf); // 初始化hdfs
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("param size=" + conf.size());
}
/**
* 扫描测试文件是否存在
*/
public void scanFiles() {
try {
Path path = new Path(ROOT_PATH);
System.out.println(hdfs.exists(path));
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 按行读取文本文件
* @param file
* @return
*/
public List<String> lines(String file) {
List<String> list = new LinkedList<>();
Path f = new Path(file);
try {
FSDataInputStream input = hdfs.open(f);
InputStreamReader inr = new InputStreamReader(input);
BufferedReader read = new BufferedReader(inr);
String line;
while ((line = read.readLine()) != null) {
list.add(line);
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
}
注意:
1)若缺少依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.2.0</version>
</dependency>
将导致如下错误!!
java.io.Exception: No FileSystem for scheme: hdfs
2)测试写文件或者创建目录方法接口时,可能会出现权限问题
Pemission denied for test