大数据存储技术（2）—— HDFS分布式文件系统

public static FileSystem.get(Configuration conf) throws IOException
public static FileSystem.get(URI uri, Configuration conf) throws IOException

[root@bigdata zhc]# hdfs dfs
Usage: hadoop fs [generic options]
	[-appendToFile <localsrc> ... <dst>]
	[-cat [-ignoreCrc] <src> ...]
	[-checksum <src> ...]
	[-chgrp [-R] GROUP PATH...]
	[-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...]
	[-chown [-R] [OWNER][:[GROUP]] PATH...]
	[-copyFromLocal [-f] [-p] [-l] [-d] [-t <thread count>] <localsrc> ... <dst>]
	[-copyToLocal [-f] [-p] [-ignoreCrc] [-crc] <src> ... <localdst>]
	[-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] [-e] <path> ...]
	[-cp [-f] [-p | -p[topax]] [-d] <src> ... <dst>]
	[-createSnapshot <snapshotDir> [<snapshotName>]]
	[-deleteSnapshot <snapshotDir> <snapshotName>]
	[-df [-h] [<path> ...]]
	[-du [-s] [-h] [-v] [-x] <path> ...]
	[-expunge]
	[-find <path> ... <expression> ...]
	[-get [-f] [-p] [-ignoreCrc] [-crc] <src> ... <localdst>]
	[-getfacl [-R] <path>]
	[-getfattr [-R] {-n name | -d} [-e en] <path>]
	[-getmerge [-nl] [-skip-empty-file] <src> <localdst>]
	[-head <file>]
	[-help [cmd ...]]
	[-ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] [-e] [<path> ...]]
	[-mkdir [-p] <path> ...]
	[-moveFromLocal <localsrc> ... <dst>]
	[-moveToLocal <src> <localdst>]
	[-mv <src> ... <dst>]
	[-put [-f] [-p] [-l] [-d] <localsrc> ... <dst>]
	[-renameSnapshot <snapshotDir> <oldName> <newName>]
	[-rm [-f] [-r|-R] [-skipTrash] [-safely] <src> ...]
	[-rmdir [--ignore-fail-on-non-empty] <dir> ...]
	[-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]]
	[-setfattr {-n name [-v value] | -x name} <path>]
	[-setrep [-R] [-w] <rep> <path> ...]
	[-stat [format] <path> ...]
	[-tail [-f] [-s <sleep interval>] <file>]
	[-test -[defsz] <path>]
	[-text [-ignoreCrc] <src> ...]
	[-touch [-a] [-m] [-t TIMESTAMP ] [-c] <path> ...]
	[-touchz <path> ...]
	[-truncate [-w] <length> <path> ...]
	[-usage [cmd ...]]

Generic options supported are:
-conf <configuration file>        specify an application configuration file
-D <property=value>               define a value for a given property
-fs <file:///|hdfs://namenode:port> specify default filesystem URL to use, overrides 'fs.defaultFS' property from configurations.
-jt <local|resourcemanager:port>  specify a ResourceManager
-files <file1,...>                specify a comma-separated list of files to be copied to the map reduce cluster
-libjars <jar1,...>               specify a comma-separated list of jar files to be included in the classpath
-archives <archive1,...>          specify a comma-separated list of archives to be unarchived on the compute machines

The general command line syntax is:
command [genericOptions] [commandOptions]

[root@bigdata zhc]# start-dfs.sh
[root@bigdata zhc]# start-yarn.sh

[root@bigdata zhc]# hdfs dfs -help rm

[root@bigdata zhc]# hdfs dfs -ls /

[root@bigdata zhc]# hdfs dfs -mkdir -p /sanguo/shuguo/

[root@bigdata zhc]# touch kongming.txt
[root@bigdata zhc]# hdfs dfs -moveFromLocal ./kongming.txt /sanguo/shuguo

[root@bigdata zhc]# touch liubei.txt
[root@bigdata zhc]# vi liubei.txt
输入
san gu mao lu
[root@bigdata zhc]# hdfs dfs -appendToFile liubei.txt /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -cat /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -chmod 666 /sanguo/shuguo/kongming.txt
[root@bigdata zhc]# hdfs dfs -chown atguigu:atguigu /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -copyFromLocal README.txt /

[root@bigdata zhc]# hdfs dfs -copyToLocal /sanguo/shuguo/kongming.txt ./

[root@bigdata zhc]# hdfs dfs -cp /sanguo/shuguo/kongming.txt /zhuge.txt

[root@bigdata zhc]# hdfs dfs -mv /zhuge.txt /sanguo/shuguo/

[root@bigdata zhc]# hdfs dfs -get /sanguo/shuguo/kongming.txt ./

[root@bigdata zhc]# hdfs dfs -getmerge /user/atguigu/test/* ./zaiyiqi.txt

[root@bigdata zhc]# hdfs dfs -put ./zaiyiqi.txt /user/atguigu/test/

[root@bigdata zhc]# hdfs dfs -tail /sanguo/shuguo/kongming.txt

[root@bigdata zhc]# hdfs dfs -rm /user/atguigu/test/jinlian2.txt

[root@bigdata zhc]# hdfs dfs -mkdir /test
[root@bigdata zhc]# hdfs dfs -rmdir /test

[root@bigdata zhc]# hdfs dfs -du -s -h /user/atguigu/test
2.7 K /user/atguigu/test
[root@bigdata zhc]# hdfs dfs -du -h /user/atguigu/test
1.3 K /user/atguigu/test/README.txt
15 /user/atguigu/test/jinlian.txt
1.4 K /user/atguigu/test/zaiyiqi.txt

[root@bigdata zhc]# hdfs dfs -setrep 10 /sanguo/shuguo/kongming.txt 

@Test
public void testCopyFromLocalFile() throws IOException,InterruptedException,URISyntaxException {
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    configuration.set("dfs.replication", "2");
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"),configuration,"atguigu");
    // 2 上传文件
    fs.copyFromLocalFile(new Path("e:/banzhang.txt"), new Path("/banzhang.txt"));
    // 3 关闭资源
    fs.close();
    System.out.println("over");
}

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>dfs.replication</name>
      <value>1</value>
  </property>
</configuration>

@Test
public void testCopyToLocalFile() throws IOException,InterruptedException,URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu");
    // 2 执行下载操作
    // boolean delSrc 指是否将原文件删除
    // Path src 指要下载的文件路径
    // Path dst 指将文件下载到的路径
    // boolean useRawLocalFileSystem 是否开启文件校验
    fs.copyToLocalFile(false, new Path("/banzhang.txt"), new Path("e:/banhua.txt"), true);
    // 3 关闭资源
    fs.close();
}

@Test
public void testDelete() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu");
    // 2 执行删除
    fs.delete(new Path("/0508/"), true);
    // 3 关闭资源
    fs.close();
}

@Test
public void testRename() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu"); 
    // 2 修改文件名称
    fs.rename(new Path("/banzhang.txt"), new Path("/banhua.txt"));
    // 3 关闭资源
    fs.close();
}

@Test
public void testListFiles() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件系统
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://bigdata:9000"), configuration, "atguigu"); 
    // 2 获取文件详情
    RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
    while(listFiles.hasNext()){
        LocatedFileStatus status = listFiles.next();
        // 输出详情
        // 文件名称
        System.out.println(status.getPath().getName());
        // 长度
        System.out.println(status.getLen());
        // 权限
        System.out.println(status.getPermission());
        // 分组
        System.out.println(status.getGroup());
        // 获取存储的块信息
        BlockLocation[] blockLocations = status.getBlockLocations();
        for (BlockLocation blockLocation : blockLocations) {
            // 获取块存储的主机节点
            String[] hosts = blockLocation.getHosts();
            for (String host : hosts) {
                System.out.println(host);
            }
        }
        System.out.println("-----------班长的分割线----------");
    }
    // 3 关闭资源
    fs.close();
}

@Test
public void testListStatus() throws IOException, InterruptedException, URISyntaxException{
    // 1 获取文件配置信息
    Configuration configuration = new Configuration();
    FileSystem fs =FileSystem.get(newURI("hdfs://bigdata:9000"),configuration,"atguigu");
    // 2 判断是文件还是文件夹
    FileStatus[] listStatus = fs.listStatus(new Path("/"));
    for (FileStatus fileStatus : listStatus) {
        // 如果是文件
        if (fileStatus.isFile()) {
            System.out.println("f:"+fileStatus.getPath().getName());
        }else {
            System.out.println("d:"+fileStatus.getPath().getName());
        }
    }
    // 3 关闭资源
    fs.close();
}