Scala -如何合并HDFS位置的增量文件

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}

object IncrementalFileMerge {
  def mergeIncrementalFiles(sourcePaths: Array[String], targetPath: String): Unit = {
    val conf = new Configuration()
    val fs = FileSystem.get(conf)

    val sourceFiles = sourcePaths.map(new Path(_))
    val targetFile = new Path(targetPath)

    // 创建目标文件
    fs.create(targetFile)

    // 合并增量文件
    FileUtil.copyMerge(fs, sourceFiles, fs, targetFile, false, conf, null)
  }

  def main(args: Array[String]): Unit = {
    val sourcePaths = Array("/path/to/incremental/file1", "/path/to/incremental/file2")
    val targetPath = "/path/to/merged/file"

    mergeIncrementalFiles(sourcePaths, targetPath)
  }
}

name := "IncrementalFileMerge"
version := "1.0"
scalaVersion := "2.13.5"

libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "3.3.1"