如何在使用spark批处理应用程序从亚马逊s3读取输入数据时读取每个文件的最后修改/到达时间

val data = spark.read.text("s3a://your-bucket-name/path/to/files")

import org.apache.hadoop.fs.{FileSystem, Path}

val hadoopConf = spark.sparkContext.hadoopConfiguration
val fs = FileSystem.get(hadoopConf)
val filePath = new Path("s3a://your-bucket-name/path/to/files")

val fileStatus = fs.getFileStatus(filePath)
val modificationTime = fileStatus.getModificationTime
val accessTime = fileStatus.getAccessTime

val fileTimes = data.rdd.map { row =>
  val filePath = new Path(row.getString(0))
  val fileStatus = fs.getFileStatus(filePath)
  val modificationTime = fileStatus.getModificationTime
  val accessTime = fileStatus.getAccessTime
  (filePath.toString, modificationTime, accessTime)
}