在spark scala中获取s3目录的大小

libraryDependencies += "software.amazon.awssdk" % "s3" % "2.17.44"

import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.s3.S3Client
import software.amazon.awssdk.services.s3.model.{GetObjectMetadataRequest, ListObjectsV2Request}

import scala.collection.JavaConverters._

val region = Region.AP_NORTHEAST_1 // 例如，使用亚太地区东京1的区域
val credentialsProvider = DefaultCredentialsProvider.create()
val s3Client = S3Client.builder().region(region).credentialsProvider(credentialsProvider).build()

val bucketName = "your-s3-bucket-name"
val directoryPath = "your-s3-directory-path"

val listObjectsRequest = ListObjectsV2Request.builder()
  .bucket(bucketName)
  .prefix(directoryPath)
  .build()

val objectSummaries = s3Client.listObjectsV2(listObjectsRequest).contents().asScala.toList
val totalSize = objectSummaries.map(_.size()).sum

println(s"The size of directory '$directoryPath' in S3 bucket '$bucketName' is $totalSize bytes.")