使用org.apache.hadoop从pyspark中的s3读取文件

pip install pyspark

pyspark --jars /path/to/hadoop-aws.jar,/path/to/aws-java-sdk-babel.jar

export AWS_ACCESS_KEY_ID=your_aws_access_key_id
export AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key

from pyspark.sql import SparkSession

spark = SparkSession.builder \
    .appName("S3 Read Example") \
    .config("spark.hadoop.fs.s3a.access.key", "your_aws_access_key_id") \
    .config("spark.hadoop.fs.s3a.secret.key", "your_\
aws_secret_access_key") \
    .getOrCreate()

df = spark.read.csv("s3a://your-bucket-name/path/to/your/file.csv", header=True, inferSchema=True)
df.show()