Spark作业读取dataframe中排序的AVRO文件，但在没有命令的情况下写入kafka

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.avro._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.streaming._

val spark = SparkSession.builder()
  .appName("Spark AVRO to Kafka")
  .master("local[*]")  // 根据实际情况设置master
  .getOrCreate()

val avroDF = spark.read
  .format("avro")
  .load("path/to/sorted_avro_file.avro")

val kafkaDF = avroDF
  .select(to_json(struct(avroDF.columns.map(col): _*)).alias("value"))

val kafkaParams = Map(
  "bootstrap.servers" -> "kafka_broker1:9092,kafka_broker2:9092",
  "key.serializer" -> "org.apache.kafka.common.serialization.StringSerializer",
  "value.serializer" -> "org.apache.kafka.common.serialization.StringSerializer",
  "topic" -> "your_topic_name"
)

kafkaDF
  .write
  .format("kafka")
  .options(kafkaParams)
  .save()