从Python airflow dag代码中调用Spark Scala函数

from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime

default_args = {
    'owner': 'your_name',
    'start_date': datetime(2022, 1, 1),
}

dag = DAG('spark_scala_example', default_args=default_args, schedule_interval=None)

run_spark_scala = BashOperator(
    task_id='run_spark_scala',
    bash_command='/path/to/your_script.sh',
    dag=dag
)

run_spark_scala

from airflow import DAG
from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator
from datetime import datetime

default_args = {
    'owner': 'your_name',
    'start_date': datetime(2022, 1, 1),
}

dag = DAG('spark_scala_example', default_args=default_args, schedule_interval=None)

run_spark_scala = SparkSubmitOperator(
    task_id='run_spark_scala',
    application='/path/to/your_spark_app.jar',
    name='your_spark_app',
    conn_id='spark_default',
    dag=dag
)

run_spark_scala