teste

teste_dag.py

@@ -17,8 +17,10 @@ repo_home = '/opt/airflow/dags/repo'
         "input_file": Param("fichas_cadastro_individual_5000.jsonl", type="string"),
     },
 )
-def process_records(params: dict):
-    task_spark = SparkSubmitOperator(
+def process_records():
+    @task
+    def task_spark(params: dict):
+        return SparkSubmitOperator(
             task_id="landing_to_bronze",
             application=f"{repo_home}/gen_bronze.py",
             application_args=[
@@ -27,18 +29,24 @@ def process_records(params: dict):
             packages="org.apache.hadoop:hadoop-aws:3.3.4"
         )
 
-    task_dbt_setup = BashOperator(
+    @task
+    def task_dbt_setup():
+        return BashOperator(
             task_id="dbt_setup",
             bash_command=f"rm -rf /tmp/proj_teste && cp -r {repo_home}/proj_teste /tmp/proj_teste",
         )
 
-    task_bronze_to_gold = BashOperator(
+    @task
+    def task_bronze_to_gold():
+        return BashOperator(
             task_id="bronze_to_silver_to_gold",
             cwd="/tmp/proj_teste",
             bash_command="dbt deps && dbt build",
         )
     
-    task_spark >> task_dbt_setup >> task_bronze_to_gold
+    task1 = task_spark()
+    task2 = task_dbt_setup()
+    task3 = task_bronze_to_gold()
 
 
 dag = process_records()