From a5a6b8b8797c8782d1e4e2cf28ac2d6d15867e33 Mon Sep 17 00:00:00 2001
From: edvs19 <edvs19@inf.ufpr.br>
Date: Thu, 6 Mar 2025 17:10:44 -0300
Subject: [PATCH] teste
---
teste_dag.py | 46 ++++++++++++++++++++--------------------------
1 file changed, 20 insertions(+), 26 deletions(-)
diff --git a/teste_dag.py b/teste_dag.py
index aaa690a..1ef297d 100644
--- a/teste_dag.py
+++ b/teste_dag.py
@@ -17,32 +17,26 @@ repo_home = '/opt/airflow/dags/repo'
"input_file": Param("fichas_cadastro_individual_5000.jsonl", type="string"),
},
)
-def process_records():
- @task
- def task_spark(params: dict):
- return SparkSubmitOperator(
- task_id="landing_to_bronze",
- application=f"{repo_home}/gen_bronze.py",
- application_args=[
- params["input_file"]
- ],
- packages="org.apache.hadoop:hadoop-aws:3.3.4"
- )
-
- @task
- def task_dbt_setup():
- return BashOperator(
- task_id="dbt_setup",
- bash_command=f"rm -rf /tmp/proj_teste && cp -r {repo_home}/proj_teste /tmp/proj_teste",
- )
-
- @task
- def task_bronze_to_gold():
- return BashOperator(
- task_id="bronze_to_silver_to_gold",
- cwd="/tmp/proj_teste",
- bash_command="dbt deps && dbt build",
- )
+def process_records(params: dict):
+ task_spark = SparkSubmitOperator(
+ task_id="landing_to_bronze",
+ application=f"{repo_home}/gen_bronze.py",
+ application_args=[
+ params["input_file"]
+ ],
+ packages="org.apache.hadoop:hadoop-aws:3.3.4"
+ )
+
+ task_dbt_setup = BashOperator(
+ task_id="dbt_setup",
+ bash_command=f"rm -rf /tmp/proj_teste && cp -r {repo_home}/proj_teste /tmp/proj_teste",
+ )
+
+ task_bronze_to_gold = BashOperator(
+ task_id="bronze_to_silver_to_gold",
+ cwd="/tmp/proj_teste",
+ bash_command="dbt deps && dbt build",
+ )
task_spark >> task_dbt_setup >> task_bronze_to_gold
--
GitLab