From c6b36a8918e1e5fab2d31dd7d282aafaee40c418 Mon Sep 17 00:00:00 2001
From: edvs19 <edvs19@inf.ufpr.br>
Date: Thu, 6 Mar 2025 19:06:52 -0300
Subject: [PATCH] teste
---
teste_dag.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/teste_dag.py b/teste_dag.py
index 6a50745..faeb6ef 100644
--- a/teste_dag.py
+++ b/teste_dag.py
@@ -19,14 +19,17 @@ with DAG(
"input_file": Param("invalid_file.jsonl", type="string"),
},
) as dag1:
- task_spark = SparkSubmitOperator(
- task_id="landing_to_bronze",
- application=f"{repo_home}/gen_bronze.py",
- application_args=[
- "{{ params.input_file }}"
- ],
- packages="org.apache.hadoop:hadoop-aws:3.3.4"
- )
+
+ @task
+ def task_spark(params: dict):
+ SparkSubmitOperator(
+ task_id="landing_to_bronze",
+ application=f"{repo_home}/gen_bronze.py",
+ application_args=[
+ params["input_file"]
+ ],
+ packages="org.apache.hadoop:hadoop-aws:3.3.4"
+ )
task_dbt_setup = BashOperator(
task_id="dbt_setup",
--
GitLab