Skip to content
Snippets Groups Projects
Commit c6b36a89 authored by edvs19's avatar edvs19
Browse files

teste

parent 561e36a6
No related branches found
No related tags found
No related merge requests found
...@@ -19,14 +19,17 @@ with DAG( ...@@ -19,14 +19,17 @@ with DAG(
"input_file": Param("invalid_file.jsonl", type="string"), "input_file": Param("invalid_file.jsonl", type="string"),
}, },
) as dag1: ) as dag1:
task_spark = SparkSubmitOperator(
task_id="landing_to_bronze", @task
application=f"{repo_home}/gen_bronze.py", def task_spark(params: dict):
application_args=[ SparkSubmitOperator(
"{{ params.input_file }}" task_id="landing_to_bronze",
], application=f"{repo_home}/gen_bronze.py",
packages="org.apache.hadoop:hadoop-aws:3.3.4" application_args=[
) params["input_file"]
],
packages="org.apache.hadoop:hadoop-aws:3.3.4"
)
task_dbt_setup = BashOperator( task_dbt_setup = BashOperator(
task_id="dbt_setup", task_id="dbt_setup",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment