Skip to content
Snippets Groups Projects
Commit a5a6b8b8 authored by edvs19's avatar edvs19
Browse files

teste

parent 4f80b702
No related branches found
No related tags found
No related merge requests found
......@@ -17,10 +17,8 @@ repo_home = '/opt/airflow/dags/repo'
"input_file": Param("fichas_cadastro_individual_5000.jsonl", type="string"),
},
)
def process_records():
@task
def task_spark(params: dict):
return SparkSubmitOperator(
def process_records(params: dict):
task_spark = SparkSubmitOperator(
task_id="landing_to_bronze",
application=f"{repo_home}/gen_bronze.py",
application_args=[
......@@ -29,16 +27,12 @@ def process_records():
packages="org.apache.hadoop:hadoop-aws:3.3.4"
)
@task
def task_dbt_setup():
return BashOperator(
task_dbt_setup = BashOperator(
task_id="dbt_setup",
bash_command=f"rm -rf /tmp/proj_teste && cp -r {repo_home}/proj_teste /tmp/proj_teste",
)
@task
def task_bronze_to_gold():
return BashOperator(
task_bronze_to_gold = BashOperator(
task_id="bronze_to_silver_to_gold",
cwd="/tmp/proj_teste",
bash_command="dbt deps && dbt build",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment