diff --git a/dags.py b/dags.py index dc1939231abcc3b02eebb6efac8f4f6356a872c1..c0af3e4e4468588ded6b7e89dd03359733ff8a50 100644 --- a/dags.py +++ b/dags.py @@ -14,27 +14,6 @@ from airflow.operators.python import PythonOperator repo_home = '/opt/airflow/dags/repo' -# Função para limpar o buckets do Minio -def clean_buckets(): - # Configurar cliente do Minio - minio_client = Minio( - "http://minio.minio-cluster.svc.cluster.local:9000", - access_key="minioadmin", - secret_key="minioadmin", - secure=True - ) - - bucket_names = ["bronze", "silver", "gold"] - - for bucket_name in bucket_names: - try: - # Listar objetos no bucket e remover - for obj in minio_client.list_objects(bucket_name): - minio_client.remove_object(bucket_name, obj.object_name) - print(f"Objeto {obj.object_name} removido com sucesso!") - except ResponseError as err: - print(f"Erro ao limpar o bucket: {err}") - @dag() def process_records(): @@ -59,12 +38,7 @@ def process_records(): bash_command="dbt deps && dbt build", ) - task_4 = PythonOperator( - task_id="clean_buckets", - python_callable=clean_buckets - ) - - task_1 >> task_2 >> task_3 >> task_4 + task_1 >> task_2 >> task_3 dag = process_records()