diff --git a/clean.py b/clean.py index a2c4a69bea63514723d3faff9bedf4fd0eef7d24..cc7ad37c6e3b1caa84a435723b7b47269898e663 100644 --- a/clean.py +++ b/clean.py @@ -10,6 +10,7 @@ spark = SparkSession.builder \ .config("spark.hadoop.fs.s3a.path.style.access", "true") \ .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \ .config("spark.hadoop.fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") \ + .config("spark.hadoop.fs.defaultFS", "s3a://") \ # Garantir que o fs padrão seja o s3a .getOrCreate() # Função para limpar os diretórios (buckets)