wip

Author: fczuardi-luizalabs

spec/pyspark/script_pyspark.py

@@ -5,13 +5,14 @@
 import os
 import pytest
 
-os.environ['JAVA_HOME'] = '/usr/lib/jvm/java-17-openjdk-amd64'
+# os.environ['JAVA_HOME'] = '/usr/lib/jvm/java-17-openjdk-amd64'
 
 session = boto3.Session()
 credentials = session.get_credentials().get_frozen_credentials()
 
 s3_endpoint_url = 'https://br-se1.magaluobjects.com'
 
+s3 = boto3.client('s3')
 def get_spark_session(path_style_access=True):
     try:
         builder = SparkSession.builder.appName("Demo") \

spec/pyspark/test_basic.py

@@ -0,0 +1,10 @@
+# make sure there is an AWS_PROFILE env var set and a configured aws profile with the name
+from pyspark.sql import SparkSession
+
+def test_pyspark_session():
+    spark = SparkSession.builder.appName("foo").getOrCreate()
+    assert True
+
+if __name__ == "__main__":
+    pytest.main([__file__])
+