Merge pull request apache#33 from schintap/YSPARK-1595-followup

[YSPARK-1595][FOLLOWUP] Add extraClassPath option to the rhel8 images for oozie workflow
redsanket · Jun 11, 2020 · 89bda14 · 89bda14
2 parents 37d0a0b + 2a45e8d
commit 89bda14
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/src/main/resources/oozie/spark_ml_image_integration/job.properties b/src/main/resources/oozie/spark_ml_image_integration/job.properties
@@ -1,6 +1,7 @@
 nameNode=hdfs://axonitered-nn1.red.ygrid.yahoo.com:8020
 jobTracker=axonitered-jt1.red.ygrid.yahoo.com:8032
 sparkTag=spark_latest
+extraClassPath=/opt/jars/spark3/
 wfRoot=spark-test-framework
 wfSuite=spark_ml_image_integration
 oozie.libpath=/user/${user.name}/${wfRoot}/lib

diff --git a/src/main/resources/oozie/spark_ml_image_integration/workflow.xml b/src/main/resources/oozie/spark_ml_image_integration/workflow.xml
@@ -36,7 +36,7 @@
             <mode>cluster</mode>
             <name>SparkRhel8MLImageIntegration</name>
             <jar>test_mlbundle_cpu_image.py</jar>
-            <spark-opts>--driver-memory 20g --executor-memory 3g --executor-cores 1 --queue default --conf spark.oath.dockerImage=ml/rhel8_mlbundle:test</spark-opts>
+            <spark-opts>--driver-memory 20g --executor-memory 3g --executor-cores 1 --queue default --conf spark.oath.dockerImage=ml/rhel8_mlbundle:test --conf spark.driver.extraClassPath=${extraClassPath} --conf spark.executor.extraClassPath=${extraClassPath}</spark-opts>
         </spark>
         <ok to="end" />
         <error to="fail" />