diff --git a/kernel/src/universal/bin/setup.sh b/kernel/src/universal/bin/setup.sh index f89289a1..ee626c38 100755 --- a/kernel/src/universal/bin/setup.sh +++ b/kernel/src/universal/bin/setup.sh @@ -5,7 +5,8 @@ SPARKKERNEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )" ASSEMBLYJAR=$(echo $SPARKKERNEL/lib/kernel-assembly-*.jar) -SPARK_HOME=`echo 'import os; print os.environ["SPARK_HOME"]' | dse pyspark` +# TODO - STEVE - fix pyspark for opensource +# SPARK_HOME=`echo 'import os; print os.environ["SPARK_HOME"]' | dse pyspark` if [ `uname` == "Darwin" ]; then JUPYTER_CONFIGS=$HOME/Library/Jupyter/kernels @@ -13,14 +14,14 @@ else JUPYTER_CONFIGS=$HOME/.local/share/jupyter/kernels fi -PY4J=$(ls $SPARK_HOME/python/lib/py4j*.zip) +# TODO - Steve - fix pyspark for opensource +# PY4J=$(ls $SPARK_HOME/python/lib/py4j*.zip) SPARKLOCAL="$JUPYTER_CONFIGS/spark-dse-local" SPARKCLUSTER="$JUPYTER_CONFIGS/spark-dse-cluster" PYSPARKLOCAL="$JUPYTER_CONFIGS/pyspark-dse-local" PYSPARKCLUSTER="$JUPYTER_CONFIGS/pyspark-dse-cluster" - ## File resource doesn't work correctly here because it writes a byte-order mark on the file ## and python can't then parse it @@ -35,10 +36,11 @@ cat >$SPARKLOCAL/kernel.json <$SPARKCLUSTER/kernel.json "display_name": "Spark-DSE Cluster", "language": "scala", "argv": [ - "dse", "spark-submit", + "--packages", + "datastax:spark-cassandra-connector:1.6.0-M2-s_2.10", "$ASSEMBLYJAR", "com.ibm.spark.SparkKernel", "--profile", @@ -66,6 +69,9 @@ cat <$SPARKCLUSTER/kernel.json } EOF +exit 0 + + mkdir -p $PYSPARKLOCAL echo "Creating file $PYSPARKLOCAL/kernel.json" cat <$PYSPARKLOCAL/kernel.json diff --git a/project/Common.scala b/project/Common.scala index b7455ac9..200e6871 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -30,7 +30,7 @@ object Common { val repoEndpoint = Properties.envOrElse("REPO_ENDPOINT", if(snapshot) "/nexus/content/repositories/snapshots/" else "/nexus/content/repositories/releases/") val repoUrl = Properties.envOrElse("REPO_URL", s"http://${repoHost}:${repoPort}${repoEndpoint}") - private val versionNumber = "0.1.6.0-cassandra" + private val versionNumber = "0.1.6.0-opensource" private val buildOrganization = "com.ibm.spark" private val buildVersion = if (snapshot) s"$versionNumber-SNAPSHOT"