Skip to content

Commit

Permalink
Remove System.setProperty calls in DistributedSuite
Browse files Browse the repository at this point in the history
  • Loading branch information
JoshRosen committed Dec 24, 2014
1 parent cfe9cce commit 3f2f955
Showing 1 changed file with 5 additions and 7 deletions.
12 changes: 5 additions & 7 deletions core/src/test/scala/org/apache/spark/DistributedSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,12 @@ import org.scalatest.Matchers
import org.scalatest.time.{Millis, Span}

import org.apache.spark.storage.{RDDBlockId, StorageLevel}
import org.apache.spark.util.ResetSystemProperties

class NotSerializableClass
class NotSerializableExn(val notSer: NotSerializableClass) extends Throwable() {}


class DistributedSuite extends FunSuite with Matchers with ResetSystemProperties
with LocalSparkContext {
class DistributedSuite extends FunSuite with Matchers with LocalSparkContext {

val clusterUrl = "local-cluster[2,1,512]"

Expand Down Expand Up @@ -79,8 +77,8 @@ class DistributedSuite extends FunSuite with Matchers with ResetSystemProperties
}

test("groupByKey where map output sizes exceed maxMbInFlight") {
System.setProperty("spark.reducer.maxMbInFlight", "1")
sc = new SparkContext(clusterUrl, "test")
val conf = new SparkConf().set("spark.reducer.maxMbInFlight", "1")
sc = new SparkContext(clusterUrl, "test", conf)
// This data should be around 20 MB, so even with 4 mappers and 2 reducers, each map output
// file should be about 2.5 MB
val pairs = sc.parallelize(1 to 2000, 4).map(x => (x % 16, new Array[Byte](10000)))
Expand Down Expand Up @@ -214,8 +212,8 @@ class DistributedSuite extends FunSuite with Matchers with ResetSystemProperties
}

test("compute when only some partitions fit in memory") {
System.setProperty("spark.storage.memoryFraction", "0.01")
sc = new SparkContext(clusterUrl, "test")
val conf = new SparkConf().set("spark.storage.memoryFraction", "0.01")
sc = new SparkContext(clusterUrl, "test", conf)
// data will be 4 million * 4 bytes = 16 MB in size, but our memoryFraction set the cache
// to only 5 MB (0.01 of 512 MB), so not all of it will fit in memory; we use 20 partitions
// to make sure that *some* of them do fit though
Expand Down

0 comments on commit 3f2f955

Please sign in to comment.