Skip to content

Commit 0037b25

Browse files
committed
Add trivial resource profile example.
1 parent 96cdeb4 commit 0037b25

File tree

1 file changed

+38
-0
lines changed

1 file changed

+38
-0
lines changed
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package com.highperformancespark.examples.gpu
2+
3+
import org.apache.spark.sql.SparkSession
4+
import org.apache.spark.resource._
5+
import org.apache.spark.resource.ResourceProfileBuilder
6+
import org.apache.spark.TaskContext
7+
8+
object GPUResourceProfileExample {
9+
def main(args: Array[String]): Unit = {
10+
val spark = SparkSession.builder()
11+
.appName("GPUResourceProfileExample")
12+
.getOrCreate()
13+
run(spark)
14+
}
15+
16+
def run(spark: SparkSession) = {
17+
val sc = spark.sparkContext
18+
//tag::gpuResourceProfileExample[]
19+
// Create a resource profile requesting 2 NVIDIA GPUs per executor and 1 per task
20+
val gpuResourceProfile = new ResourceProfileBuilder()
21+
.require(new ExecutorResourceRequests().resource("gpu", 2))
22+
.require(new TaskResourceRequests().resource("gpu", 1))
23+
.build()
24+
25+
// Use resource profile to run on a machine with GPUs.
26+
val rdd = sc.parallelize(1 to 4, 4)
27+
.withResources(gpuResourceProfile)
28+
.map { i =>
29+
// Do some special GPU stuff here my friend
30+
i
31+
}
32+
//end::gpuResourceProfileExample[]
33+
34+
rdd.collect().foreach(println)
35+
36+
spark.stop()
37+
}
38+
}

0 commit comments

Comments
 (0)