File tree Expand file tree Collapse file tree 1 file changed +38
-0
lines changed
core/src/main/scala/com/high-performance-spark-examples/tools Expand file tree Collapse file tree 1 file changed +38
-0
lines changed Original file line number Diff line number Diff line change 1+ package com .highperformancespark .examples .gpu
2+
3+ import org .apache .spark .sql .SparkSession
4+ import org .apache .spark .resource ._
5+ import org .apache .spark .resource .ResourceProfileBuilder
6+ import org .apache .spark .TaskContext
7+
8+ object GPUResourceProfileExample {
9+ def main (args : Array [String ]): Unit = {
10+ val spark = SparkSession .builder()
11+ .appName(" GPUResourceProfileExample" )
12+ .getOrCreate()
13+ run(spark)
14+ }
15+
16+ def run (spark : SparkSession ) = {
17+ val sc = spark.sparkContext
18+ // tag::gpuResourceProfileExample[]
19+ // Create a resource profile requesting 2 NVIDIA GPUs per executor and 1 per task
20+ val gpuResourceProfile = new ResourceProfileBuilder ()
21+ .require(new ExecutorResourceRequests ().resource(" gpu" , 2 ))
22+ .require(new TaskResourceRequests ().resource(" gpu" , 1 ))
23+ .build()
24+
25+ // Use resource profile to run on a machine with GPUs.
26+ val rdd = sc.parallelize(1 to 4 , 4 )
27+ .withResources(gpuResourceProfile)
28+ .map { i =>
29+ // Do some special GPU stuff here my friend
30+ i
31+ }
32+ // end::gpuResourceProfileExample[]
33+
34+ rdd.collect().foreach(println)
35+
36+ spark.stop()
37+ }
38+ }
You can’t perform that action at this time.
0 commit comments