Get the JNI examples working with modern versions of the tools.

* Try and debug native builds some more

* Slightly adjust CMakeLists.txt and build.sbt

* Enable fortran because life is sad.

* JDK11+ seems reasonable for "high performance"

Fix the library we are loading.

---------

Co-authored-by: Grigory Pomadchin <[email protected]>

.github/workflows/ci.yml

@@ -10,7 +10,6 @@ jobs:
         include:
           - java: 17
           - java: 11
-          - java: 8
     runs-on: ubuntu-latest
     steps:
     - name: Checkout

.gitignore

@@ -11,6 +11,7 @@ lib_managed/
 src_managed/
 project/boot/
 project/plugins/project/
+.bsp
 
 # Scala-IDE specific
 .scala_dependencies
@@ -23,6 +24,15 @@ project/plugins/project/
 *~
 sbt/*launch*.jar
 
+# VSCode specific
+.vscode
+.history
+
+# Metals
+.metals
+.bloop
+metals.sbt
+
 # python
 *.pyc
 .tox

build.sbt

@@ -1,3 +1,7 @@
+lazy val root = (project in file("."))
+  .aggregate(core, native)
+
+
 organization := "com.highperformancespark"
 
 //tag::addSparkScalaFix[]
@@ -22,14 +26,20 @@ name := "examples"
 publishMavenStyle := true
 
 version := "0.0.1"
+resolvers ++= Seq(
+  "JBoss Repository" at "https://repository.jboss.org/nexus/content/repositories/releases/",
+  "Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/",
+  "Apache HBase" at "https://repository.apache.org/content/repositories/releases",
+  "Twitter Maven Repo" at "https://maven.twttr.com/",
+  "scala-tools" at "https://oss.sonatype.org/content/groups/scala-tools",
+  "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/",
+  "Typesafe repository" at "https://repo.typesafe.com/typesafe/releases/",
+  "Second Typesafe repo" at "https://repo.typesafe.com/typesafe/maven-releases/",
+  "Mesosphere Public Repository" at "https://downloads.mesosphere.io/maven",
+  Resolver.sonatypeRepo("public")
+)
 
-javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
-
-parallelExecution in Test := false
-
-fork := true
-
-javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-Djna.nosys=true")
+licenses := Seq("Apache License 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html"))
 
 def specialOptions = {
   // We only need these extra props for JRE>17
@@ -45,56 +55,49 @@ def specialOptions = {
   }
 }
 
-Test / javaOptions ++= specialOptions
 
 val sparkVersion = settingKey[String]("Spark version")
 val sparkTestingVersion = settingKey[String]("Spark testing base version without Spark version part")
 
-// 2.4.5 is the highest version we have with the old spark-testing-base deps
-sparkVersion := System.getProperty("sparkVersion", "3.3.0")
-sparkTestingVersion := "1.4.0"
-
-// additional libraries
-libraryDependencies ++= Seq(
-  "org.apache.spark" %% "spark-core"                % sparkVersion.value,
-  "org.apache.spark" %% "spark-streaming"           % sparkVersion.value,
-  "org.apache.spark" %% "spark-sql"                 % sparkVersion.value,
-  "org.apache.spark" %% "spark-hive"                % sparkVersion.value,
-  "org.apache.spark" %% "spark-hive-thriftserver"   % sparkVersion.value,
-  "org.apache.spark" %% "spark-catalyst"            % sparkVersion.value,
-  "org.apache.spark" %% "spark-yarn"                % sparkVersion.value,
-  "org.apache.spark" %% "spark-mllib"               % sparkVersion.value,
-  "com.holdenkarau" %% "spark-testing-base"         % s"${sparkVersion.value}_${sparkTestingVersion.value}",
-  //tag::scalaLogging[]
-  "com.typesafe.scala-logging" %% "scala-logging" % "3.9.4",
-  //end::scalaLogging[]
-  "net.java.dev.jna" % "jna" % "5.12.1")
-
-
-scalacOptions ++= Seq("-deprecation", "-unchecked")
-
-pomIncludeRepository := { x => false }
-
-resolvers ++= Seq(
-  "JBoss Repository" at "https://repository.jboss.org/nexus/content/repositories/releases/",
-  "Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/",
-  "Apache HBase" at "https://repository.apache.org/content/repositories/releases",
-  "Twitter Maven Repo" at "https://maven.twttr.com/",
-  "scala-tools" at "https://oss.sonatype.org/content/groups/scala-tools",
-  "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/",
-  "Typesafe repository" at "https://repo.typesafe.com/typesafe/releases/",
-  "Second Typesafe repo" at "https://repo.typesafe.com/typesafe/maven-releases/",
-  "Mesosphere Public Repository" at "https://downloads.mesosphere.io/maven",
-  Resolver.sonatypeRepo("public")
-)
-
-licenses := Seq("Apache License 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html"))
-
-// JNI
-
-enablePlugins(JniNative)
 
-sourceDirectory in nativeCompile := sourceDirectory.value
+// Core (non-JNI bits)
+
+lazy val core = (project in file("core")) // regular scala code with @native methods
+  .dependsOn(native % Runtime)
+  .settings(javah / target := (native / nativeCompile / sourceDirectory).value / "include")
+  .settings(sbtJniCoreScope := Compile)
+  .settings(
+    javacOptions ++= Seq("-source", "1.8", "-target", "1.8"),
+    parallelExecution in Test := false,
+    fork := true,
+    javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-Djna.nosys=true"),
+    Test / javaOptions ++= specialOptions,
+    // 2.4.5 is the highest version we have with the old spark-testing-base deps
+    sparkVersion := System.getProperty("sparkVersion", "3.3.0"),
+    sparkTestingVersion := "1.4.0",
+    // additional libraries
+    libraryDependencies ++= Seq(
+      "org.apache.spark" %% "spark-core"                % sparkVersion.value,
+      "org.apache.spark" %% "spark-streaming"           % sparkVersion.value,
+      "org.apache.spark" %% "spark-sql"                 % sparkVersion.value,
+      "org.apache.spark" %% "spark-hive"                % sparkVersion.value,
+      "org.apache.spark" %% "spark-hive-thriftserver"   % sparkVersion.value,
+      "org.apache.spark" %% "spark-catalyst"            % sparkVersion.value,
+      "org.apache.spark" %% "spark-yarn"                % sparkVersion.value,
+      "org.apache.spark" %% "spark-mllib"               % sparkVersion.value,
+      "com.holdenkarau" %% "spark-testing-base"         % s"${sparkVersion.value}_${sparkTestingVersion.value}",
+      //tag::scalaLogging[]
+      "com.typesafe.scala-logging" %% "scala-logging" % "3.9.4",
+      //end::scalaLogging[]
+      "net.java.dev.jna" % "jna" % "5.12.1"),
+    scalacOptions ++= Seq("-deprecation", "-unchecked"),
+    pomIncludeRepository := { x => false },
+  )
+
+// JNI Magic!
+lazy val native = (project in file("native")) // native code and build script
+  .settings(nativeCompile / sourceDirectory := sourceDirectory.value)
+  .enablePlugins(JniNative) // JniNative needs to be explicitly enabled
 
 //tag::xmlVersionConflict[]
 // See https://github.com/scala/bug/issues/12632

native/src/CMakeLists.txt

@@ -0,0 +1,50 @@
+################################################################
+# A minimal CMake file that is compatible with sbt-jni         #
+#                                                              #
+# All settings required by sbt-jni have been marked so, please #
+# add/modify/remove settings to build your specific library.   #
+################################################################
+
+cmake_minimum_required(VERSION 3.12)
+
+option(SBT "Set if invoked from sbt-jni" OFF)
+
+# Define project and related variables
+# (required by sbt-jni) please use semantic versioning
+#
+project (high-performance-spark)
+enable_language(Fortran)
+set(PROJECT_VERSION_MAJOR 0)
+set(PROJECT_VERSION_MINOR 0)
+set(PROJECT_VERSION_PATCH 0)
+
+# Setup JNI
+find_package(JNI REQUIRED)
+if (JNI_FOUND)
+    message (STATUS "JNI include directories: ${JNI_INCLUDE_DIRS}")
+endif()
+
+# Include directories
+include_directories(.)
+include_directories(include)
+include_directories(${JNI_INCLUDE_DIRS})
+
+# Sources
+file(GLOB LIB_SRC
+  "*.c"
+  "*.f95"
+  "*.f*"
+  "*.cc"
+  "*.cpp"
+  "./c/*.c"
+  "./c/*.cpp"
+  "./fortran/*.f95"
+  "./fortran/*.f*"
+)
+
+# Setup installation targets
+# (required by sbt-jni) major version should always be appended to library name
+#
+set (LIB_NAME ${PROJECT_NAME}${PROJECT_VERSION_MAJOR})
+add_library(${LIB_NAME} SHARED ${LIB_SRC})
+install(TARGETS ${LIB_NAME} LIBRARY DESTINATION .)