build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

buildscript {
  repositories {
    jcenter()
    gradlePluginPortal()
    maven { url "http://palantir.bintray.com/releases" }
    maven { url "https://plugins.gradle.org/m2/" }
  }
  dependencies {
    classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0'
    classpath 'com.palantir.baseline:gradle-baseline-java:0.55.0'
    classpath 'com.diffplug.spotless:spotless-plugin-gradle:3.14.0'
    classpath 'gradle.plugin.org.inferred:gradle-processors:2.1.0'
    classpath 'me.champeau.gradle:jmh-gradle-plugin:0.4.8'
    classpath 'com.netflix.nebula:nebula-publishing-plugin:9.5.0'
  }
}

plugins {
  id 'nebula.netflixoss' version '4.1.0'
  id 'com.palantir.git-version' version '0.9.1'
  id 'com.palantir.consistent-versions' version '1.9.2'
}

if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
  throw new GradleException("This build must be run with Java 8")
}

allprojects {
  group = "org.apache.iceberg"
  apply plugin: 'com.palantir.baseline-idea'
  version = gitVersion()
  repositories {
    maven { url  "http://palantir.bintray.com/releases" }
    mavenCentral()
    mavenLocal()
  }
}

apply plugin: 'com.palantir.baseline-config'

subprojects {
  apply plugin: 'nebula.javadoc-jar'
  apply plugin: 'nebula.source-jar'
  apply plugin: 'java'
  apply plugin: 'maven' // make pom files for deployment
  apply plugin: 'nebula.maven-base-publish'

  compileJava {
    options.encoding = "UTF-8"
  }

  compileTestJava {
    options.encoding = "UTF-8"
  }

  configurations {
    testCompile.extendsFrom compileOnly
    all {
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }

  ext {
    jmhVersion = '1.21'
  }

  sourceCompatibility = '1.8'
  targetCompatibility = '1.8'

  dependencies {
    compile 'org.slf4j:slf4j-api'
    compile 'com.google.guava:guava'

    testCompile 'junit:junit'
    testCompile 'org.slf4j:slf4j-simple'
    testCompile 'org.mockito:mockito-core'
  }

  publishing {
    publications {
      nebula(MavenPublication) {
        from components.java
        versionMapping {
          allVariants {
            fromResolutionResult()
          }
        }
      }
    }
  }
}

task aggregateJavadoc(type: Javadoc) {
  // use the branch name in place of version in Javadoc
  project.rootProject.version = versionDetails().branchName

  dependsOn subprojects.javadoc
  source subprojects.javadoc.source
  destinationDir project.rootProject.file("site/docs/javadoc/${versionDetails().branchName}")
  classpath = project.rootProject.files(subprojects.javadoc.classpath)
}

task removeJavadoc(type: Exec) {
  commandLine 'rm', '-rf', "site/docs/javadoc/${versionDetails().branchName}"
}

task refreshJavadoc(type: Exec) {
  dependsOn aggregateJavadoc
  dependsOn removeJavadoc
  aggregateJavadoc.mustRunAfter removeJavadoc
  commandLine 'git', 'add', "site/docs/javadoc/${versionDetails().branchName}"
}

task deploySite(type: Exec) {
  workingDir 'site'
  commandLine 'mkdocs', 'gh-deploy'
}

// The following code is temporary to allow for incrementally migrating projects to use Baseline.

// We enable baseline-idea everywhere so that everyone can use IntelliJ to build code against the
// Baseline style guide.
def baselineProjects = [ project("iceberg-api"), project("iceberg-common"), project("iceberg-core"),
                         project("iceberg-data"), project("iceberg-orc"), project("iceberg-spark"),
                         project("iceberg-hive") ]


configure(subprojects - baselineProjects) {
  // error-prone is brought in with baseline-idea, but we're not prepared to handle error-prone
  // linting errors everywhere yet.
  pluginManager.withPlugin("com.palantir.baseline-error-prone") {
    tasks.withType(JavaCompile).configureEach { task ->
      options.errorprone.enabled = false
    }
  }
}

configure(baselineProjects) {
  // Currently, if any subproject applies the blanket Baseline plugin, it forces the Baseline plugin
  // to be applied to ALL projects. And we are not prepared to address all of the build errors that
  // occur as a result at this time. Furthermore, baseline-format will not work out of the box for
  // us - see below.

  // Thus we concede to applying all of the Baseline plugins individually on all the projects we are
  // ready to enforce linting on.
  apply plugin: 'org.inferred.processors'
  apply plugin: 'com.palantir.baseline-checkstyle'
  apply plugin: 'com.palantir.baseline-error-prone'
  apply plugin: 'com.palantir.baseline-scalastyle'
  apply plugin: 'com.palantir.baseline-class-uniqueness'
  apply plugin: 'com.palantir.baseline-reproducibility'
  apply plugin: 'com.palantir.baseline-exact-dependencies'
  apply plugin: 'com.palantir.baseline-release-compatibility'

  // Can't use the built-in Baseline spotless format because it's opinionated about the import
  // order of having static imports after non-static imports, and this cannot be overridden.

  // So we apply Spotless manually to get a similar effect to baseline-format, but change the
  // import order.
  pluginManager.withPlugin('com.diffplug.gradle.spotless') {
    spotless {
      java {
        target 'src/main/java/**/*.java', 'src/main/test/**/*.java'
        removeUnusedImports()
        importOrder(['', 'static '])
        trimTrailingWhitespace()
        indentWithSpaces 2
        endWithNewline()
      }
    }
  }
}

def jmhProjects = [ project("iceberg-spark") ]

configure(jmhProjects) {
  apply plugin: 'me.champeau.gradle.jmh'

  jmh {
    jmhVersion = jmhVersion
    failOnError = true
    forceGC = true
    includeTests = true
    humanOutputFile = file(jmhOutputPath)
    include = [jmhIncludeRegex]
  }
}

project(':iceberg-api') {
  dependencies {
    testCompile "org.apache.avro:avro"
    testCompile 'joda-time:joda-time'
  }
}

project(':iceberg-common') {}

project(':iceberg-core') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')

    compile("org.apache.avro:avro") {
      exclude group: 'org.tukaani' // xz compression is not supported
    }

    compile "com.fasterxml.jackson.core:jackson-databind"
    compile "com.fasterxml.jackson.core:jackson-core"
    compile "com.github.ben-manes.caffeine:caffeine"
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }
}

project(':iceberg-data') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')
    compileOnly project(':iceberg-parquet')

    testCompile("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }

  test {
    // Only for TestSplitScan as of Gradle 5.0+
    maxHeapSize '1500m'
  }
}

project(':iceberg-hive') {
  dependencies {
    compile project(':iceberg-core')

    compileOnly "org.apache.avro:avro"

    compileOnly("org.apache.hive:hive-metastore") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
    }

    // By default, hive-exec is a fat/uber jar and it exports a guava library
    // that's really old. We use the core classifier to be able to override our guava
    // version. Luckily, hive-exec seems to work okay so far with this version of guava
    // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
    testCompile("org.apache.hive:hive-exec::core") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
    }

    testCompile("org.apache.hive:hive-metastore") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
    }

    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }

  task testJar(type: Jar){
    archiveClassifier = 'tests'
    from sourceSets.test.output
  }

  configurations {
    testArtifacts
  }

  artifacts {
    testArtifacts testJar
  }
}

project(':iceberg-orc') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')

    compile("org.apache.orc:orc-core::nohive") {
      exclude group: 'org.apache.hadoop'
      exclude group: 'commons-lang'
      // These artifacts are shaded and included in the orc-core fat jar
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.hive', module: 'hive-storage-api'
    }

    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

  }
}

project(':iceberg-parquet') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')

    compile("org.apache.parquet:parquet-avro") {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly "org.apache.avro:avro"
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-spark') {
  apply plugin: 'scala'

  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')
    compile project(':iceberg-core')
    compile project(':iceberg-orc')
    compile project(':iceberg-parquet')
    compile project(':iceberg-hive')

    compileOnly "org.apache.avro:avro"
    compileOnly("org.apache.spark:spark-hive_2.11") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile "org.apache.hadoop:hadoop-hdfs::tests"
    testCompile "org.apache.hadoop:hadoop-common::tests"
    testCompile("org.apache.hadoop:hadoop-minicluster") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
    testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts')
  }
}

project(':iceberg-pig') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')
    compile project(':iceberg-core')
    compile project(':iceberg-parquet')

    compileOnly("org.apache.pig:pig") {
      exclude group: "junit", module: "junit"
    }
    compileOnly("org.apache.hadoop:hadoop-mapreduce-client-core")
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile "org.apache.hadoop:hadoop-hdfs::tests"
    testCompile "org.apache.hadoop:hadoop-common::tests"
    testCompile("org.apache.hadoop:hadoop-minicluster") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

// the runtime jar is a self-contained artifact for testing in a notebook
project(':iceberg-spark-runtime') {
  apply plugin: 'com.github.johnrengelman.shadow'

  tasks.build.dependsOn tasks.shadowJar
  tasks.install.dependsOn tasks.shadowJar
  tasks.javadocJar.dependsOn tasks.shadowJar

  configurations {
    compile {
      // included in Spark
      exclude group: 'org.slf4j'
      exclude group: 'org.apache.commons'
      exclude group: 'commons-pool'
      exclude group: 'commons-codec'
      exclude group: 'org.xerial.snappy'
      exclude group: 'javax.xml.bind'
    }
  }

  dependencies {
    compileOnly project(':iceberg-spark')
  }

  shadowJar {
    // shade compileOnly dependencies to avoid including in transitive dependencies
    configurations = [project.configurations.compileOnly]

    zip64 true

    // Relocate dependencies to avoid conflicts
    relocate 'com.google', 'org.apache.iceberg.shaded.com.google'
    relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
    relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
    relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
    relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
    relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
    relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
    relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
    relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
    // relocate Avro's jackson dependency to share parquet-jackson locations
    relocate 'org.codehaus.jackson', 'org.apache.iceberg.shaded.org.apache.parquet.shaded.org.codehaus.jackson'
    relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
    relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'

    archiveName = "iceberg-spark-runtime-${version}.${extension}"
  }
}