diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 92e2682..5ee423a 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -9,47 +9,54 @@ on: pull_request: branches: - master + workflow_dispatch: # be able to manually trigger a workflow run + +env: + SCALA_NEXT_VERSION: '3.6.2' + # TODO: perherps, it would be nice to allow failure on `SCALA_NEXT_VERSION` as a warning + # to avoid too strict dev experiences in the future improvements. jobs: test: runs-on: ubuntu-latest strategy: matrix: - java: ['11', '17'] + java: ['11', '17', '21'] steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} - name: Run tests - run: ./mill -i __.publishArtifacts + __.test + run: ./mill -i -DscalaNextVersion=${{ env.SCALA_NEXT_VERSION }} __.publishArtifacts + __.test check-binary-compatibility: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 11 + java-version: 17 - name: Check Binary Compatibility - run: ./mill -i __.mimaReportBinaryIssues + run: ./mill -i -DscalaNextVersion=${{ env.SCALA_NEXT_VERSION }} __.mimaReportBinaryIssues publish-sonatype: if: github.repository == 'com-lihaoyi/requests-scala' && contains(github.ref, 'refs/tags/') needs: test runs-on: ubuntu-latest env: - SONATYPE_PGP_PRIVATE_KEY: ${{ secrets.SONATYPE_PGP_PRIVATE_KEY }} - SONATYPE_PGP_PRIVATE_KEY_PASSWORD: ${{ secrets.SONATYPE_PGP_PRIVATE_KEY_PASSWORD }} - SONATYPE_USER: ${{ secrets.SONATYPE_USER }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + MILL_SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + MILL_SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + MILL_PGP_SECRET_BASE64: ${{ secrets.SONATYPE_PGP_PRIVATE_KEY }} + MILL_PGP_PASSPHRASE: ${{ secrets.SONATYPE_PGP_PRIVATE_KEY_PASSWORD }} LANG: "en_US.UTF-8" LC_MESSAGES: "en_US.UTF-8" LC_ALL: "en_US.UTF-8" + steps: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -57,20 +64,8 @@ jobs: distribution: 'temurin' java-version: 11 - name: Publish to Maven Central - run: | - if [[ $(git tag --points-at HEAD) != '' ]]; then - echo $SONATYPE_PGP_PRIVATE_KEY | base64 --decode > gpg_key - gpg --import --no-tty --batch --yes gpg_key - rm gpg_key - ./mill -i mill.scalalib.PublishModule/publishAll \ - --sonatypeCreds $SONATYPE_USER:$SONATYPE_PASSWORD \ - --gpgArgs --passphrase=$SONATYPE_PGP_PRIVATE_KEY_PASSWORD,--no-tty,--pinentry-mode,loopback,--batch,--yes,-a,-b \ - --publishArtifacts __.publishArtifacts \ - --readTimeout 600000 \ - --awaitTimeout 600000 \ - --release true \ - --signed true - fi + run: ./mill -i mill.scalalib.SonatypeCentralPublishModule/ + - name: Create GitHub Release id: create_gh_release uses: actions/create-release@v1.1.4 @@ -79,5 +74,4 @@ jobs: with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} - body: draft: false diff --git a/.gitignore b/.gitignore index 8d3ee55..a687f79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ target/ *.iml -.idea +.idea/ +.vscode/ .settings .classpath .project .cache .sbtserver +.scala-build/ project/.sbtserver tags nohup.out diff --git a/.mill-version b/.mill-version deleted file mode 100644 index 219a60c..0000000 --- a/.mill-version +++ /dev/null @@ -1,2 +0,0 @@ -0.11.5 - diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..2cc4941 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,12 @@ +version = "3.8.3" + +maxColumn = 100 +runner.dialect = scala213 + +newlines.beforeCurlyLambdaParams = multilineWithCaseOnly +rewrite.trailingCommas.style = always + +assumeStandardLibraryStripMargin = true +docstrings.style = Asterisk + +project.git = true diff --git a/build.mill b/build.mill new file mode 100644 index 0000000..a32f316 --- /dev/null +++ b/build.mill @@ -0,0 +1,74 @@ +//| mill-version: 1.0.0 +//| mvnDeps: +//| - com.github.lolgab::mill-mima_mill1:0.2.0 +package build + +import mill._ +import scalalib._ +import scalanativelib._ +import mill.scalalib.publish.{Developer, License, PomSettings, VersionControl} + +import mill.util.VcsVersion +import com.github.lolgab.mill.mima._ + +val scalaNextVersion = sys.props.get("scalaNextVersion") +val scalaVersions = List("2.12.20", "2.13.15", "3.3.4") ++ scalaNextVersion +val scalaNativeVer = "0.5.6" + +trait MimaCheck extends Mima { + def mimaPreviousVersions = Seq("0.6.9", "0.7.0", "0.7.1", "0.8.0","0.8.2", "0.9.0").distinct + + override def mimaBinaryIssueFilters = Seq( + ProblemFilter.exclude[ReversedMissingMethodProblem]("requests.BaseSession.send"), + ProblemFilter.exclude[DirectMissingMethodProblem]("requests.Response.string"), + ) +} + +trait RequestsPublishModule extends PublishModule with MimaCheck { + def artifactName = "requests" + + def publishVersion = VcsVersion.vcsState().format() + + def pomSettings = PomSettings( + description = "Scala port of the popular Python Requests HTTP client", + organization = "com.lihaoyi", + url = "https://github.com/com-lihaoyi/requests-scala", + licenses = Seq(License.MIT), + versionControl = VersionControl.github("com-lihaoyi", "requests-scala"), + developers = Seq( + Developer("lihaoyi", "Li Haoyi", "https://github.com/lihaoyi") + ) + ) + + def mvnDeps = Seq(mvn"com.lihaoyi::geny::1.1.1") +} + +trait RequestsCrossScalaModule extends CrossScalaModule with ScalaModule { + def moduleDir = build.moduleDir / "requests" + def sources = Task.Sources("src") +} + +trait RequestsTestModule extends TestModule.Utest { + def mvnDeps = Seq( + mvn"com.lihaoyi::utest::0.7.10", + mvn"com.lihaoyi::ujson::1.3.13", + mvn"com.dimafeng::testcontainers-scala-core:0.41.3" + ) +} + +object requests extends Module { + trait RequestsJvmModule extends RequestsCrossScalaModule with RequestsPublishModule { + object test extends ScalaTests with RequestsTestModule + } + object jvm extends Cross[RequestsJvmModule](scalaVersions) + + // trait RequestsNativeModule extends ScalaNativeModule with RequestsPublishModule { + // override def scalaNativeVersion = scalaNativeVer + // + // def mvnDeps = + // super.mvnDeps() ++ Seq(mvn"com.github.lolgab::scala-native-crypto::0.1.0") + // + // object test extends ScalaNativeTests with RequestsTestModule + // } + // object native extends Cross[RequestsNativeModule](scalaVersions) +} diff --git a/build.sc b/build.sc deleted file mode 100644 index ad28e89..0000000 --- a/build.sc +++ /dev/null @@ -1,46 +0,0 @@ -import mill._ -import mill.scalalib.publish.{Developer, License, PomSettings, VersionControl} -import scalalib._ -import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0` -import $ivy.`com.github.lolgab::mill-mima::0.0.23` - -import de.tobiasroeser.mill.vcs.version.VcsVersion -import com.github.lolgab.mill.mima._ - -val dottyVersion = sys.props.get("dottyVersion") - -val scalaVersions = List("2.12.17", "2.13.10", "2.11.12", "3.1.1") ++ dottyVersion - -object requests extends Cross[RequestsModule](scalaVersions) -trait RequestsModule extends CrossScalaModule with PublishModule with Mima { - def publishVersion = VcsVersion.vcsState().format() - def mimaPreviousVersions = - ( - Seq("0.7.0", "0.7.1", "0.8.2") ++ - Option.when(VcsVersion.vcsState().commitsSinceLastTag != 0)(VcsVersion.vcsState().lastTag).flatten - ).distinct - override def mimaBinaryIssueFilters = Seq( - ProblemFilter.exclude[ReversedMissingMethodProblem]("requests.BaseSession.send"), - ProblemFilter.exclude[DirectMissingMethodProblem]("requests.Response.string") - ) - - def pomSettings = PomSettings( - description = "Scala port of the popular Python Requests HTTP client", - organization = "com.lihaoyi", - url = "https://github.com/com-lihaoyi/requests-scala", - licenses = Seq(License.MIT), - versionControl = VersionControl.github("com-lihaoyi", "requests-scala"), - developers = Seq( - Developer("lihaoyi", "Li Haoyi","https://github.com/lihaoyi") - ) - ) - - def ivyDeps = Agg(ivy"com.lihaoyi::geny::1.0.0") - - object test extends ScalaTests with TestModule.Utest { - def ivyDeps = Agg( - ivy"com.lihaoyi::utest::0.7.10", - ivy"com.lihaoyi::ujson::1.3.13" - ) - } -} diff --git a/mill b/mill index 280389b..22777fc 100755 --- a/mill +++ b/mill @@ -1,51 +1,307 @@ #!/usr/bin/env sh -# This is a wrapper script, that automatically download mill from GitHub release pages -# You can give the required mill version with MILL_VERSION env variable -# If no version is given, it falls back to the value of DEFAULT_MILL_VERSION +# This is a wrapper script, that automatically selects or downloads Mill from Maven Central or GitHub release pages. +# +# This script determines the Mill version to use by trying these sources +# - env-variable `MILL_VERSION` +# - local file `.mill-version` +# - local file `.config/mill-version` +# - `mill-version` from YAML fronmatter of current buildfile +# - if accessible, find the latest stable version available on Maven Central (https://repo1.maven.org/maven2) +# - env-variable `DEFAULT_MILL_VERSION` +# +# If a version has the suffix '-native' a native binary will be used. +# If a version has the suffix '-jvm' an executable jar file will be used, requiring an already installed Java runtime. +# If no such suffix is found, the script will pick a default based on version and platform. +# +# Once a version was determined, it tries to use either +# - a system-installed mill, if found and it's version matches +# - an already downloaded version under ~/.cache/mill/download +# +# If no working mill version was found on the system, +# this script downloads a binary file from Maven Central or Github Pages (this is version dependent) +# into a cache location (~/.cache/mill/download). +# +# Mill Project URL: https://github.com/com-lihaoyi/mill +# Script Version: 1.0.0-M1-21-7b6fae-DIRTY892b63e8 +# +# If you want to improve this script, please also contribute your changes back! +# This script was generated from: dist/scripts/src/mill.sh +# +# Licensed under the Apache License, Version 2.0 set -e if [ -z "${DEFAULT_MILL_VERSION}" ] ; then - DEFAULT_MILL_VERSION=0.11.5 + DEFAULT_MILL_VERSION=1.0.0-RC1 fi -if [ -z "$MILL_VERSION" ] ; then + +if [ -z "${GITHUB_RELEASE_CDN}" ] ; then + GITHUB_RELEASE_CDN="" +fi + + +MILL_REPO_URL="https://github.com/com-lihaoyi/mill" + +if [ -z "${CURL_CMD}" ] ; then + CURL_CMD=curl +fi + +# Explicit commandline argument takes precedence over all other methods +if [ "$1" = "--mill-version" ] ; then + echo "The --mill-version option is no longer supported." 1>&2 +fi + +MILL_BUILD_SCRIPT="" + +if [ -f "build.mill" ] ; then + MILL_BUILD_SCRIPT="build.mill" +elif [ -f "build.mill.scala" ] ; then + MILL_BUILD_SCRIPT="build.mill.scala" +elif [ -f "build.sc" ] ; then + MILL_BUILD_SCRIPT="build.sc" +fi + +# Please note, that if a MILL_VERSION is already set in the environment, +# We reuse it's value and skip searching for a value. + +# If not already set, read .mill-version file +if [ -z "${MILL_VERSION}" ] ; then if [ -f ".mill-version" ] ; then - MILL_VERSION="$(head -n 1 .mill-version 2> /dev/null)" + MILL_VERSION="$(tr '\r' '\n' < .mill-version | head -n 1 2> /dev/null)" elif [ -f ".config/mill-version" ] ; then - MILL_VERSION="$(head -n 1 .config/mill-version 2> /dev/null)" - elif [ -f "mill" ] && [ "$0" != "mill" ] ; then - MILL_VERSION=$(grep -F "DEFAULT_MILL_VERSION=" "mill" | head -n 1 | cut -d= -f2) - else - MILL_VERSION=$DEFAULT_MILL_VERSION + MILL_VERSION="$(tr '\r' '\n' < .config/mill-version | head -n 1 2> /dev/null)" + elif [ -n "${MILL_BUILD_SCRIPT}" ] ; then + MILL_VERSION="$(cat ${MILL_BUILD_SCRIPT} | grep '//[|] *mill-version: *' | sed 's;//| *mill-version: *;;')" fi fi -if [ "x${XDG_CACHE_HOME}" != "x" ] ; then - MILL_DOWNLOAD_PATH="${XDG_CACHE_HOME}/mill/download" -else - MILL_DOWNLOAD_PATH="${HOME}/.cache/mill/download" +MILL_USER_CACHE_DIR="${XDG_CACHE_HOME:-${HOME}/.cache}/mill" + +if [ -z "${MILL_DOWNLOAD_PATH}" ] ; then + MILL_DOWNLOAD_PATH="${MILL_USER_CACHE_DIR}/download" fi -MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/${MILL_VERSION}" -version_remainder="$MILL_VERSION" -MILL_MAJOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}" -MILL_MINOR_VERSION="${version_remainder%%.*}"; version_remainder="${version_remainder#*.}" +# If not already set, try to fetch newest from Github +if [ -z "${MILL_VERSION}" ] ; then + # TODO: try to load latest version from release page + echo "No mill version specified." 1>&2 + echo "You should provide a version via a '//| mill-version: ' comment or a '.mill-version' file." 1>&2 + + mkdir -p "${MILL_DOWNLOAD_PATH}" + LANG=C touch -d '1 hour ago' "${MILL_DOWNLOAD_PATH}/.expire_latest" 2>/dev/null || ( + # we might be on OSX or BSD which don't have -d option for touch + # but probably a -A [-][[hh]mm]SS + touch "${MILL_DOWNLOAD_PATH}/.expire_latest"; touch -A -010000 "${MILL_DOWNLOAD_PATH}/.expire_latest" + ) || ( + # in case we still failed, we retry the first touch command with the intention + # to show the (previously suppressed) error message + LANG=C touch -d '1 hour ago' "${MILL_DOWNLOAD_PATH}/.expire_latest" + ) + + # POSIX shell variant of bash's -nt operator, see https://unix.stackexchange.com/a/449744/6993 + # if [ "${MILL_DOWNLOAD_PATH}/.latest" -nt "${MILL_DOWNLOAD_PATH}/.expire_latest" ] ; then + if [ -n "$(find -L "${MILL_DOWNLOAD_PATH}/.latest" -prune -newer "${MILL_DOWNLOAD_PATH}/.expire_latest")" ]; then + # we know a current latest version + MILL_VERSION=$(head -n 1 "${MILL_DOWNLOAD_PATH}"/.latest 2> /dev/null) + fi + + if [ -z "${MILL_VERSION}" ] ; then + # we don't know a current latest version + echo "Retrieving latest mill version ..." 1>&2 + LANG=C ${CURL_CMD} -s -i -f -I ${MILL_REPO_URL}/releases/latest 2> /dev/null | grep --ignore-case Location: | sed s'/^.*tag\///' | tr -d '\r\n' > "${MILL_DOWNLOAD_PATH}/.latest" + MILL_VERSION=$(head -n 1 "${MILL_DOWNLOAD_PATH}"/.latest 2> /dev/null) + fi + + if [ -z "${MILL_VERSION}" ] ; then + # Last resort + MILL_VERSION="${DEFAULT_MILL_VERSION}" + echo "Falling back to hardcoded mill version ${MILL_VERSION}" 1>&2 + else + echo "Using mill version ${MILL_VERSION}" 1>&2 + fi +fi + +MILL_NATIVE_SUFFIX="-native" +MILL_JVM_SUFFIX="-jvm" +FULL_MILL_VERSION=$MILL_VERSION +ARTIFACT_SUFFIX="" +set_artifact_suffix(){ + if [ "$(expr substr $(uname -s) 1 5 2>/dev/null)" = "Linux" ]; then + if [ "$(uname -m)" = "aarch64" ]; then + ARTIFACT_SUFFIX="-native-linux-aarch64" + else + ARTIFACT_SUFFIX="-native-linux-amd64" + fi + elif [ "$(uname)" = "Darwin" ]; then + if [ "$(uname -m)" = "arm64" ]; then + ARTIFACT_SUFFIX="-native-mac-aarch64" + else + ARTIFACT_SUFFIX="-native-mac-amd64" + fi + else + echo "This native mill launcher supports only Linux and macOS." 1>&2 + exit 1 + fi +} + +case "$MILL_VERSION" in + *"$MILL_NATIVE_SUFFIX") + MILL_VERSION=${MILL_VERSION%"$MILL_NATIVE_SUFFIX"} + set_artifact_suffix + ;; + + *"$MILL_JVM_SUFFIX") + MILL_VERSION=${MILL_VERSION%"$MILL_JVM_SUFFIX"} + ;; + + *) + case "$MILL_VERSION" in + 0.1.*) ;; + 0.2.*) ;; + 0.3.*) ;; + 0.4.*) ;; + 0.5.*) ;; + 0.6.*) ;; + 0.7.*) ;; + 0.8.*) ;; + 0.9.*) ;; + 0.10.*) ;; + 0.11.*) ;; + 0.12.*) ;; + *) + set_artifact_suffix + esac + ;; +esac + +MILL="${MILL_DOWNLOAD_PATH}/$MILL_VERSION$ARTIFACT_SUFFIX" + +try_to_use_system_mill() { + if [ "$(uname)" != "Linux" ]; then + return 0 + fi + + MILL_IN_PATH="$(command -v mill || true)" -if [ ! -s "$MILL_EXEC_PATH" ] ; then - mkdir -p "$MILL_DOWNLOAD_PATH" - if [ "$MILL_MAJOR_VERSION" -gt 0 ] || [ "$MILL_MINOR_VERSION" -ge 5 ] ; then - ASSEMBLY="-assembly" + if [ -z "${MILL_IN_PATH}" ]; then + return 0 fi - DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download - MILL_VERSION_TAG=$(echo $MILL_VERSION | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/') - MILL_DOWNLOAD_URL="https://github.com/lihaoyi/mill/releases/download/${MILL_VERSION_TAG}/$MILL_VERSION${ASSEMBLY}" - curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL" - chmod +x "$DOWNLOAD_FILE" - mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH" + + SYSTEM_MILL_FIRST_TWO_BYTES=$(head --bytes=2 "${MILL_IN_PATH}") + if [ "${SYSTEM_MILL_FIRST_TWO_BYTES}" = "#!" ]; then + # MILL_IN_PATH is (very likely) a shell script and not the mill + # executable, ignore it. + return 0 + fi + + SYSTEM_MILL_PATH=$(readlink -e "${MILL_IN_PATH}") + SYSTEM_MILL_SIZE=$(stat --format=%s "${SYSTEM_MILL_PATH}") + SYSTEM_MILL_MTIME=$(stat --format=%y "${SYSTEM_MILL_PATH}") + + if [ ! -d "${MILL_USER_CACHE_DIR}" ]; then + mkdir -p "${MILL_USER_CACHE_DIR}" + fi + + SYSTEM_MILL_INFO_FILE="${MILL_USER_CACHE_DIR}/system-mill-info" + if [ -f "${SYSTEM_MILL_INFO_FILE}" ]; then + parseSystemMillInfo() { + LINE_NUMBER="${1}" + # Select the line number of the SYSTEM_MILL_INFO_FILE, cut the + # variable definition in that line in two halves and return + # the value, and finally remove the quotes. + sed -n "${LINE_NUMBER}p" "${SYSTEM_MILL_INFO_FILE}" |\ + cut -d= -f2 |\ + sed 's/"\(.*\)"/\1/' + } + + CACHED_SYSTEM_MILL_PATH=$(parseSystemMillInfo 1) + CACHED_SYSTEM_MILL_VERSION=$(parseSystemMillInfo 2) + CACHED_SYSTEM_MILL_SIZE=$(parseSystemMillInfo 3) + CACHED_SYSTEM_MILL_MTIME=$(parseSystemMillInfo 4) + + if [ "${SYSTEM_MILL_PATH}" = "${CACHED_SYSTEM_MILL_PATH}" ] \ + && [ "${SYSTEM_MILL_SIZE}" = "${CACHED_SYSTEM_MILL_SIZE}" ] \ + && [ "${SYSTEM_MILL_MTIME}" = "${CACHED_SYSTEM_MILL_MTIME}" ]; then + if [ "${CACHED_SYSTEM_MILL_VERSION}" = "${MILL_VERSION}" ]; then + MILL="${SYSTEM_MILL_PATH}" + return 0 + else + return 0 + fi + fi + fi + + SYSTEM_MILL_VERSION=$(${SYSTEM_MILL_PATH} --version | head -n1 | sed -n 's/^Mill.*version \(.*\)/\1/p') + + cat < "${SYSTEM_MILL_INFO_FILE}" +CACHED_SYSTEM_MILL_PATH="${SYSTEM_MILL_PATH}" +CACHED_SYSTEM_MILL_VERSION="${SYSTEM_MILL_VERSION}" +CACHED_SYSTEM_MILL_SIZE="${SYSTEM_MILL_SIZE}" +CACHED_SYSTEM_MILL_MTIME="${SYSTEM_MILL_MTIME}" +EOF + + if [ "${SYSTEM_MILL_VERSION}" = "${MILL_VERSION}" ]; then + MILL="${SYSTEM_MILL_PATH}" + fi +} +try_to_use_system_mill + +# If not already downloaded, download it +if [ ! -s "${MILL}" ] || [ "$MILL_TEST_DRY_RUN_LAUNCHER_SCRIPT" = "1" ] ; then + case $MILL_VERSION in + 0.0.* | 0.1.* | 0.2.* | 0.3.* | 0.4.* ) + DOWNLOAD_SUFFIX="" + DOWNLOAD_FROM_MAVEN=0 + ;; + 0.5.* | 0.6.* | 0.7.* | 0.8.* | 0.9.* | 0.10.* | 0.11.0-M* ) + DOWNLOAD_SUFFIX="-assembly" + DOWNLOAD_FROM_MAVEN=0 + ;; + *) + DOWNLOAD_SUFFIX="-assembly" + DOWNLOAD_FROM_MAVEN=1 + ;; + esac + case $MILL_VERSION in + 0.12.0 | 0.12.1 | 0.12.2 | 0.12.3 | 0.12.4 | 0.12.5 | 0.12.6 | 0.12.7 | 0.12.8 | 0.12.9 | 0.12.10 | 0.12.11 ) + DOWNLOAD_EXT="jar" + ;; + 0.12.* ) + DOWNLOAD_EXT="exe" + ;; + 0.* ) + DOWNLOAD_EXT="jar" + ;; + *) + DOWNLOAD_EXT="exe" + ;; + esac + + DOWNLOAD_FILE=$(mktemp mill.XXXXXX) + if [ "$DOWNLOAD_FROM_MAVEN" = "1" ] ; then + DOWNLOAD_URL="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist${ARTIFACT_SUFFIX}/${MILL_VERSION}/mill-dist${ARTIFACT_SUFFIX}-${MILL_VERSION}.${DOWNLOAD_EXT}" + else + MILL_VERSION_TAG=$(echo "$MILL_VERSION" | sed -E 's/([^-]+)(-M[0-9]+)?(-.*)?/\1\2/') + DOWNLOAD_URL="${GITHUB_RELEASE_CDN}${MILL_REPO_URL}/releases/download/${MILL_VERSION_TAG}/${MILL_VERSION}${DOWNLOAD_SUFFIX}" + unset MILL_VERSION_TAG + fi + + if [ "$MILL_TEST_DRY_RUN_LAUNCHER_SCRIPT" = "1" ] ; then + echo $DOWNLOAD_URL + echo $MILL + exit 0 + fi + # TODO: handle command not found + echo "Downloading mill ${MILL_VERSION} from ${DOWNLOAD_URL} ..." 1>&2 + ${CURL_CMD} -f -L -o "${DOWNLOAD_FILE}" "${DOWNLOAD_URL}" + chmod +x "${DOWNLOAD_FILE}" + mkdir -p "${MILL_DOWNLOAD_PATH}" + mv "${DOWNLOAD_FILE}" "${MILL}" + unset DOWNLOAD_FILE - unset MILL_DOWNLOAD_URL + unset DOWNLOAD_SUFFIX fi if [ -z "$MILL_MAIN_CLI" ] ; then @@ -53,13 +309,19 @@ if [ -z "$MILL_MAIN_CLI" ] ; then fi MILL_FIRST_ARG="" -if [ "$1" = "--bsp" ] || [ "$1" = "-i" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then +if [ "$1" = "--bsp" ] || [ "$1" = "-i" ] || [ "$1" = "--interactive" ] || [ "$1" = "--no-server" ] || [ "$1" = "--no-daemon" ] || [ "$1" = "--repl" ] || [ "$1" = "--help" ] ; then # Need to preserve the first position of those listed options MILL_FIRST_ARG=$1 shift fi unset MILL_DOWNLOAD_PATH +unset MILL_OLD_DOWNLOAD_PATH +unset OLD_MILL unset MILL_VERSION +unset MILL_REPO_URL -exec $MILL_EXEC_PATH $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@" +# -D mill.main.cli is for compatibility with Mill 0.10.9 - 0.13.0-M2 +# We don't quote MILL_FIRST_ARG on purpose, so we can expand the empty value without quotes +# shellcheck disable=SC2086 +exec "${MILL}" $MILL_FIRST_ARG -D "mill.main.cli=${MILL_MAIN_CLI}" "$@" diff --git a/readme.md b/readme.md index ee6b807..9ed648c 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# Requests-Scala 0.9.0-RC1 +# Requests-Scala 0.9.0 [![Join the chat at https://gitter.im/lihaoyi/requests-scala](https://badges.gitter.im/lihaoyi/requests-scala.svg)](https://gitter.im/lihaoyi/requests-scala?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -26,7 +26,7 @@ For a hands-on introduction to this library, take a look at the following blog p ## Contents -- [Requests-Scala 0.8.3](#requests-scala-081) +- [Requests-Scala 0.9.0](#requests-scala-081) - [Contents](#contents) - [Getting Started](#getting-started) - [Making a Request](#making-a-request) @@ -45,6 +45,7 @@ For a hands-on introduction to this library, take a look at the following blog p - [Sessions](#sessions) - [Why Requests-Scala?](#why-requests-scala) - [Changelog](#changelog) + - [0.9.0](#090) - [0.8.0](#080) - [0.7.1](#071) - [0.7.0](#070) @@ -65,9 +66,9 @@ For a hands-on introduction to this library, take a look at the following blog p Use the following import to get you started: ```scala -ivy"com.lihaoyi::requests:0.9.0-RC1" // mill -"com.lihaoyi" %% "requests" % "0.9.0-RC1" // sbt -compile "com.lihaoyi:requests_2.12:0.9.0-RC1" //gradle +ivy"com.lihaoyi::requests:0.9.0" // mill +"com.lihaoyi" %% "requests" % "0.9.0" // sbt +compile "com.lihaoyi:requests_2.12:0.9.0" //gradle ``` ## Making a Request @@ -293,6 +294,14 @@ requests.get( ) ``` +To pass in a single header multiple times, you can pass them as a comma separated list: + +```scala +requests.get( + "https://api.github.com/some/endpoint", + headers = Map("user-agent" -> "my-app/0.0.1,other-app/0.0.2") +) +``` ### Timeouts @@ -673,7 +682,7 @@ codebase or project! ## Changelog -### 0.9.0-RC1 +### 0.9.0 - Use JDK 11 HttpClient ([#158](https://github.com/com-lihaoyi/requests-scala/pull/158)). Note that this means we are dropping compatibility with JDK 8, and will require JDK 11 and above diff --git a/requests/src/requests/Exceptions.scala b/requests/src/requests/Exceptions.scala index 943d727..f578d95 100644 --- a/requests/src/requests/Exceptions.scala +++ b/requests/src/requests/Exceptions.scala @@ -1,18 +1,29 @@ package requests // base class for all custom exceptions thrown by requests. -class RequestsException(val message: String, val cause: Option[Throwable] = None) extends Exception(message, cause.getOrElse(null)) +class RequestsException( + val message: String, + val cause: Option[Throwable] = None, +) extends Exception(message, cause.getOrElse(null)) -class TimeoutException(val url: String, val readTimeout: Int, val connectTimeout: Int) -extends RequestsException(s"Request to $url timed out. (readTimeout: $readTimeout, connectTimout: $connectTimeout)") +class TimeoutException( + val url: String, + val readTimeout: Int, + val connectTimeout: Int, +) extends RequestsException( + s"Request to $url timed out. (readTimeout: $readTimeout, connectTimout: $connectTimeout)", + ) class UnknownHostException(val url: String, val host: String) -extends RequestsException(s"Unknown host $host in url $url") + extends RequestsException(s"Unknown host $host in url $url") class InvalidCertException(val url: String, cause: Throwable) -extends RequestsException(s"Unable to validate SSL certificates for $url", Some(cause)) + extends RequestsException( + s"Unable to validate SSL certificates for $url", + Some(cause), + ) class RequestFailedException(val response: Response) -extends RequestsException( - s"Request to ${response.url} failed with status code ${response.statusCode}\n${response.text()}" -) + extends RequestsException( + s"Request to ${response.url} failed with status code ${response.statusCode}\n${response.text()}", + ) diff --git a/requests/src/requests/Model.scala b/requests/src/requests/Model.scala index 6ba7135..2a98243 100644 --- a/requests/src/requests/Model.scala +++ b/requests/src/requests/Model.scala @@ -12,107 +12,105 @@ import java.util.zip.{DeflaterOutputStream, GZIPOutputStream} import javax.net.ssl.SSLContext /** - * Mechanisms for compressing the upload stream; supports Gzip and Deflate - * by default - */ -trait Compress{ + * Mechanisms for compressing the upload stream; supports Gzip and Deflate by default + */ +trait Compress { def headers: Seq[(String, String)] def wrap(x: OutputStream): OutputStream } -object Compress{ - object Gzip extends Compress{ - def headers = Seq( - "Content-Encoding" -> "gzip" - ) +object Compress { + object Gzip extends Compress { + def headers = Seq("Content-Encoding" -> "gzip") def wrap(x: OutputStream) = new GZIPOutputStream(x) } - object Deflate extends Compress{ - def headers = Seq( - "Content-Encoding" -> "deflate" - ) + object Deflate extends Compress { + def headers = Seq("Content-Encoding" -> "deflate") def wrap(x: OutputStream) = new DeflaterOutputStream(x) } - object None extends Compress{ + object None extends Compress { def headers = Nil def wrap(x: OutputStream) = x } } /** - * The equivalent of configuring a [[Requester.apply]] or [[Requester.stream]] - * call, but without invoking it. Useful if you want to further customize it - * and make the call later via the overloads of `apply`/`stream` that take a - * [[Request]]. - */ -case class Request(url: String, - auth: RequestAuth = RequestAuth.Empty, - params: Iterable[(String, String)] = Nil, - headers: Iterable[(String, String)] = Nil, - readTimeout: Int = 0, - connectTimeout: Int = 0, - proxy: (String, Int) = null, - cert: Cert = null, - sslContext: SSLContext = null, - cookies: Map[String, HttpCookie] = Map(), - cookieValues: Map[String, String] = Map(), - maxRedirects: Int = 5, - verifySslCerts: Boolean = true, - autoDecompress: Boolean = true, - compress: Compress = Compress.None, - keepAlive: Boolean = true, - check: Boolean = true) + * The equivalent of configuring a [[Requester.apply]] or [[Requester.stream]] call, but without + * invoking it. Useful if you want to further customize it and make the call later via the overloads + * of `apply`/`stream` that take a [[Request]]. + */ +case class Request( + url: String, + auth: RequestAuth = RequestAuth.Empty, + params: Iterable[(String, String)] = Nil, + headers: Iterable[(String, String)] = Nil, + readTimeout: Int = 0, + connectTimeout: Int = 0, + proxy: (String, Int) = null, + cert: Cert = null, + sslContext: SSLContext = null, + cookies: Map[String, HttpCookie] = Map(), + cookieValues: Map[String, String] = Map(), + maxRedirects: Int = 5, + verifySslCerts: Boolean = true, + autoDecompress: Boolean = true, + compress: Compress = Compress.None, + keepAlive: Boolean = true, + check: Boolean = true, +) /** - * Represents the different things you can upload in the body of a HTTP - * request. By default, allows form-encoded key-value pairs, arrays of bytes, - * strings, files, and InputStreams. These types can be passed directly to - * the `data` parameter of [[Requester.apply]] and will be wrapped automatically - * by the implicit constructors. - */ -trait RequestBlob{ + * Represents the different things you can upload in the body of a HTTP request. By default, allows + * form-encoded key-value pairs, arrays of bytes, strings, files, and InputStreams. These types can + * be passed directly to the `data` parameter of [[Requester.apply]] and will be wrapped + * automatically by the implicit constructors. + */ +trait RequestBlob { def headers: Seq[(String, String)] = Nil def write(out: java.io.OutputStream): Unit } -object RequestBlob{ - object EmptyRequestBlob extends RequestBlob{ +object RequestBlob { + object EmptyRequestBlob extends RequestBlob { def write(out: java.io.OutputStream): Unit = () override def headers = Seq("Content-Length" -> "0") } - implicit class ByteSourceRequestBlob[T](x: T)(implicit f: T => geny.Writable) extends RequestBlob{ + implicit class ByteSourceRequestBlob[T](x: T)(implicit f: T => geny.Writable) + extends RequestBlob { private[this] val s = f(x) override def headers = super.headers ++ - s.httpContentType.map("Content-Type" -> _) ++ - s.contentLength.map("Content-Length" -> _.toString) + s.httpContentType.map("Content-Type" -> _) ++ + s.contentLength.map("Content-Length" -> _.toString) def write(out: java.io.OutputStream) = s.writeBytesTo(out) } - implicit class FileRequestBlob(x: java.io.File) extends RequestBlob{ + + implicit class FileRequestBlob(x: java.io.File) extends RequestBlob { override def headers = super.headers ++ Seq( "Content-Type" -> "application/octet-stream", - "Content-Length" -> x.length().toString + "Content-Length" -> x.length().toString, ) def write(out: java.io.OutputStream) = Util.transferTo(new FileInputStream(x), out) } - implicit class NioFileRequestBlob(x: java.nio.file.Path) extends RequestBlob{ + + implicit class NioFileRequestBlob(x: java.nio.file.Path) extends RequestBlob { override def headers = super.headers ++ Seq( "Content-Type" -> "application/octet-stream", - "Content-Length" -> java.nio.file.Files.size(x).toString + "Content-Length" -> java.nio.file.Files.size(x).toString, ) - def write(out: java.io.OutputStream) = Util.transferTo(java.nio.file.Files.newInputStream(x), out) + def write(out: java.io.OutputStream) = + Util.transferTo(java.nio.file.Files.newInputStream(x), out) } - implicit class FormEncodedRequestBlob(val x: Iterable[(String, String)]) extends RequestBlob{ + implicit class FormEncodedRequestBlob(val x: Iterable[(String, String)]) extends RequestBlob { val serialized = Util.urlEncode(x).getBytes - override def headers = super.headers ++ Seq( - "Content-Type" -> "application/x-www-form-urlencoded" - ) + override def headers = + super.headers ++ Seq("Content-Type" -> "application/x-www-form-urlencoded") def write(out: java.io.OutputStream) = { out.write(serialized) } } - implicit class MultipartFormRequestBlob(val parts: Iterable[MultiItem]) extends RequestBlob{ + implicit class MultipartFormRequestBlob(val parts: Iterable[MultiItem]) extends RequestBlob { val boundary = UUID.randomUUID().toString val crlf = "\r\n" val pref = "--" @@ -122,23 +120,28 @@ object RequestBlob{ // encode params up front for the length calculation - val partBytes = parts.map(p => (p.name.getBytes(), if (p.filename == null) Array[Byte]() else p.filename.getBytes(), p)) - - override def headers = Seq( - "Content-Type" -> s"multipart/form-data; boundary=$boundary" + val partBytes = parts.map(p => + ( + p.name.getBytes(), + if (p.filename == null) Array[Byte]() else p.filename.getBytes(), + p, + ), ) + + override def headers = Seq("Content-Type" -> s"multipart/form-data; boundary=$boundary") def write(out: java.io.OutputStream) = { def writeBytes(s: String): Unit = out.write(s.getBytes()) partBytes.foreach { - case(name, filename, part) => + case (name, filename, part) => writeBytes(pref + boundary + crlf) - part.data.headers.foreach { case (headerName, headerValue) => - writeBytes(s"$headerName: $headerValue$crlf") + part.data.headers.foreach { + case (headerName, headerValue) => + writeBytes(s"$headerName: $headerValue$crlf") } writeBytes(ContentDisposition) out.write(name) - if (filename.nonEmpty){ + if (filename.nonEmpty) { writeBytes(filenameSnippet) out.write(filename) } @@ -156,46 +159,50 @@ object RequestBlob{ } case class MultiPart(items: MultiItem*) extends RequestBlob.MultipartFormRequestBlob(items) -case class MultiItem(name: String, - data: RequestBlob, - filename: String = null) +case class MultiItem(name: String, data: RequestBlob, filename: String = null) /** - * Wraps the array of bytes returned in the body of a HTTP response - */ -class ResponseBlob(val bytes: Array[Byte]){ + * Wraps the array of bytes returned in the body of a HTTP response + */ +class ResponseBlob(val bytes: Array[Byte]) { override def toString = s"ResponseBlob(${bytes.length} bytes)" def text = new String(bytes) override def hashCode() = java.util.Arrays.hashCode(bytes) - override def equals(obj: scala.Any) = obj match{ + override def equals(obj: scala.Any) = obj match { case r: ResponseBlob => java.util.Arrays.equals(bytes, r.bytes) - case _ => false + case _ => false } } - /** * Represents a HTTP response * - * @param url the URL that the original request was made to - * @param statusCode the status code of the response - * @param statusMessage a string that describes the status code. - * This is not the reason phrase sent by the server, - * but a string describing [[statusCode]], as hardcoded in this library - * @param headers the raw headers the server sent back with the response - * @param data the response body; may contain HTML, JSON, or binary or textual data - * @param history the response of any redirects that were performed before - * arriving at the current response + * @param url + * the URL that the original request was made to + * @param statusCode + * the status code of the response + * @param statusMessage + * a string that describes the status code. This is not the reason phrase sent by the server, but + * a string describing [[statusCode]], as hardcoded in this library + * @param headers + * the raw headers the server sent back with the response + * @param data + * the response body; may contain HTML, JSON, or binary or textual data + * @param history + * the response of any redirects that were performed before arriving at the current response */ -case class Response(url: String, - statusCode: Int, - @deprecated("Value is inferred from `statusCode`", "0.9.0") - statusMessage: String, - data: geny.Bytes, - headers: Map[String, Seq[String]], - history: Option[Response]) extends geny.ByteData with geny.Readable{ +case class Response( + url: String, + statusCode: Int, + @deprecated("Value is inferred from `statusCode`", "0.9.0") + statusMessage: String, + data: geny.Bytes, + headers: Map[String, Seq[String]], + history: Option[Response], +) extends geny.ByteData + with geny.Readable { def bytes = data.array @@ -203,15 +210,16 @@ case class Response(url: String, def contents = data.array /** - * Returns the cookies set by this response, and by any redirects that lead up to it - */ - val cookies: Map[String, HttpCookie] = history.toSeq.flatMap(_.cookies).toMap ++ headers - .get("set-cookie") - .iterator - .flatten - .flatMap(java.net.HttpCookie.parse(_).asScala) - .map(x => x.getName -> x) - .toMap + * Returns the cookies set by this response, and by any redirects that lead up to it + */ + val cookies: Map[String, HttpCookie] = + history.toSeq.flatMap(_.cookies).toMap ++ headers + .get("set-cookie") + .iterator + .flatten + .flatMap(java.net.HttpCookie.parse(_).asScala) + .map(x => x.getName -> x) + .toMap def contentType = headers.get("content-type").flatMap(_.headOption) @@ -229,35 +237,45 @@ case class Response(url: String, override def contentLength: Option[Long] = Some(data.array.length) } -case class StreamHeaders(url: String, - statusCode: Int, - @deprecated("Value is inferred from `statusCode`", "0.9.0") - statusMessage: String, - headers: Map[String, Seq[String]], - history: Option[Response]){ +case class StreamHeaders( + url: String, + statusCode: Int, + @deprecated("Value is inferred from `statusCode`", "0.9.0") + statusMessage: String, + headers: Map[String, Seq[String]], + history: Option[Response], +) { def is2xx = statusCode.toString.charAt(0) == '2' def is3xx = statusCode.toString.charAt(0) == '3' def is4xx = statusCode.toString.charAt(0) == '4' def is5xx = statusCode.toString.charAt(0) == '5' } + /** - * Different ways you can authorize a HTTP request; by default, HTTP Basic - * auth and Proxy auth are supported - */ -trait RequestAuth{ + * Different ways you can authorize a HTTP request; by default, HTTP Basic auth and Proxy auth are + * supported + */ +trait RequestAuth { def header: Option[String] } -object RequestAuth{ - object Empty extends RequestAuth{ +object RequestAuth { + object Empty extends RequestAuth { def header = None } + implicit def implicitBasic(x: (String, String)): Basic = new Basic(x._1, x._2) - class Basic(username: String, password: String) extends RequestAuth{ - def header = Some("Basic " + java.util.Base64.getEncoder.encodeToString((username + ":" + password).getBytes())) + + class Basic(username: String, password: String) extends RequestAuth { + def header = Some( + "Basic " + java.util.Base64.getEncoder.encodeToString((username + ":" + password).getBytes()), + ) } - case class Proxy(username: String, password: String) extends RequestAuth{ - def header = Some("Proxy-Authorization " + java.util.Base64.getEncoder.encodeToString((username + ":" + password).getBytes())) + case class Proxy(username: String, password: String) extends RequestAuth { + def header = Some( + "Proxy-Authorization " + java.util.Base64.getEncoder + .encodeToString((username + ":" + password).getBytes()), + ) } case class Bearer(token: String) extends RequestAuth { def header = Some(s"Bearer $token") @@ -265,7 +283,7 @@ object RequestAuth{ } sealed trait Cert -object Cert{ +object Cert { implicit def implicitP12(path: String): P12 = P12(path, None) implicit def implicitP12(x: (String, String)): P12 = P12(x._1, Some(x._2)) case class P12(p12: String, pwd: Option[String] = None) extends Cert diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index 7b4b158..a778788 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -10,13 +10,13 @@ import java.util.function.Supplier import java.util.zip.{GZIPInputStream, InflaterInputStream} import scala.collection.JavaConverters._ +import scala.collection.immutable.ListMap import scala.collection.mutable import scala.concurrent.{ExecutionException, Future} import javax.net.ssl.SSLContext - -trait BaseSession{ +trait BaseSession { def headers: Map[String, String] def cookies: mutable.Map[String, HttpCookie] def readTimeout: Int @@ -44,14 +44,15 @@ trait BaseSession{ def send(method: String) = Requester(method, this) } -object BaseSession{ +object BaseSession { val defaultHeaders = Map( "User-Agent" -> "requests-scala", "Accept-Encoding" -> "gzip, deflate", - "Accept" -> "*/*" + "Accept" -> "*/*", ) } -object Requester{ + +object Requester { val officialHttpMethods = Set("GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE") private lazy val methodField: java.lang.reflect.Field = { val m = classOf[HttpURLConnection].getDeclaredField("method") @@ -59,55 +60,68 @@ object Requester{ m } } -case class Requester(verb: String, - sess: BaseSession){ + +case class Requester(verb: String, sess: BaseSession) { private val upperCaseVerb = verb.toUpperCase /** - * Makes a single HTTP request, and returns a [[Response]] object. Requires - * all uploaded request `data` to be provided up-front, and aggregates all - * downloaded response `data` before returning it in the response. If you - * need streaming access to the upload and download, use the [[Requester.stream]] - * function instead. - * - * @param url The URL to which you want to make this HTTP request - * @param auth HTTP authentication you want to use with this request; defaults to none - * @param params URL params to pass to this request, for `GET`s and `DELETE`s - * @param headers Custom headers to use, in addition to the defaults - * @param data Body data to pass to this request, for POSTs and PUTs. Can be a - * Map[String, String] of form data, bulk data as a String or Array[Byte], - * or MultiPart form data. - * @param readTimeout How long to wait for data to be read before timing out - * @param connectTimeout How long to wait for a connection before timing out - * @param proxy Host and port of a proxy you want to use - * @param cert Client certificate configuration - * @param sslContext Client sslContext configuration - * @param cookies Custom cookies to send up with this request - * @param maxRedirects How many redirects to automatically resolve; defaults to 5. - * You can also set it to 0 to prevent Requests from resolving - * redirects for you - * @param verifySslCerts Set this to false to ignore problems with SSL certificates - * @param check Throw an exception on a 4xx or 5xx response code. Defaults to `true` - */ - def apply(url: String, - auth: RequestAuth = sess.auth, - params: Iterable[(String, String)] = Nil, - headers: Iterable[(String, String)] = Nil, - data: RequestBlob = RequestBlob.EmptyRequestBlob, - readTimeout: Int = sess.readTimeout, - connectTimeout: Int = sess.connectTimeout, - proxy: (String, Int) = sess.proxy, - cert: Cert = sess.cert, - sslContext: SSLContext = sess.sslContext, - cookies: Map[String, HttpCookie] = Map(), - cookieValues: Map[String, String] = Map(), - maxRedirects: Int = sess.maxRedirects, - verifySslCerts: Boolean = sess.verifySslCerts, - autoDecompress: Boolean = sess.autoDecompress, - compress: Compress = sess.compress, - keepAlive: Boolean = true, - check: Boolean = sess.check, - chunkedUpload: Boolean = sess.chunkedUpload): Response = { + * Makes a single HTTP request, and returns a [[Response]] object. Requires all uploaded request + * `data` to be provided up-front, and aggregates all downloaded response `data` before returning + * it in the response. If you need streaming access to the upload and download, use the + * [[Requester.stream]] function instead. + * + * @param url + * The URL to which you want to make this HTTP request + * @param auth + * HTTP authentication you want to use with this request; defaults to none + * @param params + * URL params to pass to this request, for `GET`s and `DELETE`s + * @param headers + * Custom headers to use, in addition to the defaults + * @param data + * Body data to pass to this request, for POSTs and PUTs. Can be a Map[String, String] of form + * data, bulk data as a String or Array[Byte], or MultiPart form data. + * @param readTimeout + * How many milliseconds to wait for data to be read before timing out + * @param connectTimeout + * How many milliseconds to wait for a connection before timing out + * @param proxy + * Host and port of a proxy you want to use + * @param cert + * Client certificate configuration + * @param sslContext + * Client sslContext configuration + * @param cookies + * Custom cookies to send up with this request + * @param maxRedirects + * How many redirects to automatically resolve; defaults to 5. You can also set it to 0 to + * prevent Requests from resolving redirects for you + * @param verifySslCerts + * Set this to false to ignore problems with SSL certificates + * @param check + * Throw an exception on a 4xx or 5xx response code. Defaults to `true` + */ + def apply( + url: String, + auth: RequestAuth = sess.auth, + params: Iterable[(String, String)] = Nil, + headers: Iterable[(String, String)] = Nil, + data: RequestBlob = RequestBlob.EmptyRequestBlob, + readTimeout: Int = sess.readTimeout, + connectTimeout: Int = sess.connectTimeout, + proxy: (String, Int) = sess.proxy, + cert: Cert = sess.cert, + sslContext: SSLContext = sess.sslContext, + cookies: Map[String, HttpCookie] = Map(), + cookieValues: Map[String, String] = Map(), + maxRedirects: Int = sess.maxRedirects, + verifySslCerts: Boolean = sess.verifySslCerts, + autoDecompress: Boolean = sess.autoDecompress, + compress: Compress = sess.compress, + keepAlive: Boolean = true, + check: Boolean = sess.check, + chunkedUpload: Boolean = sess.chunkedUpload, + ): Response = { val out = new ByteArrayOutputStream() var streamHeaders: StreamHeaders = null @@ -133,7 +147,7 @@ case class Requester(verb: String, keepAlive = keepAlive, check = check, chunkedUpload = chunkedUpload, - onHeadersReceived = sh => streamHeaders = sh + onHeadersReceived = sh => streamHeaders = sh, ) w.writeBytesTo(out) @@ -144,50 +158,49 @@ case class Requester(verb: String, statusMessage = streamHeaders.statusMessage, data = new geny.Bytes(out.toByteArray), headers = streamHeaders.headers, - history = streamHeaders.history + history = streamHeaders.history, ) } /** - * Performs a streaming HTTP request. Most of the parameters are the same as - * [[apply]], except that the `data` parameter is missing, and no [[Response]] - * object is returned. Instead, the caller gets access via three callbacks - * (described below). This provides a lower-level API than [[Requester.apply]], - * allowing the caller fine-grained access to the upload/download streams - * so they can direct them where-ever necessary without first aggregating all - * the data into memory. - * - * @param onHeadersReceived the second callback to be called, this provides - * access to the response's status code, status - * message, headers, and any previous re-direct - * responses. Returns a boolean, where `false` can - * be used to - * - * @return a `Writable` that can be used to write the output data to any - * destination - */ - def stream(url: String, - auth: RequestAuth = sess.auth, - params: Iterable[(String, String)] = Nil, - blobHeaders: Iterable[(String, String)] = Nil, - headers: Iterable[(String, String)] = Nil, - data: RequestBlob = RequestBlob.EmptyRequestBlob, - readTimeout: Int = sess.readTimeout, - connectTimeout: Int = sess.connectTimeout, - proxy: (String, Int) = sess.proxy, - cert: Cert = sess.cert, - sslContext: SSLContext = sess.sslContext, - cookies: Map[String, HttpCookie] = Map(), - cookieValues: Map[String, String] = Map(), - maxRedirects: Int = sess.maxRedirects, - verifySslCerts: Boolean = sess.verifySslCerts, - autoDecompress: Boolean = sess.autoDecompress, - compress: Compress = sess.compress, - keepAlive: Boolean = true, - check: Boolean = true, - chunkedUpload: Boolean = false, - redirectedFrom: Option[Response] = None, - onHeadersReceived: StreamHeaders => Unit = null): geny.Readable = new geny.Readable { + * Performs a streaming HTTP request. Most of the parameters are the same as [[apply]], except + * that the `data` parameter is missing, and no [[Response]] object is returned. Instead, the + * caller gets access via three callbacks (described below). This provides a lower-level API than + * [[Requester.apply]], allowing the caller fine-grained access to the upload/download streams so + * they can direct them where-ever necessary without first aggregating all the data into memory. + * + * @param onHeadersReceived + * the second callback to be called, this provides access to the response's status code, status + * message, headers, and any previous re-direct responses. Returns a boolean, where `false` can + * be used to + * + * @return + * a `Writable` that can be used to write the output data to any destination + */ + def stream( + url: String, + auth: RequestAuth = sess.auth, + params: Iterable[(String, String)] = Nil, + blobHeaders: Iterable[(String, String)] = Nil, + headers: Iterable[(String, String)] = Nil, + data: RequestBlob = RequestBlob.EmptyRequestBlob, + readTimeout: Int = sess.readTimeout, + connectTimeout: Int = sess.connectTimeout, + proxy: (String, Int) = sess.proxy, + cert: Cert = sess.cert, + sslContext: SSLContext = sess.sslContext, + cookies: Map[String, HttpCookie] = Map(), + cookieValues: Map[String, String] = Map(), + maxRedirects: Int = sess.maxRedirects, + verifySslCerts: Boolean = sess.verifySslCerts, + autoDecompress: Boolean = sess.autoDecompress, + compress: Compress = sess.compress, + keepAlive: Boolean = true, + check: Boolean = true, + chunkedUpload: Boolean = false, + redirectedFrom: Option[Response] = None, + onHeadersReceived: StreamHeaders => Unit = null, + ): geny.Readable = new geny.Readable { def readBytesThrough[T](f: java.io.InputStream => T): T = { val url0 = new java.net.URL(url) @@ -214,7 +227,7 @@ case class Requester(verb: String, else if (!verifySslCerts) Util.noVerifySSLContext else - SSLContext.getDefault + SSLContext.getDefault, ) .connectTimeout(Duration.ofMillis(connectTimeout)) .build() @@ -228,7 +241,8 @@ case class Requester(verb: String, val allCookies = sessionCookieValues ++ cookieValues - val (contentLengthHeader, otherBlobHeaders) = blobHeaders.partition(_._1.equalsIgnoreCase("Content-Length")) + val (contentLengthHeader, otherBlobHeaders) = + blobHeaders.partition(_._1.equalsIgnoreCase("Content-Length")) val allHeaders = otherBlobHeaders ++ @@ -237,11 +251,20 @@ case class Requester(verb: String, compress.headers ++ auth.header.map("Authorization" -> _) ++ (if (allCookies.isEmpty) None - else Some("Cookie" -> allCookies - .map { case (k, v) => s"""$k="$v"""" } - .mkString("; ") - )) - val allHeadersFlat = allHeaders.toList.flatMap { case (k, v) => Seq(k, v) } + else + Some( + "Cookie" -> allCookies + .map { case (k, v) => s"""$k="$v"""" } + .mkString("; "), + )) + val lastOfEachHeader = + allHeaders.foldLeft(ListMap.empty[String, (String, String)]) { + case (acc, (k, v)) => + acc.updated(k.toLowerCase, k -> v) + } + val headersKeyValueAlternating = lastOfEachHeader.values.toList.flatMap { + case (k, v) => Seq(k, v) + } val requestBodyInputStream = new PipedInputStream() val requestBodyOutputStream = new PipedOutputStream(requestBodyInputStream) @@ -252,23 +275,27 @@ case class Requester(verb: String, }) val requestBuilder = - HttpRequest.newBuilder() + HttpRequest + .newBuilder() .uri(url1.toURI) .timeout(Duration.ofMillis(readTimeout)) - .headers(allHeadersFlat: _*) - .method(upperCaseVerb, + .headers(headersKeyValueAlternating: _*) + .method( + upperCaseVerb, (contentLengthHeader.headOption.map(_._2), compress) match { - case (Some("0"), _) => HttpRequest.BodyPublishers.noBody() - case (Some(n), Compress.None) => HttpRequest.BodyPublishers.fromPublisher(bodyPublisher, n.toInt) - case _ => bodyPublisher - } + case (Some("0"), _) => HttpRequest.BodyPublishers.noBody() + case (Some(n), Compress.None) => + HttpRequest.BodyPublishers.fromPublisher(bodyPublisher, n.toInt) + case _ => bodyPublisher + }, ) - val fut = httpClient.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()) + val fut = httpClient.sendAsync( + requestBuilder.build(), + HttpResponse.BodyHandlers.ofInputStream(), + ) - usingOutputStream(compress.wrap(requestBodyOutputStream)) { os => - data.write(os) - } + usingOutputStream(compress.wrap(requestBodyOutputStream)) { os => data.write(os) } val response = try @@ -276,27 +303,36 @@ case class Requester(verb: String, catch { case e: ExecutionException => throw e.getCause match { - case e: javax.net.ssl.SSLHandshakeException => new InvalidCertException(url, e) + case e: javax.net.ssl.SSLHandshakeException => new InvalidCertException(url, e) case _: HttpConnectTimeoutException | _: HttpTimeoutException => new TimeoutException(url, readTimeout, connectTimeout) - case e: java.net.UnknownHostException => - new UnknownHostException(url, e.getMessage) - case e: java.net.ConnectException => - new UnknownHostException(url, e.getMessage) - case e => - new RequestsException(e.getMessage, Some(e)) + case e: java.net.UnknownHostException => new UnknownHostException(url, e.getMessage) + case e: java.net.ConnectException => new UnknownHostException(url, e.getMessage) + case e => new RequestsException(e.getMessage, Some(e)) } } val responseCode = response.statusCode() val headerFields = - response.headers().map.asScala + response + .headers() + .map + .asScala .filter(_._1 != null) - .map { case (k, v) => (k.toLowerCase(), v.asScala.toList) }.toMap - - val deGzip = autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("gzip")) + .map { case (k, v) => (k.toLowerCase(), v.asScala.toList) } + .toMap + + val deGzip = autoDecompress && headerFields + .get("content-encoding") + .toSeq + .flatten + .exists(_.contains("gzip")) val deDeflate = - autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("deflate")) + autoDecompress && headerFields + .get("content-encoding") + .toSeq + .flatten + .exists(_.contains("deflate")) def persistCookies() = { if (sess.persistCookies) { headerFields @@ -308,9 +344,11 @@ case class Requester(verb: String, } } - if (responseCode.toString.startsWith("3") && - responseCode.toString != "304" && - maxRedirects > 0) { + if ( + responseCode.toString.startsWith("3") && + responseCode.toString != "304" && + maxRedirects > 0 + ) { val out = new ByteArrayOutputStream() Util.transferTo(response.body, out) val bytes = out.toByteArray @@ -321,7 +359,7 @@ case class Requester(verb: String, statusMessage = StatusMessages.byStatusCode.getOrElse(responseCode, ""), data = new geny.Bytes(bytes), headers = headerFields, - history = redirectedFrom + history = redirectedFrom, ) persistCookies() val newUrl = current.headers("location").head @@ -347,7 +385,7 @@ case class Requester(verb: String, check = check, chunkedUpload = chunkedUpload, redirectedFrom = Some(current), - onHeadersReceived = onHeadersReceived + onHeadersReceived = onHeadersReceived, ).readBytesThrough(f) } else { persistCookies() @@ -356,7 +394,7 @@ case class Requester(verb: String, statusCode = responseCode, statusMessage = StatusMessages.byStatusCode.getOrElse(responseCode, ""), headers = headerFields, - history = redirectedFrom + history = redirectedFrom, ) if (onHeadersReceived != null) onHeadersReceived(streamHeaders) @@ -368,17 +406,20 @@ case class Requester(verb: String, // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 if (upperCaseVerb == "HEAD") f(new ByteArrayInputStream(Array())) else if (stream != null) { - try f( - if (deGzip) new GZIPInputStream(stream) - else if (deDeflate) new InflaterInputStream(stream) - else stream - ) finally if (!keepAlive) stream.close() + try + f( + if (deGzip) new GZIPInputStream(stream) + else if (deDeflate) new InflaterInputStream(stream) + else stream, + ) + finally if (!keepAlive) stream.close() } else { f(new ByteArrayInputStream(Array())) } } - if (streamHeaders.statusCode == 304 || streamHeaders.is2xx || !check) processWrappedStream(f) + if (streamHeaders.statusCode == 304 || streamHeaders.is2xx || !check) + processWrappedStream(f) else { val errorOutput = new ByteArrayOutputStream() processWrappedStream(geny.Internal.transfer(_, errorOutput)) @@ -389,49 +430,55 @@ case class Requester(verb: String, statusMessage = streamHeaders.statusMessage, data = new geny.Bytes(errorOutput.toByteArray), headers = streamHeaders.headers, - history = streamHeaders.history - ) + history = streamHeaders.history, + ), ) } } } } - private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T): Unit = - try fn(os) finally os.close() + private def usingOutputStream[T](os: OutputStream)( + fn: OutputStream => T, + ): Unit = + try fn(os) + finally os.close() /** - * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration - */ - def apply(r: Request, data: RequestBlob, chunkedUpload: Boolean): Response = apply( - r.url, - r.auth, - r.params, - r.headers, - data, - r.readTimeout, - r.connectTimeout, - r.proxy, - r.cert, - r.sslContext, - r.cookies, - r.cookieValues, - r.maxRedirects, - r.verifySslCerts, - r.autoDecompress, - r.compress, - r.keepAlive, - r.check, - chunkedUpload - ) + * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration + */ + def apply(r: Request, data: RequestBlob, chunkedUpload: Boolean): Response = + apply( + r.url, + r.auth, + r.params, + r.headers, + data, + r.readTimeout, + r.connectTimeout, + r.proxy, + r.cert, + r.sslContext, + r.cookies, + r.cookieValues, + r.maxRedirects, + r.verifySslCerts, + r.autoDecompress, + r.compress, + r.keepAlive, + r.check, + chunkedUpload, + ) /** * Overload of [[Requester.stream]] that takes a [[Request]] object as configuration */ - def stream(r: Request, - data: RequestBlob, - chunkedUpload: Boolean, - onHeadersReceived: StreamHeaders => Unit): geny.Writable = + def stream( + r: Request, + data: RequestBlob, + chunkedUpload: Boolean, + onHeadersReceived: StreamHeaders => Unit, + ): geny.Writable = stream( url = r.url, auth = r.auth, @@ -454,6 +501,6 @@ case class Requester(verb: String, check = r.check, chunkedUpload = chunkedUpload, redirectedFrom = None, - onHeadersReceived = onHeadersReceived + onHeadersReceived = onHeadersReceived, ) } diff --git a/requests/src/requests/Session.scala b/requests/src/requests/Session.scala index e1b4633..29d5aad 100644 --- a/requests/src/requests/Session.scala +++ b/requests/src/requests/Session.scala @@ -6,39 +6,45 @@ import javax.net.ssl.SSLContext import scala.collection.mutable /** - * A long-lived session; this can be used to automatically persist cookies - * from one request to the next, or to set default configuration that will - * be shared between requests. These configuration flags can all be - * over-ridden by the parameters on [[Requester.apply]] or [[Requester.stream]] - * - * @param auth HTTP authentication you want to use with this request; defaults to none - * @param headers Custom headers to use, in addition to the defaults - * @param readTimeout How long to wait for data to be read before timing out - * @param connectTimeout How long to wait for a connection before timing out - * @param proxy Host and port of a proxy you want to use - * @param cookies Custom cookies to send up with this request - * @param maxRedirects How many redirects to automatically resolve; defaults to 5. - * You can also set it to 0 to prevent Requests from resolving - * redirects for you - * @param verifySslCerts Set this to false to ignore problems with SSL certificates - */ -case class Session(headers: Map[String, String] = BaseSession.defaultHeaders, - cookieValues: Map[String, String] = Map(), - cookies: mutable.Map[String, HttpCookie] = mutable.LinkedHashMap.empty[String, HttpCookie], - auth: RequestAuth = RequestAuth.Empty, - proxy: (String, Int) = null, - cert: Cert = null, - sslContext: SSLContext = null, - persistCookies: Boolean = true, - maxRedirects: Int = 5, - readTimeout: Int = 10 * 1000, - connectTimeout: Int = 10 * 1000, - verifySslCerts: Boolean = true, - autoDecompress: Boolean = true, - compress: Compress = Compress.None, - chunkedUpload: Boolean = false, - check: Boolean = true) - extends BaseSession{ - - for((k, v) <- cookieValues) cookies(k) = new HttpCookie(k, v) + * A long-lived session; this can be used to automatically persist cookies from one request to the + * next, or to set default configuration that will be shared between requests. These configuration + * flags can all be over-ridden by the parameters on [[Requester.apply]] or [[Requester.stream]] + * + * @param auth + * HTTP authentication you want to use with this request; defaults to none + * @param headers + * Custom headers to use, in addition to the defaults + * @param readTimeout + * How long to wait for data to be read before timing out + * @param connectTimeout + * How long to wait for a connection before timing out + * @param proxy + * Host and port of a proxy you want to use + * @param cookies + * Custom cookies to send up with this request + * @param maxRedirects + * How many redirects to automatically resolve; defaults to 5. You can also set it to 0 to prevent + * Requests from resolving redirects for you + * @param verifySslCerts + * Set this to false to ignore problems with SSL certificates + */ +case class Session( + headers: Map[String, String] = BaseSession.defaultHeaders, + cookieValues: Map[String, String] = Map(), + cookies: mutable.Map[String, HttpCookie] = mutable.LinkedHashMap.empty[String, HttpCookie], + auth: RequestAuth = RequestAuth.Empty, + proxy: (String, Int) = null, + cert: Cert = null, + sslContext: SSLContext = null, + persistCookies: Boolean = true, + maxRedirects: Int = 5, + readTimeout: Int = 10 * 1000, + connectTimeout: Int = 10 * 1000, + verifySslCerts: Boolean = true, + autoDecompress: Boolean = true, + compress: Compress = Compress.None, + chunkedUpload: Boolean = false, + check: Boolean = true, +) extends BaseSession { + for ((k, v) <- cookieValues) cookies(k) = new HttpCookie(k, v) } diff --git a/requests/src/requests/StatusMessages.scala b/requests/src/requests/StatusMessages.scala index 56930cb..bc5fb30 100644 --- a/requests/src/requests/StatusMessages.scala +++ b/requests/src/requests/StatusMessages.scala @@ -90,6 +90,6 @@ object StatusMessages { 527 -> "Railgun Error", 530 -> "Site is Frozen", 598 -> "Network Read Timeout Error", - 599 -> "Network Connect Timeout Error" + 599 -> "Network Connect Timeout Error", ) } diff --git a/requests/src/requests/Util.scala b/requests/src/requests/Util.scala index 3d6c425..5b11a0d 100644 --- a/requests/src/requests/Util.scala +++ b/requests/src/requests/Util.scala @@ -7,11 +7,13 @@ import java.security.cert.X509Certificate import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManager, X509TrustManager} object Util { - def transferTo(is: InputStream, - os: OutputStream, - bufferSize: Int = 8 * 1024) = { + def transferTo( + is: InputStream, + os: OutputStream, + bufferSize: Int = 8 * 1024, + ) = { val buffer = new Array[Byte](bufferSize) - while ( { + while ({ is.read(buffer) match { case -1 => false case n => @@ -22,8 +24,9 @@ object Util { } def urlEncode(x: Iterable[(String, String)]) = { - x.map{case (k, v) => URLEncoder.encode(k, "UTF-8") + "=" + URLEncoder.encode(v, "UTF-8")} - .mkString("&") + x.map { + case (k, v) => URLEncoder.encode(k, "UTF-8") + "=" + URLEncoder.encode(v, "UTF-8") + }.mkString("&") } private[requests] val noVerifySSLContext = { @@ -39,9 +42,11 @@ object Util { private[requests] val noVerifySocketFactory = noVerifySSLContext.getSocketFactory - private[requests] def clientCertSSLContext(cert: Cert, verifySslCerts: Boolean) = cert match { + private[requests] def clientCertSSLContext( + cert: Cert, + verifySslCerts: Boolean, + ) = cert match { case Cert.P12(path, password) => - val pass = password.map(_.toCharArray).getOrElse(Array.emptyCharArray) val keyManagers = { @@ -61,8 +66,10 @@ object Util { } @deprecated("No longer used", "0.9.0") - private[requests] def clientCertSocketFactory(cert: Cert, verifySslCerts: Boolean) = - clientCertSSLContext(cert, verifySslCerts).getSocketFactory + private[requests] def clientCertSocketFactory( + cert: Cert, + verifySslCerts: Boolean, + ) = clientCertSSLContext(cert, verifySslCerts).getSocketFactory private lazy val trustAllCerts = Array[TrustManager](new X509TrustManager() { def getAcceptedIssuers = new Array[X509Certificate](0) diff --git a/requests/src/requests/package.scala b/requests/src/requests/package.scala index 655f5b0..0a15a4e 100644 --- a/requests/src/requests/package.scala +++ b/requests/src/requests/package.scala @@ -34,4 +34,4 @@ package object requests extends _root_.requests.BaseSession { def chunkedUpload: Boolean = false def check: Boolean = true -} \ No newline at end of file +} diff --git a/requests/test/resources/badssl.com-client.md b/requests/test/resources/badssl.com-client.md index 16fda8e..20bebcf 100644 --- a/requests/test/resources/badssl.com-client.md +++ b/requests/test/resources/badssl.com-client.md @@ -31,4 +31,6 @@ And then answer ` ` for no password. Remove temporary files. -```rm badssl.com.ca-cert.ca badssl.com.crt badssl.com.private.key badssl.com.private-nopass.key temp.pem``` \ No newline at end of file +``` +rm badssl.com.ca-cert.ca badssl.com.crt badssl.com.private.key badssl.com.private-nopass.key temp.pem +``` diff --git a/requests/test/src-2/requests/Scala2RequestTests.scala b/requests/test/src-2/requests/Scala2RequestTests.scala index 9a9d787..e97bc73 100644 --- a/requests/test/src-2/requests/Scala2RequestTests.scala +++ b/requests/test/src-2/requests/Scala2RequestTests.scala @@ -3,41 +3,45 @@ package requests import utest._ import ujson._ -object Scala2RequestTests extends TestSuite{ - val tests = Tests{ - - test("params"){ - - test("post"){ - for(chunkedUpload <- Seq(true, false)) { - val res1 = requests.post( - "https://httpbin.org/post", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = chunkedUpload - ).text() +object Scala2RequestTests extends HttpbinTestSuite { + val tests = Tests { + test("params") { + test("post") { + for (chunkedUpload <- Seq(true, false)) { + val res1 = requests + .post( + s"http://$localHttpbin/post", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = chunkedUpload, + ) + .text() assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } test("put") { for (chunkedUpload <- Seq(true, false)) { - val res1 = requests.put( - "https://httpbin.org/put", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = chunkedUpload - ).text() + val res1 = requests + .put( + s"http://$localHttpbin/put", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = chunkedUpload, + ) + .text() assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } - test("send"){ - requests.send("get")("https://httpbin.org/get?hello=world&foo=baz") + test("send") { + requests.send("get")(s"http://$localHttpbin/get?hello=world&foo=baz") - val res1 = requests.send("put")( - "https://httpbin.org/put", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = true - ).text + val res1 = requests + .send("put")( + s"http://$localHttpbin/put", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = true, + ) + .text assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } diff --git a/requests/test/src/requests/FileUtils.scala b/requests/test/src/requests/FileUtils.scala index e2a8d8b..95ddc4b 100644 --- a/requests/test/src/requests/FileUtils.scala +++ b/requests/test/src/requests/FileUtils.scala @@ -7,11 +7,15 @@ import javax.net.ssl.{KeyManagerFactory, SSLContext} object FileUtils { - def createSslContext(keyStorePath: String, keyStorePassword: String): SSLContext = { + def createSslContext( + keyStorePath: String, + keyStorePassword: String, + ): SSLContext = { val stream: InputStream = new FileInputStream(keyStorePath) val sslContext = SSLContext.getInstance("TLS") - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + val keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) val keyStore = KeyStore.getInstance("PKCS12") keyStore.load(stream, keyStorePassword.toCharArray) keyManagerFactory.init(keyStore, keyStorePassword.toCharArray) diff --git a/requests/test/src/requests/HttpbinTestSuite.scala b/requests/test/src/requests/HttpbinTestSuite.scala new file mode 100644 index 0000000..d5d4566 --- /dev/null +++ b/requests/test/src/requests/HttpbinTestSuite.scala @@ -0,0 +1,21 @@ +package requests + +import com.dimafeng.testcontainers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait +import utest._ + +abstract class HttpbinTestSuite extends TestSuite { + private val containerDef = GenericContainer.Def( + "kennethreitz/httpbin", + exposedPorts = Seq(80), + waitStrategy = Wait.forHttp("/"), + ) + private val container = containerDef.start() + + val localHttpbin: String = + s"${container.containerIpAddress}:${container.mappedPort(80)}" + + override def utestAfterAll(): Unit = { + container.stop() + } +} diff --git a/requests/test/src/requests/ModelTests.scala b/requests/test/src/requests/ModelTests.scala index 4a4982d..dae218c 100644 --- a/requests/test/src/requests/ModelTests.scala +++ b/requests/test/src/requests/ModelTests.scala @@ -6,7 +6,7 @@ import java.nio.file.{FileSystems, Path} import utest._ -object ModelTests extends TestSuite{ +object ModelTests extends TestSuite { val tests = Tests { test("multipart file uploads should contain application/octet-stream content type") { val path = getClass.getResource("/license.zip").getPath @@ -14,32 +14,19 @@ object ModelTests extends TestSuite{ val nioPath = FileSystems.getDefault.getPath(path) val fileKey = "fileKey" val fileName = "fileName" - - val javaFileMultipart = MultiPart( - MultiItem( - fileKey, - file, - fileName - ) - ) - - val nioPathMultipart = MultiPart( - MultiItem( - fileKey, - nioPath, - fileName - ) - ) - + + val javaFileMultipart = MultiPart(MultiItem(fileKey, file, fileName)) + val nioPathMultipart = MultiPart(MultiItem(fileKey, nioPath, fileName)) + val javaFileOutputStream = new ByteArrayOutputStream() val nioPathOutputStream = new ByteArrayOutputStream() - + javaFileMultipart.write(javaFileOutputStream) nioPathMultipart.write(nioPathOutputStream) - + val javaFileString = new String(javaFileOutputStream.toByteArray) val nioPathString = new String(nioPathOutputStream.toByteArray) - + assert(javaFileString.contains("Content-Type: application/octet-stream")) assert(nioPathString.contains("Content-Type: application/octet-stream")) } diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala index 6d935ac..52c48cc 100644 --- a/requests/test/src/requests/RequestTests.scala +++ b/requests/test/src/requests/RequestTests.scala @@ -3,278 +3,292 @@ package requests import utest._ import ujson._ -object RequestTests extends TestSuite{ - val tests = Tests{ - test("matchingMethodWorks"){ - val requesters = Seq( - requests.delete, - requests.get, - requests.post, - requests.put - ) +object RequestTests extends HttpbinTestSuite { - for(protocol <- Seq("http", "https")){ - for(r <- requesters){ - for(r2 <- requesters){ - val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}", check = false) - if (r.verb == r2.verb) assert(res.statusCode == 200) - else assert(res.statusCode == 405) + val tests = Tests { + test("matchingMethodWorks") { + val requesters = Seq(requests.delete, requests.get, requests.post, requests.put) + val baseUrl = s"http://$localHttpbin" + for (r <- requesters) { + for (r2 <- requesters) { + val res = r(s"$baseUrl/${r2.verb.toLowerCase()}", check = false) + if (r.verb == r2.verb) assert(res.statusCode == 200) + else assert(res.statusCode == 405) - if (r.verb == r2.verb){ - val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}") - assert(res.statusCode == 200) - }else intercept[RequestFailedException]{ - r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}") + if (r.verb == r2.verb) { + val res = r(s"$baseUrl/${r2.verb.toLowerCase()}") + assert(res.statusCode == 200) + } else { + intercept[RequestFailedException] { + r(s"$baseUrl/${r2.verb.toLowerCase()}") } } } } } - test("params"){ - test("get"){ + test("params") { + test("get") { // All in URL - val res1 = requests.get("https://httpbin.org/get?hello=world&foo=baz").text() + val res1 = + requests.get(s"http://$localHttpbin/get?hello=world&foo=baz").text() assert(read(res1).obj("args") == Obj("foo" -> "baz", "hello" -> "world")) // All in params val res2 = requests.get( - "https://httpbin.org/get", - params = Map("hello" -> "world", "foo" -> "baz") + s"http://$localHttpbin/get", + params = Map("hello" -> "world", "foo" -> "baz"), ) assert(read(res2).obj("args") == Obj("foo" -> "baz", "hello" -> "world")) // Mixed URL and params - val res3 = requests.get( - "https://httpbin.org/get?hello=world", - params = Map("foo" -> "baz") - ).text() + val res3 = requests + .get( + s"http://$localHttpbin/get?hello=world", + params = Map("foo" -> "baz"), + ) + .text() assert(read(res3).obj("args") == Obj("foo" -> "baz", "hello" -> "world")) // Needs escaping val res4 = requests.get( - "https://httpbin.org/get?hello=world", - params = Map("++-- lol" -> " !@#$%") + s"http://$localHttpbin/get?hello=world", + params = Map("++-- lol" -> " !@#$%"), ) assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world")) } } - test("multipart"){ - for(chunkedUpload <- Seq(true, false)) { - val response = requests.post( - "http://httpbin.org/post", - data = MultiPart( - MultiItem("file1", "Hello!".getBytes, "foo.txt"), - MultiItem("file2", "Goodbye!") - ), - chunkedUpload = chunkedUpload - ).text() + test("multipart") { + for (chunkedUpload <- Seq(true, false)) { + val response = requests + .post( + s"http://$localHttpbin/post", + data = MultiPart( + MultiItem("file1", "Hello!".getBytes, "foo.txt"), + MultiItem("file2", "Goodbye!"), + ), + chunkedUpload = chunkedUpload, + ) + .text() assert(read(response).obj("files") == Obj("file1" -> "Hello!")) assert(read(response).obj("form") == Obj("file2" -> "Goodbye!")) } } - test("cookies"){ - test("session"){ + test("cookies") { + test("session") { val s = requests.Session(cookieValues = Map("hello" -> "world")) - val res1 = s.get("https://httpbin.org/cookies").text().trim + val res1 = s.get(s"http://$localHttpbin/cookies").text().trim assert(read(res1) == Obj("cookies" -> Obj("hello" -> "world"))) - s.get("https://httpbin.org/cookies/set?freeform=test") - val res2 = s.get("https://httpbin.org/cookies").text().trim + s.get(s"http://$localHttpbin/cookies/set?freeform=test") + val res2 = s.get(s"http://$localHttpbin/cookies").text().trim assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test", "hello" -> "world"))) } - test("raw"){ - val res1 = requests.get("https://httpbin.org/cookies").text().trim + test("raw") { + val res1 = requests.get(s"http://$localHttpbin/cookies").text().trim assert(read(res1) == Obj("cookies" -> Obj())) - requests.get("https://httpbin.org/cookies/set?freeform=test") - val res2 = requests.get("https://httpbin.org/cookies").text().trim + requests.get(s"http://$localHttpbin/cookies/set?freeform=test") + val res2 = requests.get(s"http://$localHttpbin/cookies").text().trim assert(read(res2) == Obj("cookies" -> Obj())) } - test("space"){ + test("space") { val s = requests.Session(cookieValues = Map("hello" -> "hello, world")) - val res1 = s.get("https://httpbin.org/cookies").text().trim + val res1 = s.get(s"http://$localHttpbin/cookies").text().trim assert(read(res1) == Obj("cookies" -> Obj("hello" -> "hello, world"))) - s.get("https://httpbin.org/cookies/set?freeform=test+test") - val res2 = s.get("https://httpbin.org/cookies").text().trim - assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world"))) + s.get(s"http://$localHttpbin/cookies/set?freeform=test+test") + val res2 = s.get(s"http://$localHttpbin/cookies").text().trim + assert( + read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world")), + ) } } - test("redirects"){ - test("max"){ - val res1 = requests.get("https://httpbin.org/absolute-redirect/4") + test("redirects") { + test("max") { + val res1 = requests.get(s"http://$localHttpbin/absolute-redirect/4") assert(res1.statusCode == 200) - val res2 = requests.get("https://httpbin.org/absolute-redirect/5") + val res2 = requests.get(s"http://$localHttpbin/absolute-redirect/5") assert(res2.statusCode == 200) - val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false) + val res3 = requests.get(s"http://$localHttpbin/absolute-redirect/6", check = false) assert(res3.statusCode == 302) - val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10) + val res4 = requests.get(s"http://$localHttpbin/absolute-redirect/6", maxRedirects = 10) assert(res4.statusCode == 200) } - test("maxRelative"){ - val res1 = requests.get("https://httpbin.org/relative-redirect/4") + test("maxRelative") { + val res1 = requests.get(s"http://$localHttpbin/relative-redirect/4") assert(res1.statusCode == 200) - val res2 = requests.get("https://httpbin.org/relative-redirect/5") + val res2 = requests.get(s"http://$localHttpbin/relative-redirect/5") assert(res2.statusCode == 200) - val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false) + val res3 = requests.get(s"http://$localHttpbin/relative-redirect/6", check = false) assert(res3.statusCode == 302) - val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10) + val res4 = requests.get(s"http://$localHttpbin/relative-redirect/6", maxRedirects = 10) assert(res4.statusCode == 200) } } - test("test_reproduction"){ - requests.get("http://httpbin.org/status/304").text() - + test("test_reproduction") { + requests.get(s"http://$localHttpbin/status/304").text() } - test("streaming"){ - val res1 = requests.get("http://httpbin.org/stream/5").text() + + test("streaming") { + val res1 = requests.get(s"http://$localHttpbin/stream/5").text() assert(res1.linesIterator.length == 5) - val res2 = requests.get("http://httpbin.org/stream/52").text() + val res2 = requests.get(s"http://$localHttpbin/stream/52").text() assert(res2.linesIterator.length == 52) } - test("timeouts"){ - test("read"){ + test("timeouts") { + test("read") { intercept[TimeoutException] { - requests.get("https://httpbin.org/delay/1", readTimeout = 10) + requests.get(s"http://$localHttpbin/delay/1", readTimeout = 10) } - requests.get("https://httpbin.org/delay/1", readTimeout = 2000) + requests.get(s"http://$localHttpbin/delay/1", readTimeout = 2000) intercept[TimeoutException] { - requests.get("https://httpbin.org/delay/3", readTimeout = 2000) + requests.get(s"http://$localHttpbin/delay/3", readTimeout = 2000) } } - test("connect"){ + test("connect") { intercept[TimeoutException] { - requests.get("https://httpbin.org/delay/1", connectTimeout = 1) + // use remote httpbin.org so it needs more time to connect + requests.get(s"https://httpbin.org/delay/1", connectTimeout = 1) } } } - test("failures"){ - intercept[UnknownHostException]{ + test("failures") { + intercept[UnknownHostException] { requests.get("https://doesnt-exist-at-all.com/") } - intercept[InvalidCertException]{ + intercept[InvalidCertException] { requests.get("https://expired.badssl.com/") } requests.get("https://doesnt-exist.com/", verifySslCerts = false) - intercept[java.net.MalformedURLException]{ + intercept[java.net.MalformedURLException] { requests.get("://doesnt-exist.com/") } } - test("decompress"){ - val res1 = requests.get("https://httpbin.org/gzip") - assert(read(res1.text()).obj("headers").obj("Host").str == "httpbin.org") + test("decompress") { + val res1 = requests.get(s"http://$localHttpbin/gzip") + assert(read(res1.text()).obj("headers").obj("Host").str == localHttpbin) - val res2 = requests.get("https://httpbin.org/deflate") - assert(read(res2).obj("headers").obj("Host").str == "httpbin.org") + val res2 = requests.get(s"http://$localHttpbin/deflate") + assert(read(res2).obj("headers").obj("Host").str == localHttpbin) - val res3 = requests.get("https://httpbin.org/gzip", autoDecompress = false) + val res3 = requests.get(s"http://$localHttpbin/gzip", autoDecompress = false) assert(res3.bytes.length < res1.bytes.length) - val res4 = requests.get("https://httpbin.org/deflate", autoDecompress = false) + val res4 = requests.get(s"http://$localHttpbin/deflate", autoDecompress = false) assert(res4.bytes.length < res2.bytes.length) - (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length) + ( + res1.bytes.length, + res2.bytes.length, + res3.bytes.length, + res4.bytes.length, + ) } - test("compression"){ + test("compression") { val res1 = requests.post( - "https://httpbin.org/post", + s"http://$localHttpbin/post", compress = requests.Compress.None, - data = new RequestBlob.ByteSourceRequestBlob("Hello World") + data = new RequestBlob.ByteSourceRequestBlob("Hello World"), ) assert(res1.text().contains(""""Hello World"""")) val res2 = requests.post( - "https://httpbin.org/post", + s"http://$localHttpbin/post", compress = requests.Compress.Gzip, - data = new RequestBlob.ByteSourceRequestBlob("I am cow") + data = new RequestBlob.ByteSourceRequestBlob("I am cow"), + ) + assert( + read(new String(res2.bytes))("data").toString + .contains("data:application/octet-stream;base64,H4sIAAAAAA"), ) - assert(read(new String(res2.bytes))("data").toString.contains("data:application/octet-stream;base64,H4sIAAAAAA")) val res3 = requests.post( - "https://httpbin.org/post", + s"http://$localHttpbin/post", compress = requests.Compress.Deflate, - data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") + data = new RequestBlob.ByteSourceRequestBlob("Hear me moo"), ) - assert(read(new String(res3.bytes))("data").toString == - """"data:application/octet-stream;base64,eJzzSE0sUshNVcjNzwcAFokD3g=="""") - } + assert( + read(new String(res3.bytes))( + "data", + ).toString == """"data:application/octet-stream;base64,eJzzSE0sUshNVcjNzwcAFokD3g=="""", + ) + } - test("headers"){ - test("default"){ - val res = requests.get("https://httpbin.org/headers").text() + test("headers") { + test("default") { + val res = requests.get(s"http://$localHttpbin/headers").text() val hs = read(res)("headers").obj assert(hs("User-Agent").str == "requests-scala") assert(hs("Accept-Encoding").str == "gzip, deflate") assert(hs("Accept").str == "*/*") - test("hasNoCookie"){ + test("hasNoCookie") { assert(!hs.contains("Cookie")) } } } - test("clientCertificate"){ - val base = "./requests/test/resources" + test("clientCertificate") { + val base = sys.env("MILL_TEST_RESOURCE_DIR") val url = "https://client.badssl.com" - val instruction = "https://github.com/lihaoyi/requests-scala/blob/master/requests/test/resources/badssl.com-client.md" - val certificateExpiredMessage = s"WARNING: Certificate may have expired and needs to be updated. Please check: $instruction and/or file issue" - test("passwordProtected"){ + val instruction = + "https://github.com/lihaoyi/requests-scala/blob/master/requests/test/resources/badssl.com-client.md" + val certificateExpiredMessage = + s"WARNING: Certificate may have expired and needs to be updated. Please check: $instruction and/or file issue" + test("passwordProtected") { val res = requests.get( url, cert = Cert.implicitP12(s"$base/badssl.com-client.p12", "badssl.com"), - check = false + check = false, ) if (res.statusCode == 400) println(certificateExpiredMessage) else assert(res.statusCode == 200) } - test("noPassword"){ + test("noPassword") { val res = requests.get( "https://client.badssl.com", cert = Cert.implicitP12(s"$base/badssl.com-client-nopass.p12"), - check = false + check = false, ) if (res.statusCode == 400) println(certificateExpiredMessage) else assert(res.statusCode == 200) } - test("sslContext"){ + test("sslContext") { val res = requests.get( "https://client.badssl.com", sslContext = FileUtils.createSslContext(s"$base/badssl.com-client.p12", "badssl.com"), - check = false + check = false, ) if (res.statusCode == 400) println(certificateExpiredMessage) else assert(res.statusCode == 200) } - test("noCert"){ - val res = requests.get( - "https://client.badssl.com", - check = false - ) + test("noCert") { + val res = requests.get("https://client.badssl.com", check = false) assert(res.statusCode == 400) } } - test("selfSignedCertificate"){ + test("selfSignedCertificate") { val res = requests.get( "https://self-signed.badssl.com", - verifySslCerts = false + verifySslCerts = false, ) assert(res.statusCode == 200) } - test("gzipError"){ + test("gzipError") { val response = requests.head("https://api.github.com/users/lihaoyi") assert(response.statusCode == 200) assert(response.data.array.isEmpty) @@ -283,9 +297,9 @@ object RequestTests extends TestSuite{ } /** - * Compress with each compression mode and call server. Server expands - * and passes it back so we can compare - */ + * Compress with each compression mode and call server. Server expands and passes it back so we + * can compare + */ test("compressionData") { import requests.Compress._ val str = "I am deflater mouse" @@ -295,11 +309,23 @@ object RequestTests extends TestSuite{ requests.post( s"http://localhost:$port/echo", compress = c, - data = new RequestBlob.ByteSourceRequestBlob(str) + data = new RequestBlob.ByteSourceRequestBlob(str), ) assert(str == response.data.toString) } } } + + // Ensure when duplicate headers are passed to requests, we only pass the last one + // to the server. This preserves the 0.8.x behavior, and can always be overriden + // by passing a comma-separated list of headers instead + test("duplicateHeaders") { + val res = requests.get( + s"http://$localHttpbin/get", + headers = Seq("x-y" -> "a", "x-y" -> "b"), + ) + // make sure it's not "a,b" + assert(ujson.read(res)("headers")("X-Y") == Str("b")) + } } } diff --git a/requests/test/src/requests/ServerUtils.scala b/requests/test/src/requests/ServerUtils.scala index 81b0ea0..341ce83 100644 --- a/requests/test/src/requests/ServerUtils.scala +++ b/requests/test/src/requests/ServerUtils.scala @@ -16,7 +16,8 @@ object ServerUtils { } private class EchoServer extends HttpHandler { - private val server: HttpServer = HttpServer.create(new InetSocketAddress(0), 0) + private val server: HttpServer = + HttpServer.create(new InetSocketAddress(0), 0) server.createContext("/echo", this) server.setExecutor(null); // default executor server.start() @@ -43,10 +44,11 @@ object ServerUtils { } } - /** Stream uncompresser - * @param c - * Compression mode - */ + /** + * Stream uncompresser + * @param c + * Compression mode + */ private class Plumper(c: Compress) { private def wrap(is: InputStream): InputStream = @@ -77,5 +79,4 @@ object ServerUtils { sb.toString() } } - }