From 572dfacb57f34e7efd3694830cb05fb9bc205770 Mon Sep 17 00:00:00 2001 From: Victor Duvert Date: Fri, 7 May 2021 15:40:52 +0200 Subject: [PATCH 1/3] add hawk authentication --- docs/HTTP-batchsource.md | 18 +++ docs/HTTP-streamingsource.md | 18 +++ pom.xml | 8 +- .../source/common/BaseHttpSourceConfig.java | 107 ++++++++++++++ .../http/source/common/http/HawkUtil.java | 43 ++++++ .../http/source/common/http/HttpClient.java | 33 ++++- widgets/HTTP-batchsource.json | 132 ++++++++++++++++- widgets/HTTP-streamingsource.json | 136 +++++++++++++++++- 8 files changed, 477 insertions(+), 18 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java diff --git a/docs/HTTP-batchsource.md b/docs/HTTP-batchsource.md index 37ed1e2d..02efe928 100644 --- a/docs/HTTP-batchsource.md +++ b/docs/HTTP-batchsource.md @@ -390,6 +390,24 @@ is stopped. **Refresh Token:** Token used to receive accessToken, which is end product of OAuth2. +### Hawk Authentication + +**HAWK Authentication Enabled:** If true, plugin will perform HAWK authentication. + +**HAWK Auth ID:** HAWK Authentication ID + +**Hawk Auth Key:** HAWK Authentication Key + +**Algorithm:** Hash Algorithm used + +**ext:** Any application-specific information to be sent with the request. Ex: some-app-extra-data + +**app:** This provides binding between the credentials and the application in a way that prevents an attacker from ticking an application to use credentials issued to someone else. + +**dlg:** The application id of the application the credentials were directly issued to. + +**Include Payload Hash:** HAWK authentication provides optional support for payload validation. If this option is selected, the payload hash will be calculated and included in MAC calculation and in Authorization header + ### SSL/TLS **Verify HTTPS Trust Certificates:** If false, untrusted trust certificates (e.g. self signed), will not lead to an diff --git a/docs/HTTP-streamingsource.md b/docs/HTTP-streamingsource.md index 4cad538e..ef2b9c19 100644 --- a/docs/HTTP-streamingsource.md +++ b/docs/HTTP-streamingsource.md @@ -397,6 +397,24 @@ is stopped. **Refresh Token:** Token used to receive accessToken, which is end product of OAuth2. +### Hawk Authentication + +**HAWK Authentication Enabled:** If true, plugin will perform HAWK authentication. + +**HAWK Auth ID:** HAWK Authentication ID + +**Hawk Auth Key:** HAWK Authentication Key + +**Algorithm:** Hash Algorithm used + +**ext:** Any application-specific information to be sent with the request. Ex: some-app-extra-data + +**app:** This provides binding between the credentials and the application in a way that prevents an attacker from ticking an application to use credentials issued to someone else. + +**dlg:** The application id of the application the credentials were directly issued to. + +**Include Payload Hash:** HAWK authentication provides optional support for payload validation. If this option is selected, the payload hash will be calculated and included in MAC calculation and in Authorization header + ### SSL/TLS **Verify HTTPS Trust Certificates:** If false, untrusted trust certificates (e.g. self signed), will not lead to an diff --git a/pom.xml b/pom.xml index d49ef2e5..74bc1d88 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ 2.8.5 2.3.0 4.5.9 - 2.4.0-SNAPSHOT + 2.6.0 2.9.9 4.11 2.7.1 @@ -354,7 +354,11 @@ jython-standalone ${jython.version} - + + com.wealdtech.hawk + hawk-core + 1.0.0 + diff --git a/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java b/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java index a554dd6c..f99da291 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java +++ b/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java @@ -16,6 +16,7 @@ package io.cdap.plugin.http.source.common; import com.google.common.base.Strings; +import com.wealdtech.hawk.HawkCredentials; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -87,6 +88,14 @@ public abstract class BaseHttpSourceConfig extends ReferencePluginConfig { public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; public static final String PROPERTY_SCOPES = "scopes"; public static final String PROPERTY_REFRESH_TOKEN = "refreshToken"; + public static final String PROPERTY_HAWK_AUTH_ENABLED = "hawkAuthEnabled"; + public static final String PROPERTY_HAWK_AUTH_ID = "hawkAuthID"; + public static final String PROPERTY_HAWK_AUTH_KEY = "hawkAuthKey"; + public static final String PROPERTY_HAWK_AUTH_ALGORITHM = "hawkAlgorithm"; + public static final String PROPERTY_HAWK_AUTH_EXT = "hawkExt"; + public static final String PROPERTY_HAWK_AUTH_APP = "hawkApp"; + public static final String PROPERTY_HAWK_AUTH_DLG = "hawkDlg"; + public static final String PROPERTY_HAWK_PAYLOAD_HASH_ENABLED = "hawkPayloadHashEnabled"; public static final String PROPERTY_VERIFY_HTTPS = "verifyHttps"; public static final String PROPERTY_KEYSTORE_FILE = "keystoreFile"; public static final String PROPERTY_KEYSTORE_TYPE = "keystoreType"; @@ -316,6 +325,56 @@ public abstract class BaseHttpSourceConfig extends ReferencePluginConfig { @Macro protected String refreshToken; + @Name(PROPERTY_HAWK_AUTH_ENABLED) + @Description("If true, plugin will perform OAuth2 authentication.") + protected String hawkAuthEnabled; + + @Nullable + @Name(PROPERTY_HAWK_AUTH_ID) + @Description("The HAWK Authentication ID") + @Macro + protected String hawkAuthID; + + @Nullable + @Name(PROPERTY_HAWK_AUTH_KEY) + @Description("The HAWK Authentication Key") + @Macro + protected String hawkAuthKey; + + @Nullable + @Name(PROPERTY_HAWK_AUTH_ALGORITHM) + @Description("The HAWK Algorithm") + @Macro + protected String hawkAlgorithm; + + @Nullable + @Name(PROPERTY_HAWK_AUTH_EXT) + @Description("Advanced parameter : Any application-specific information to be sent with the request. " + + "Ex: some-app-extra-data") + @Macro + protected String hawkExt; + + @Nullable + @Name(PROPERTY_HAWK_AUTH_APP) + @Description("Advanced parameter : This provides binding between the credentials and the application " + + "in a way that prevents an attacker from ticking an application to use credentials issued to someone else.") + @Macro + protected String hawkApp; + + @Nullable + @Name(PROPERTY_HAWK_AUTH_DLG) + @Description("Advanced parameter : The application id of the application the credentials were directly issued to.") + @Macro + protected String hawkDlg; + + @Nullable + @Name(PROPERTY_HAWK_PAYLOAD_HASH_ENABLED) + @Description("Advanced parameter : HAWK authentication provides optional support for payload validation. " + + "If this option is selected, the payload hash will be calculated and included in MAC calculation " + + "and in Authorization header") + @Macro + protected String hawkPayloadHashEnabled; + @Name(PROPERTY_VERIFY_HTTPS) @Description("If false, untrusted trust certificates (e.g. self signed), will not lead to an" + "error. Do not disable this in production environment on a network you do not entirely trust. " + @@ -563,6 +622,46 @@ public String getRefreshToken() { return refreshToken; } + + public boolean getHawkAuthEnabled() { + return Boolean.parseBoolean(hawkAuthEnabled); + } + + @Nullable + public String getHawkAuthID() { + return hawkAuthID; + } + + @Nullable + public String getHawkAuthKey() { + return hawkAuthKey; + } + + @Nullable + public HawkCredentials.Algorithm getHawkAlgorithm() { + return HawkCredentials.Algorithm.parse(hawkAlgorithm); + } + + @Nullable + public String getHawkExt() { + return hawkExt; + } + + @Nullable + public String getHawkApp() { + return hawkApp; + } + + @Nullable + public String getHawkDlg() { + return hawkDlg; + } + + @Nullable + public boolean getHawkPayloadHashEnabled() { + return Boolean.parseBoolean(hawkPayloadHashEnabled); + } + public Boolean getVerifyHttps() { return Boolean.parseBoolean(verifyHttps); } @@ -794,6 +893,14 @@ PAGINATION_INDEX_PLACEHOLDER, getPaginationType()), assertIsSet(getRefreshToken(), PROPERTY_REFRESH_TOKEN, reasonOauth2); } + // Validate HAWK auth properties + if (!containsMacro(PROPERTY_HAWK_AUTH_ENABLED) && this.getHawkAuthEnabled()) { + String reasonHAWK = "HAWK Authentication is enabled"; + assertIsSet(getHawkAuthID(), PROPERTY_HAWK_AUTH_ID, reasonHAWK); + assertIsSet(getHawkAuthKey(), PROPERTY_HAWK_AUTH_KEY, reasonHAWK); + assertIsSet(getHawkAlgorithm(), PROPERTY_HAWK_AUTH_ALGORITHM, reasonHAWK); + } + if (!containsMacro(PROPERTY_VERIFY_HTTPS) && !getVerifyHttps()) { assertIsNotSet(getTrustStoreFile(), PROPERTY_TRUSTSTORE_FILE, String.format("trustore settings are ignored due to disabled %s", PROPERTY_VERIFY_HTTPS)); diff --git a/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java b/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java new file mode 100644 index 00000000..3858e9f6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java @@ -0,0 +1,43 @@ +package io.cdap.plugin.http.source.common.http; + +import com.wealdtech.hawk.HawkClient; +import com.wealdtech.hawk.HawkCredentials; +import org.apache.http.entity.StringEntity; + +import java.net.URI; + +public class HawkUtil { + + public static HawkClient createHawkClient(String authID, String authKey, HawkCredentials.Algorithm algorithm) { + HawkCredentials hawkCredentials = new HawkCredentials.Builder() + .keyId(authID) + .key(authKey) + .algorithm(algorithm) + .build(); + + return new HawkClient.Builder().credentials(hawkCredentials).build(); + } + + public static String getAuthorizationHeader( + HawkClient hawkClient, + StringEntity requestBody, + URI uri, + String method, + boolean payloadHashEnabled, + String ext, + String app, + String dlg){ + String hash = null; + if (payloadHashEnabled) { + hash = Integer.toString(requestBody.hashCode()); + } + + return hawkClient.generateAuthorizationHeader( + uri, + method, + hash, + ext, + app, + dlg); + } +} diff --git a/src/main/java/io/cdap/plugin/http/source/common/http/HttpClient.java b/src/main/java/io/cdap/plugin/http/source/common/http/HttpClient.java index bb41293c..219a31cf 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/http/HttpClient.java +++ b/src/main/java/io/cdap/plugin/http/source/common/http/HttpClient.java @@ -17,6 +17,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Strings; +import com.wealdtech.hawk.HawkClient; import io.cdap.plugin.http.source.common.BaseHttpSourceConfig; import org.apache.http.Header; import org.apache.http.HttpHost; @@ -48,6 +49,7 @@ public class HttpClient implements Closeable { private final BaseHttpSourceConfig config; private final StringEntity requestBody; private CloseableHttpClient httpClient; + private HawkClient hawkClient; public HttpClient(BaseHttpSourceConfig config) { this.config = config; @@ -65,22 +67,45 @@ public HttpClient(BaseHttpSourceConfig config) { * Executes HTTP request with parameters configured in plugin config and returns response. * Is called to load every page by pagination iterator. * - * @param uri URI of resource + * @param uriStr URI of resource * @return a response object * @throws IOException in case of a problem or the connection was aborted */ - public CloseableHttpResponse executeHTTP(String uri) throws IOException { + public CloseableHttpResponse executeHTTP(String uriStr) throws IOException { // lazy init. So we are able to initialize the class for different checks during validations etc. if (httpClient == null) { httpClient = createHttpClient(); } - - HttpEntityEnclosingRequestBase request = new HttpRequest(URI.create(uri), config.getHttpMethod()); + URI uri = URI.create(uriStr); + HttpEntityEnclosingRequestBase request = new HttpRequest(uri, config.getHttpMethod()); if (requestBody != null) { request.setEntity(requestBody); } + if (config.getHawkAuthEnabled()) { + if (hawkClient == null) { + hawkClient = HawkUtil.createHawkClient( + config.getHawkAuthID(), + config.getHawkAuthKey(), + config.getHawkAlgorithm() + ); + } + + String authorizationHeader = HawkUtil.getAuthorizationHeader( + hawkClient, + requestBody, + uri, + request.getMethod(), + config.getHawkPayloadHashEnabled(), + config.getHawkExt(), + config.getHawkApp(), + config.getHawkDlg() + + ); + request.addHeader("Authorization", authorizationHeader); + } + return httpClient.execute(request); } diff --git a/widgets/HTTP-batchsource.json b/widgets/HTTP-batchsource.json index 6c053781..7d77bc7e 100644 --- a/widgets/HTTP-batchsource.json +++ b/widgets/HTTP-batchsource.json @@ -153,6 +153,80 @@ } ] }, + { + "label": "HAWK Authentication", + "properties": [ + { + "widget-type": "toggle", + "label": "HAWK Authentication Enabled", + "name": "hawkAuthEnabled", + "widget-attributes": { + "default": "false", + "on": { + "label": "True", + "value": "true" + }, + "off": { + "label": "False", + "value": "false" + } + } + }, + { + "widget-type": "textbox", + "label": "Hawk Auth ID", + "name": "hawkAuthID" + }, + { + "widget-type": "password", + "label": "Hawk Auth Key", + "name": "hawkAuthKey" + }, + { + "widget-type": "select", + "label": "Algorithm", + "name": "hawkAlgorithm", + "widget-attributes": { + "default": "sha256", + "values": [ + "sha256", + "sha1" + ] + } + }, + { + "widget-type": "textbox", + "label": "ext", + "name": "hawkExt" + }, + { + "widget-type": "textbox", + "label": "app", + "name": "hawkApp" + }, + { + "widget-type": "textbox", + "label": "dlg", + "name": "hawkDlg" + }, + { + "widget-type": "toggle", + "label": "Include Payload Hash", + "name": "hawkPayloadHashEnabled", + "widget-attributes": { + "default": "false", + "on": { + "label": "True", + "value": "true" + }, + "off": { + "label": "False", + "value": "false" + } + } + } + ] + }, { "label": "Basic Authentication", "properties": [ @@ -503,7 +577,7 @@ "name": "Proxy authentication", "condition": { "property": "proxyUrl", - "operator": "exists", + "operator": "exists" }, "show": [ { @@ -599,13 +673,19 @@ ] }, { - "name": "OAuth 2 disabled", + "name": "OAuth 2 and HAWK disabled", "condition": { - "property": "oauth2Enabled", - "operator": "equal to", - "value": "false" + "expression": "oauth2Enabled == false && hawkAuthEnabled == false" }, "show": [ + { + "name": "oauth2Enabled", + "type": "property" + }, + { + "name": "hawkAuthEnabled", + "type": "property" + }, { "name": "username", "type": "property" @@ -650,6 +730,48 @@ } ] }, + { + "name": "HAWK enabled", + "condition": { + "property": "hawkAuthEnabled", + "operator": "equal to", + "value": "true" + }, + "show": [ + { + "name": "hawkAuthEnabled", + "type": "property" + }, + { + "name": "hawkAuthID", + "type": "property" + }, + { + "name": "hawkAuthKey", + "type": "property" + }, + { + "name": "hawkAlgorithm", + "type": "property" + }, + { + "name": "hawkExt", + "type": "property" + }, + { + "name": "hawkApp", + "type": "property" + }, + { + "name": "hawkDlg", + "type": "property" + }, + { + "name": "hawkPayloadHashEnabled", + "type": "property" + } + ] + }, { "name": "JSON/XML Formatting", "condition": { diff --git a/widgets/HTTP-streamingsource.json b/widgets/HTTP-streamingsource.json index e7abdb68..504277d0 100644 --- a/widgets/HTTP-streamingsource.json +++ b/widgets/HTTP-streamingsource.json @@ -158,6 +158,80 @@ } ] }, + { + "label": "HAWK Authentication", + "properties": [ + { + "widget-type": "toggle", + "label": "HAWK Authentication Enabled", + "name": "hawkAuthEnabled", + "widget-attributes": { + "default": "false", + "on": { + "label": "True", + "value": "true" + }, + "off": { + "label": "False", + "value": "false" + } + } + }, + { + "widget-type": "textbox", + "label": "Hawk Auth ID", + "name": "hawkAuthID" + }, + { + "widget-type": "password", + "label": "Hawk Auth Key", + "name": "hawkAuthKey" + }, + { + "widget-type": "select", + "label": "Algorithm", + "name": "hawkAlgorithm", + "widget-attributes": { + "default": "sha256", + "values": [ + "sha256", + "sha1" + ] + } + }, + { + "widget-type": "textbox", + "label": "ext", + "name": "hawkExt" + }, + { + "widget-type": "textbox", + "label": "app", + "name": "hawkApp" + }, + { + "widget-type": "textbox", + "label": "dlg", + "name": "hawkDlg" + }, + { + "widget-type": "toggle", + "label": "Include Payload Hash", + "name": "hawkPayloadHashEnabled", + "widget-attributes": { + "default": "false", + "on": { + "label": "True", + "value": "true" + }, + "off": { + "label": "False", + "value": "false" + } + } + } + ] + }, { "label": "Basic Authentication", "properties": [ @@ -503,7 +577,7 @@ "name": "Proxy authentication", "condition": { "property": "proxyUrl", - "operator": "exists", + "operator": "exists" }, "show": [ { @@ -597,15 +671,21 @@ "type": "property" } ] - }, - { - "name": "OAuth 2 disabled", + },{ + "name": "OAuth 2 and HAWK disabled", "condition": { - "property": "oauth2Enabled", - "operator": "equal to", - "value": "false" + "expression": "oauth2Enabled == false && hawkAuthEnabled == false" }, "show": [ + + { + "name": "oauth2Enabled", + "type": "property" + }, + { + "name": "hawkAuthEnabled", + "type": "property" + }, { "name": "username", "type": "property" @@ -650,6 +730,48 @@ } ] }, + { + "name": "HAWK enabled", + "condition": { + "property": "hawkAuthEnabled", + "operator": "equal to", + "value": "true" + }, + "show": [ + { + "name": "hawkAuthEnabled", + "type": "property" + }, + { + "name": "hawkAuthID", + "type": "property" + }, + { + "name": "hawkAuthKey", + "type": "property" + }, + { + "name": "hawkAlgorithm", + "type": "property" + }, + { + "name": "hawkExt", + "type": "property" + }, + { + "name": "hawkApp", + "type": "property" + }, + { + "name": "hawkDlg", + "type": "property" + }, + { + "name": "hawkPayloadHashEnabled", + "type": "property" + } + ] + }, { "name": "JSON/XML Formatting", "condition": { From a1e4cee5740213a8abfc87d85d8af742f5d982d4 Mon Sep 17 00:00:00 2001 From: Victor Duvert Date: Fri, 7 May 2021 15:50:13 +0200 Subject: [PATCH 2/3] fix unit tests --- src/test/java/io/cdap/plugin/http/etl/HttpSourceETLTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/io/cdap/plugin/http/etl/HttpSourceETLTest.java b/src/test/java/io/cdap/plugin/http/etl/HttpSourceETLTest.java index 07a594e3..00bcba7e 100644 --- a/src/test/java/io/cdap/plugin/http/etl/HttpSourceETLTest.java +++ b/src/test/java/io/cdap/plugin/http/etl/HttpSourceETLTest.java @@ -487,6 +487,7 @@ protected Map getProperties(Map sourceProperties .put("referenceName", testName.getMethodName()) .put(BaseHttpSourceConfig.PROPERTY_HTTP_METHOD, "GET") .put(BaseHttpSourceConfig.PROPERTY_OAUTH2_ENABLED, "false") + .put(BaseHttpSourceConfig.PROPERTY_HAWK_AUTH_ENABLED, "false") .put(BaseHttpSourceConfig.PROPERTY_HTTP_ERROR_HANDLING, "2..:Success,.*:Fail") .put(BaseHttpSourceConfig.PROPERTY_ERROR_HANDLING, "stopOnError") .put(BaseHttpSourceConfig.PROPERTY_RETRY_POLICY, "linear") From 3a76f92d2d2a1f68877e978c2b908fb2c9cef836 Mon Sep 17 00:00:00 2001 From: Victor Duvert Date: Fri, 7 May 2021 15:56:12 +0200 Subject: [PATCH 3/3] fix --- .../http/source/common/http/HawkUtil.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java b/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java index 3858e9f6..e0b015ed 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java +++ b/src/main/java/io/cdap/plugin/http/source/common/http/HawkUtil.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package io.cdap.plugin.http.source.common.http; import com.wealdtech.hawk.HawkClient; @@ -6,6 +21,9 @@ import java.net.URI; +/** + * A class which contains utilities to make HAWK specific calls. + */ public class HawkUtil { public static HawkClient createHawkClient(String authID, String authKey, HawkCredentials.Algorithm algorithm) { @@ -26,7 +44,7 @@ public static String getAuthorizationHeader( boolean payloadHashEnabled, String ext, String app, - String dlg){ + String dlg) { String hash = null; if (payloadHashEnabled) { hash = Integer.toString(requestBody.hashCode());