From 5564e12dcaa46949f868c5259d240678511d5ca6 Mon Sep 17 00:00:00 2001 From: ChrisForsythe Date: Mon, 7 May 2018 13:34:15 -0500 Subject: [PATCH 1/5] Stab at a minemeld adapter - Copied the abusech adapter and string replaced for the most part. - Commented instructions on how to get the url within minemeld. - This is not tested. I do not have a graylog instance to test with at this time. --- .../adapters/minemeld/BlockListMineMeld.java | 208 ++++++++++++++++++ .../adapters/minemeld/BlocklistType.java | 45 ++++ 2 files changed, 253 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlockListMineMeld.java create mode 100644 src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlocklistType.java diff --git a/src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlockListMineMeld.java b/src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlockListMineMeld.java new file mode 100644 index 0000000..e01c64b --- /dev/null +++ b/src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlockListMineMeld.java @@ -0,0 +1,208 @@ +package org.graylog.plugins.threatintel.adapters.BlockListMineMeld; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import org.graylog.autovalue.WithBeanGetter; +import org.graylog.plugins.threatintel.PluginConfigService; +import org.graylog.plugins.threatintel.tools.AdapterDisabledException; +import org.graylog2.lookup.adapters.dsvhttp.DSVParser; +import org.graylog2.lookup.adapters.dsvhttp.HTTPFileRetriever; +import org.graylog2.plugin.lookup.LookupCachePurge; +import org.graylog2.plugin.lookup.LookupDataAdapter; +import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration; +import org.graylog2.plugin.lookup.LookupResult; +import org.joda.time.Duration; +import org.joda.time.Period; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import javax.validation.constraints.Min; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +public class MineMeldBlockListAdapter extends LookupDataAdapter { + private static final Logger LOG = LoggerFactory.getLogger(MineMeldBlockListAdapter.class); + + // MineMeld updates miners at different + private static final int REFRESH_INTERVAL = Period.minutes(5).toStandardSeconds().getSeconds() / 2; + + public static final String NAME = "minemeldblocklist"; + private static final LookupResult TRUE_RESULT = LookupResult.single(true); + + private final HTTPFileRetriever httpFileRetriever; + private final PluginConfigService pluginConfigService; + private final AtomicReference> lookupRef = new AtomicReference<>(Collections.emptySet()); + private final DSVParser dsvParser; + private final BlocklistType blocklistType; + + @Inject + public MineMeldBlockListAdapter(@Assisted("id") String id, + @Assisted("name") String name, + @Assisted LookupDataAdapterConfiguration config, + MetricRegistry metricRegistry, + HTTPFileRetriever httpFileRetriever, + PluginConfigService pluginConfigService) { + super(id, name, config, metricRegistry); + this.httpFileRetriever = httpFileRetriever; + this.pluginConfigService = pluginConfigService; + blocklistType = ((Config) getConfig()).blocklistType(); + dsvParser = new DSVParser( + "#", + "\n", + ",", + "\"", + true, + blocklistType.isCaseInsensitive(), + 0, + Optional.of(0) + ); + } + + @Override + public void doStart() throws Exception { + if (!pluginConfigService.config().getCurrent().mineMeldBlockListEnabled()) { + throw new AdapterDisabledException("MineMeldAdapter service is disabled, not starting adapter. To enable it please go to System / Configurations."); + } + final Config config = ((Config) getConfig()); + LOG.debug("Starting MineMeldAdapter data adapter for blocklist {}", config.blocklistType()); + if (config.refreshInterval() < 1) { + throw new IllegalStateException("Check interval setting cannot be smaller than 1"); + } + + loadData(); + } + + @Override + protected void doStop() throws Exception { + // nothing to do + } + + @Override + public Duration refreshInterval() { + if (!pluginConfigService.config().getCurrent().mineMeldBlockListEnabled()) { + return Duration.ZERO; + } + return Duration.standardSeconds(((Config) getConfig()).refreshInterval()); + } + + @Override + protected void doRefresh(LookupCachePurge cachePurge) throws Exception { + if (!pluginConfigService.config().getCurrent().mineMeldBlockListEnabled()) { + throw new AdapterDisabledException("MineMeldAdapter service is disabled, not refreshing adapter. To enable it please go to System / Configurations."); + } + loadData(); + cachePurge.purgeAll(); + } + + private void loadData() throws IOException { + final Optional response = httpFileRetriever.fetchFileIfNotModified(blocklistType.getUrl()); + + response.ifPresent(body -> { + final Map map = dsvParser.parse(body); + lookupRef.set(map.keySet()); + }); + } + + @Override + protected LookupResult doGet(Object key) { + return lookupRef.get().contains(key.toString()) + ? TRUE_RESULT + : LookupResult.empty(); + } + + @Override + public void set(Object key, Object value) { + // not supported + } + + public interface Factory extends LookupDataAdapter.Factory { + @Override + MineMeldBlockListAdapter create(@Assisted("id") String id, + @Assisted("name") String name, + LookupDataAdapterConfiguration configuration); + + @Override + Descriptor getDescriptor(); + } + + public static class Descriptor extends LookupDataAdapter.Descriptor { + + public Descriptor() { + super(NAME, Config.class); + } + + @Override + public Config defaultConfiguration() { + return Config.builder() + .type(NAME) + .refreshInterval(REFRESH_INTERVAL) + .blocklistType(BlocklistType.DOMAINS) + .build(); + } + } + + @AutoValue + @WithBeanGetter + @JsonAutoDetect + @JsonDeserialize(builder = AutoValue_MineMeldBlockListAdapter_Config.Builder.class) + @JsonTypeName(NAME) + public static abstract class Config implements LookupDataAdapterConfiguration { + + public static Builder builder() { + return new AutoValue_MineMeldBlockListAdapter_Config.Builder(); + } + + @Override + @JsonProperty(TYPE_FIELD) + public abstract String type(); + + @JsonProperty("refresh_interval") + @Min(150) // see REFRESH_INTERVAL + public abstract long refreshInterval(); + + @Nullable + @JsonProperty("refresh_interval_unit") + public abstract TimeUnit refreshIntervalUnit(); + + @JsonProperty("blocklist_type") + public abstract BlocklistType blocklistType(); + + @Override + public Optional> validate() { + final ArrayListMultimap errors = ArrayListMultimap.create(); + + return errors.isEmpty() ? Optional.empty() : Optional.of(errors); + } + + @AutoValue.Builder + public abstract static class Builder { + @JsonProperty(TYPE_FIELD) + public abstract Builder type(String type); + + @JsonProperty("refresh_interval") + public abstract Builder refreshInterval(long refreshInterval); + + @JsonProperty("blocklist_type") + public abstract Builder blocklistType(BlocklistType blocklistType); + + @JsonProperty("refresh_interval_unit") + public abstract Builder refreshIntervalUnit(@Nullable TimeUnit refreshIntervalUnit); + + public abstract Config build(); + } + } +} diff --git a/src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlocklistType.java b/src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlocklistType.java new file mode 100644 index 0000000..a5e9c17 --- /dev/null +++ b/src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlocklistType.java @@ -0,0 +1,45 @@ +package org.graylog.plugins.threatintel.adapters.BlockListMineMeld; + +import com.google.common.base.MoreObjects; + +public enum BlocklistType { + + //These URLs need to be changed to your minemeld instance output urls. + //To find the url follow these steps: + + // 1) Log into minemeld + // 2) Click on Nodes + // 3) Find the output you want to configure to utilize with this plugin. + // 4) Copy the feed base URL. + + DOMAINS("https://FEEDBASEURL-FOR-DOMAINS", true), + URLS("https://FEEDBASEURL-FOR-URLS", true), + //keep the ?tr=1 on the IP list type in order to output into CIDR per documentation here: + //https://live.paloaltonetworks.com/t5/MineMeld-Articles/Parameters-for-the-output-feeds/ta-p/146170 + IPS("https://FEEDBASEURL-FOR-IPS?tr=1", false); + + private final String url; + private final boolean caseInsensitive; + + BlocklistType(String url, boolean caseInsensitive) { + this.url = url; + this.caseInsensitive = caseInsensitive; + } + + public String getUrl() { + return url; + } + + public boolean isCaseInsensitive() { + return caseInsensitive; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("url", url) + .add("caseInsensitive", caseInsensitive) + .toString(); + } + +} From 8019b62dc95a86baef6a610c36de29683c1dd55d Mon Sep 17 00:00:00 2001 From: Chris Forsythe Date: Wed, 9 May 2018 11:29:38 -0500 Subject: [PATCH 2/5] More minemeld changes Second take on minemeld integration. Found a lot I missed from the previous push. - Added lookup functions for domain and ip list. - Added documentation. - Modified the ThreatIntelPluginConfig for the spaumhaus plugin to rename from tor_enabled to spaumhaus_enabled. - Modified the content pack to include minemeld. --- .../threatintel/PluginConfigService.java | 3 + .../ThreatIntelPluginConfiguration.java | 10 ++- .../threatintel/ThreatIntelPluginModule.java | 11 +++ .../global/GlobalDomainLookupFunction.java | 10 +++ .../global/GlobalIpLookupFunction.java | 4 ++ .../MineMeldDomainLookupFunction.java | 69 +++++++++++++++++++ .../minemeld/MineMeldIpLookupFunction.java | 66 ++++++++++++++++++ ...eThreatIntelLookupTables-content_pack.json | 47 ++++++++++++- .../components/ThreatIntelPluginConfig.jsx | 17 ++++- .../minemeld/MineMeldAdapterDocumentation.jsx | 20 ++++++ .../minemeld/MineMeldAdapterFieldSet.jsx | 69 +++++++++++++++++++ .../minemeld/MineMeldAdapterSummary.jsx | 26 +++++++ .../components/adapters/minemeld/index.jsx | 3 + src/web/index.jsx | 8 +++ 14 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldDomainLookupFunction.java create mode 100644 src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldIpLookupFunction.java create mode 100644 src/web/components/adapters/minemeld/MineMeldAdapterDocumentation.jsx create mode 100644 src/web/components/adapters/minemeld/MineMeldAdapterFieldSet.jsx create mode 100644 src/web/components/adapters/minemeld/MineMeldAdapterSummary.jsx create mode 100644 src/web/components/adapters/minemeld/index.jsx diff --git a/src/main/java/org/graylog/plugins/threatintel/PluginConfigService.java b/src/main/java/org/graylog/plugins/threatintel/PluginConfigService.java index 95c7a16..5da4b84 100644 --- a/src/main/java/org/graylog/plugins/threatintel/PluginConfigService.java +++ b/src/main/java/org/graylog/plugins/threatintel/PluginConfigService.java @@ -70,6 +70,9 @@ public void handleUpdatedClusterConfig(ClusterConfigChangedEvent clusterConfigCh if (previous.abusechRansomEnabled() != currentVersion.abusechRansomEnabled()) { adaptersToLoad.add("abuse-ch-ransomware-domains", "abuse-ch-ransomware-ip"); } + if (previous.minemeldEnabled() != currentVersion.minemeldEnabled()) { + adaptersToLoad.add("minemeld-domains", "minemeld-ip"); + } if (previous.torEnabled() != currentVersion.torEnabled()) { adaptersToLoad.add("tor-exit-node"); } diff --git a/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginConfiguration.java b/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginConfiguration.java index 4395770..e0e33fa 100644 --- a/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginConfiguration.java +++ b/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginConfiguration.java @@ -30,19 +30,24 @@ public abstract class ThreatIntelPluginConfiguration { @JsonProperty("abusech_ransom_enabled") public abstract boolean abusechRansomEnabled(); + + @JsonProperty("minemeld_enabled") + public abstract boolean minemeldEnabled(); @JsonCreator public static ThreatIntelPluginConfiguration create(@JsonProperty("otx_enabled") boolean otxEnabled, @JsonProperty("otx_api_key") @Nullable String otxApiKey, @JsonProperty("tor_enabled") boolean torEnabled, @JsonProperty("spamhaus_enabled") boolean spamhausEnabled, - @JsonProperty("abusech_ransom_enabled") boolean abusechRansomEnabled) { + @JsonProperty("abusech_ransom_enabled") boolean abusechRansomEnabled, + @JsonProperty("minemeld_enabled") boolean ) { return builder() .otxEnabled(otxEnabled) .otxApiKey(otxApiKey) .torEnabled(torEnabled) .spamhausEnabled(spamhausEnabled) .abusechRansomEnabled(abusechRansomEnabled) + .minemeldEnabled(minemeldEnabled) .build(); } @@ -56,6 +61,7 @@ public static ThreatIntelPluginConfiguration defaults() { .torEnabled(false) .spamhausEnabled(false) .abusechRansomEnabled(false) + .minemeldEnabled(false) .build(); } @@ -72,6 +78,8 @@ public static abstract class Builder { public abstract Builder spamhausEnabled(boolean spamhausEnabled); public abstract Builder abusechRansomEnabled(boolean abusechRansomEnabled); + + public abstract Builder minemeldEnabled(boolean minemeldEnabled); public abstract ThreatIntelPluginConfiguration build(); } diff --git a/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginModule.java b/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginModule.java index 73978b5..9697005 100644 --- a/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginModule.java +++ b/src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginModule.java @@ -6,6 +6,7 @@ import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.threatintel.adapters.abusech.AbuseChRansomAdapter; +import org.graylog.plugins.threatintel.adapters.minemeld.MineMeldBlockListAdapter; import org.graylog.plugins.threatintel.adapters.otx.OTXDataAdapter; import org.graylog.plugins.threatintel.functions.DomainFunctions; import org.graylog.plugins.threatintel.functions.IPFunctions; @@ -15,6 +16,8 @@ import org.graylog.plugins.threatintel.functions.GenericLookupResult; import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomDomainLookupFunction; import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomIpLookupFunction; +import org.graylog.plugins.threatintel.functions.minemeld.MineMeldDomainLookupFunction; +import org.graylog.plugins.threatintel.functions.minemeld.MineMeldIpLookupFunction; import org.graylog.plugins.threatintel.functions.global.GlobalDomainLookupFunction; import org.graylog.plugins.threatintel.functions.global.GlobalIpLookupFunction; import org.graylog.plugins.threatintel.functions.otx.OTXDomainLookupFunction; @@ -57,6 +60,11 @@ protected void configure() { // abuse.ch Ransomware addMessageProcessorFunction(AbuseChRansomDomainLookupFunction.NAME, AbuseChRansomDomainLookupFunction.class); addMessageProcessorFunction(AbuseChRansomIpLookupFunction.NAME, AbuseChRansomIpLookupFunction.class); + + + // MineMeld Threat Feeds + addMessageProcessorFunction(MineMeldDomainLookupFunction.NAME, MineMeldDomainLookupFunction.class); + addMessageProcessorFunction(MineMeldIpLookupFunction.NAME, MineMeldIpLookupFunction.class); // Global/combined lookup addMessageProcessorFunction(GlobalIpLookupFunction.NAME, GlobalIpLookupFunction.class); @@ -69,6 +77,7 @@ protected void configure() { addMessageProcessorFunction(PrivateNetLookupFunction.NAME, PrivateNetLookupFunction.class); installLookupDataAdapter(AbuseChRansomAdapter.NAME, AbuseChRansomAdapter.class, AbuseChRansomAdapter.Factory.class, AbuseChRansomAdapter.Config.class); + installLookupDataAdapter(MineMeldBlockListAdapter.NAME, MineMeldBlockListAdapter.class, MineMeldAdapter.Factory.class, MineMeldBlockListAdapter.Config.class); installLookupDataAdapter(SpamhausEDROPDataAdapter.NAME, SpamhausEDROPDataAdapter.class, SpamhausEDROPDataAdapter.Factory.class, SpamhausEDROPDataAdapter.Config.class); installLookupDataAdapter(TorExitNodeDataAdapter.NAME, TorExitNodeDataAdapter.class, TorExitNodeDataAdapter.Factory.class, TorExitNodeDataAdapter.Config.class); installLookupDataAdapter(WhoisDataAdapter.NAME, WhoisDataAdapter.class, WhoisDataAdapter.Factory.class, WhoisDataAdapter.Config.class); @@ -79,6 +88,8 @@ protected void configure() { addDomainFunction("abusech_ransomware", AbuseChRansomDomainLookupFunction.class); addIPFunction("abusech_ransomware", AbuseChRansomIpLookupFunction.class); + addDomainFunction("minemeld", MineMeldDomainLookupFunction.class); + addIPFunction("minemeld", MineMeldIpLookupFunction.class); addIPFunction("spamhaus", SpamhausIpLookupFunction.class); addIPFunction("tor", TorExitNodeLookupFunction.class); } diff --git a/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalDomainLookupFunction.java b/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalDomainLookupFunction.java index 014e595..ea9b736 100644 --- a/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalDomainLookupFunction.java +++ b/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalDomainLookupFunction.java @@ -9,6 +9,7 @@ import org.graylog.plugins.threatintel.functions.DomainFunctions; import org.graylog.plugins.threatintel.functions.GenericLookupResult; import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomDomainLookupFunction; +import org.graylog.plugins.threatintel.functions.minemeld.MineMeldDomainLookupFunction; import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction; import org.graylog2.plugin.cluster.ClusterConfigService; import org.slf4j.Logger; @@ -67,6 +68,15 @@ boolean isEnabled(LookupTableFunction function) { return true; } + @Override + boolean isEnabled(LookupTableFunction function) { + final ThreatIntelPluginConfiguration configuration = this.threatIntelPluginConfiguration(); + if (function.getClass().equals(MineMeldDomainLookupFunction.class)) { + return configuration.minemeldEnabled(); + } + return true; + } + @Override public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() diff --git a/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalIpLookupFunction.java b/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalIpLookupFunction.java index 1132b1b..7a70778 100644 --- a/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalIpLookupFunction.java +++ b/src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalIpLookupFunction.java @@ -9,6 +9,7 @@ import org.graylog.plugins.threatintel.functions.GenericLookupResult; import org.graylog.plugins.threatintel.functions.IPFunctions; import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomIpLookupFunction; +import org.graylog.plugins.threatintel.functions.minemeld.MineMeldIpLookupFunction; import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction; import org.graylog.plugins.threatintel.functions.otx.OTXIPLookupFunction; import org.graylog.plugins.threatintel.functions.spamhaus.SpamhausIpLookupFunction; @@ -73,6 +74,9 @@ boolean isEnabled(LookupTableFunction function) { if (function.getClass().equals(AbuseChRansomIpLookupFunction.class)) { return configuration.abusechRansomEnabled(); } + if (function.getClass().equals(MineMeldIpLookupFunction.class)) { + return configuration.minemeldEnabled(); + } if (function.getClass().equals(OTXIPLookupFunction.class)) { return configuration.otxEnabled(); } diff --git a/src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldDomainLookupFunction.java b/src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldDomainLookupFunction.java new file mode 100644 index 0000000..65e9df6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldDomainLookupFunction.java @@ -0,0 +1,69 @@ +package org.graylog.plugins.threatintel.functions.minemeld; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction; +import org.graylog.plugins.threatintel.functions.GenericLookupResult; +import org.graylog.plugins.threatintel.tools.Domain; +import org.graylog2.lookup.LookupTableService; +import org.graylog2.plugin.lookup.LookupResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +public class MineMeldDomainLookupFunction extends LookupTableFunction { + + private static final Logger LOG = LoggerFactory.getLogger(MineMeldDomainLookupFunction.class); + + public static final String NAME = "minemeld_lookup_domain"; + private static final String VALUE = "domain_name"; + private static final String LOOKUP_TABLE_NAME = "minemeld-domains"; + + private final ParameterDescriptor valueParam = ParameterDescriptor.string(VALUE).description("The domain to look up. Example: foo.example.org (A trailing dot ('.') will be ignored.)").build(); + + private final LookupTableService.Function lookupFunction; + + @Inject + public MineMeldDomainLookupFunction(final LookupTableService lookupTableService) { + this.lookupFunction = lookupTableService.newBuilder().lookupTable(LOOKUP_TABLE_NAME).build(); + } + + @Override + public GenericLookupResult evaluate(FunctionArgs args, EvaluationContext context) { + String domain = valueParam.required(args, context); + if (domain == null) { + LOG.error("NULL parameter passed to abuse.ch Ransomware domain lookup."); + return null; + } + + domain = Domain.prepareDomain(domain); + + LOG.debug("Running abuse.ch Ransomware lookup for domain [{}].", domain); + + final LookupResult lookupResult = this.lookupFunction.lookup(domain.trim()); + if (lookupResult != null && !lookupResult.isEmpty() && lookupResult.singleValue() != null) { + if (lookupResult.singleValue() instanceof Boolean) { + return (Boolean)lookupResult.singleValue() ? GenericLookupResult.TRUE : GenericLookupResult.FALSE; + } + if (lookupResult.singleValue() instanceof String) { + return Boolean.valueOf((String) lookupResult.singleValue()) ? GenericLookupResult.TRUE : GenericLookupResult.FALSE; + } + } + + return GenericLookupResult.FALSE; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .description("Match a domain name against the abuse.ch Ransomware Domain Blocklist. (RW_DOMBL)") + .params(valueParam) + .returnType(GenericLookupResult.class) + .build(); + } + +} diff --git a/src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldIpLookupFunction.java b/src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldIpLookupFunction.java new file mode 100644 index 0000000..f4cb5d6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/threatintel/functions/minemeld/MineMeldIpLookupFunction.java @@ -0,0 +1,66 @@ +package org.graylog.plugins.threatintel.functions.minemeld; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction; +import org.graylog.plugins.threatintel.functions.GenericLookupResult; +import org.graylog2.lookup.LookupTableService; +import org.graylog2.plugin.lookup.LookupResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +public class MineMeldIpLookupFunction extends LookupTableFunction { + + private static final Logger LOG = LoggerFactory.getLogger(MineMeldIpLookupFunction.class); + + public static final String NAME = "minemeld_lookup_ip"; + private static final String VALUE = "ip_address"; + private static final String LOOKUP_TABLE_NAME = "minemeld-ip"; + + private final ParameterDescriptor valueParam = ParameterDescriptor.string(VALUE).description("The IPv4 or IPv6 address to look up. Example: 198.51.100.1 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334").build(); + + private final LookupTableService.Function lookupFunction; + + @Inject + public MineMeldmIpLookupFunction(final LookupTableService lookupTableService) { + this.lookupFunction = lookupTableService.newBuilder().lookupTable(LOOKUP_TABLE_NAME).build(); + } + + @Override + public GenericLookupResult evaluate(FunctionArgs args, EvaluationContext context) { + String ip = valueParam.required(args, context); + if (ip == null) { + LOG.error("NULL parameter passed to abuse.ch Ransomware IP lookup."); + return null; + } + + LOG.debug("Running abuse.ch Ransomware lookup for IP [{}].", ip); + + final LookupResult lookupResult = this.lookupFunction.lookup(ip.trim()); + if (lookupResult != null && !lookupResult.isEmpty() && lookupResult.singleValue() != null) { + if (lookupResult.singleValue() instanceof Boolean) { + return (Boolean)lookupResult.singleValue() ? GenericLookupResult.TRUE : GenericLookupResult.FALSE; + } + if (lookupResult.singleValue() instanceof String) { + return Boolean.valueOf((String) lookupResult.singleValue()) ? GenericLookupResult.TRUE : GenericLookupResult.FALSE; + } + } + + return GenericLookupResult.FALSE; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .description("Match a IPv4 or IPv6 address against the abuse.ch Ransomware IP Blocklist. (RW_IPBL)") + .params(valueParam) + .returnType(GenericLookupResult.class) + .build(); + } + +} diff --git a/src/main/resources/org/graylog/plugins/threatintel/migrations/V20170815111700_CreateThreatIntelLookupTables-content_pack.json b/src/main/resources/org/graylog/plugins/threatintel/migrations/V20170815111700_CreateThreatIntelLookupTables-content_pack.json index 32fb240..0b5a222 100644 --- a/src/main/resources/org/graylog/plugins/threatintel/migrations/V20170815111700_CreateThreatIntelLookupTables-content_pack.json +++ b/src/main/resources/org/graylog/plugins/threatintel/migrations/V20170815111700_CreateThreatIntelLookupTables-content_pack.json @@ -41,6 +41,17 @@ "default_multi_value": "", "default_multi_value_type": "NULL" }, + { + "title": "MineMeld Domains", + "description": "This is the lookup table for the MineMeld Domain list, listing infrastructure by domain names which are used for ransomware. For more information see https://ransomwaretracker.abuse.ch. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", + "name": "minemeld-domains", + "cache_name": "threat-intel-uncached-adapters", + "data_adapter_name": "minemeld-domains", + "default_single_value": "", + "default_single_value_type": "NULL", + "default_multi_value": "", + "default_multi_value_type": "NULL" + }, { "title": "Whois", "description": "This is the lookup table for the WHOIS database, listing registered users of Internet resources like IPs, Netblocks or Domain Names. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", @@ -63,6 +74,17 @@ "default_multi_value": "", "default_multi_value_type": "NULL" }, + { + "title": "MineMeld IP List", + "description": "This is the lookup table for the MineMeld IP List, listing infrastructure by IP which is used for nefarious reasons based on your threat feed configuration within MineMeld. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", + "name": "minemeld-ip", + "cache_name": "threat-intel-uncached-adapters", + "data_adapter_name": "minemeld-ip", + "default_single_value": "", + "default_single_value_type": "NULL", + "default_multi_value": "", + "default_multi_value_type": "NULL" + }, { "title": "Spamhaus DROP", "description": "This is the lookup table for Spamhaus' DROP (Don't Route Or Peer) list, containing netblocks which are \"hijacked\" or leased by professional spam or cyber-crime operations. For more information see https://www.spamhaus.org/drop. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", @@ -189,6 +211,17 @@ "registry": "ARIN" } }, + { + "title": "MineMeld Domains", + "description": "This is the lookup table for the MineMeld Domain list, listing infrastructure by domain names which are used for ransomware. For more information see https://ransomwaretracker.abuse.ch. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", + "name": "minemeld-domains", + "config": { + "type": "minemeld", + "blocklist_type": "DOMAINS", + "refresh_interval": 150, + "refresh_interval_unit": "SECONDS" + } + }, { "title": "abuse.ch ransomware Domains", "description": "This is the data adapter for the abuse.ch ransomware Domain Tracker, listing infrastructure by domain names which are used for ransomware. For more information see https://ransomwaretracker.abuse.ch. This adapter is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", @@ -217,7 +250,19 @@ "blocklist_type": "IPS", "refresh_interval": 150, "refresh_interval_unit": "SECONDS" - } + }, + { + "title": "MineMeld IP List", + "description": "This is the data adapter for the MineMeld IP List, listing infrastructure by IP which is used for nefarious reasons based on your threat feed configuration within MineMeld. This adapter is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.", + "name": "minemeld-ip", + "config": { + "type": "minemeld", + "blocklist_type": "IPS", + "refresh_interval": 150, + "refresh_interval_unit": "SECONDS" + }, + + } ] } diff --git a/src/web/components/ThreatIntelPluginConfig.jsx b/src/web/components/ThreatIntelPluginConfig.jsx index d6837e1..257bc68 100644 --- a/src/web/components/ThreatIntelPluginConfig.jsx +++ b/src/web/components/ThreatIntelPluginConfig.jsx @@ -21,6 +21,7 @@ const ThreatIntelPluginConfig = createReactClass({ tor_enabled: false, spamhaus_enabled: false, abusech_ransom_enabled: false, + minemeld_enabled: false, }, }; }, @@ -96,6 +97,10 @@ const ThreatIntelPluginConfig = createReactClass({
Abuse.ch Ransomware:
{this.state.config.abusech_ransom_enabled === true ? 'Enabled' : 'Disabled'}
+ +
Minemeld Blocklist:
+
{this.state.config.minemeld_enabled === true ? 'Enabled' : 'Disabled'}
+ @@ -122,7 +127,7 @@ const ThreatIntelPluginConfig = createReactClass({ ref="spamhausEnabled" label="Allow Spamhaus DROP/EDROP lookups?" help="Enable to include Spamhaus lookup in global pipeline function, disabling also stops refreshing the data." - name="tor_enabled" + name="spamhaus_enabled" checked={this.state.config.spamhaus_enabled} onChange={this._onCheckboxClick('spamhaus_enabled', 'spamhausEnabled')}/> @@ -134,6 +139,16 @@ const ThreatIntelPluginConfig = createReactClass({ name="tor_enabled" checked={this.state.config.abusech_ransom_enabled} onChange={this._onCheckboxClick('abusech_ransom_enabled', 'abusechRansomEnabled')}/> + + + diff --git a/src/web/components/adapters/minemeld/MineMeldAdapterDocumentation.jsx b/src/web/components/adapters/minemeld/MineMeldAdapterDocumentation.jsx new file mode 100644 index 0000000..73d7632 --- /dev/null +++ b/src/web/components/adapters/minemeld/MineMeldAdapterDocumentation.jsx @@ -0,0 +1,20 @@ +/* eslint-disable react/no-unescaped-entities */ +import React from 'react'; +import { Alert } from 'react-bootstrap'; + +class MineMeldAdapterDocumentation extends React.Component { + render() { + return (
+

MineMeld is an application provided by Palo Alto. It is open source and free to use. It aggregates threat feeds from multiple disparate sources a set (or sets) of outputs to be utilized by multiple different utilities within your organization. Documentation starts here.

+ + +

Limitations

+

Currently to get this to work you will need to modify the java source file at src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlocklistType.java to point to your minemeld instance.

+

For the IP block list do not remove the ?tr=1 at the end of the url. This tells minemeld how to format he output.

+ + +
); + } +} + +export default MineMeldAdapterDocumentation; diff --git a/src/web/components/adapters/minemeld/MineMeldAdapterFieldSet.jsx b/src/web/components/adapters/minemeld/MineMeldAdapterFieldSet.jsx new file mode 100644 index 0000000..cd62008 --- /dev/null +++ b/src/web/components/adapters/minemeld/MineMeldAdapterFieldSet.jsx @@ -0,0 +1,69 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import ObjectUtils from 'util/ObjectUtils'; + +import { Input } from 'components/bootstrap'; +import { Select, TimeUnitInput } from 'components/common'; + +class MineMeldAdapterFieldSet extends React.Component { + static propTypes = { + config: PropTypes.object.isRequired, +// eslint-disable-next-line react/no-unused-prop-types + updateConfig: PropTypes.func.isRequired, + handleFormEvent: PropTypes.func.isRequired, + validationState: PropTypes.func.isRequired, + validationMessage: PropTypes.func.isRequired, + }; + + _update = (value, unit, enabled, name) => { + const config = ObjectUtils.clone(this.props.config); + config[name] = enabled ? value : 0; + config[`${name}_unit`] = unit; + this.props.updateConfig(config); + }; + + updateRefreshInterval = (value, unit, enabled) => { + this._update(value, unit, enabled, 'refresh_interval'); + }; + + _onBlocklistTypeSelect = (id) => { + const config = ObjectUtils.clone(this.props.config); + config.blocklist_type = id; + this.props.updateConfig(config); + }; + + render() { + const config = this.props.config; + const blocklistTypes = [ + { label: 'Domain blocklist', value: 'DOMAINS' }, + { label: 'URL blocklist', value: 'URLS' }, + { label: 'IP blocklist', value: 'IPS' }, + ]; + return (
+ +