Stab at a minemeld adapter

src/main/java/org/graylog/plugins/threatintel/PluginConfigService.java

@@ -70,6 +70,9 @@ public void handleUpdatedClusterConfig(ClusterConfigChangedEvent clusterConfigCh
             if (previous.abusechRansomEnabled() != currentVersion.abusechRansomEnabled()) {
                 adaptersToLoad.add("abuse-ch-ransomware-domains", "abuse-ch-ransomware-ip");
             }
+            if (previous.minemeldEnabled() != currentVersion.minemeldEnabled()) {
+                adaptersToLoad.add("minemeld-domains", "minemeld-ip");
+            }
             if (previous.torEnabled() != currentVersion.torEnabled()) {
                 adaptersToLoad.add("tor-exit-node");
             }

src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginConfiguration.java

@@ -30,19 +30,24 @@ public abstract class ThreatIntelPluginConfiguration {
 
     @JsonProperty("abusech_ransom_enabled")
     public abstract boolean abusechRansomEnabled();
+
+    @JsonProperty("minemeld_enabled")
+    public abstract boolean minemeldEnabled();
 
     @JsonCreator
     public static ThreatIntelPluginConfiguration create(@JsonProperty("otx_enabled") boolean otxEnabled,
                                                         @JsonProperty("otx_api_key") @Nullable String otxApiKey,
                                                         @JsonProperty("tor_enabled") boolean torEnabled,
                                                         @JsonProperty("spamhaus_enabled") boolean spamhausEnabled,
-                                                        @JsonProperty("abusech_ransom_enabled") boolean abusechRansomEnabled) {
+                                                        @JsonProperty("abusech_ransom_enabled") boolean abusechRansomEnabled,
+                                                        @JsonProperty("minemeld_enabled") boolean minemeldEnabled ) {
         return builder()
                 .otxEnabled(otxEnabled)
                 .otxApiKey(otxApiKey)
                 .torEnabled(torEnabled)
                 .spamhausEnabled(spamhausEnabled)
                 .abusechRansomEnabled(abusechRansomEnabled)
+                .minemeldEnabled(minemeldEnabled)
                 .build();
     }
 
@@ -56,6 +61,7 @@ public static ThreatIntelPluginConfiguration defaults() {
                 .torEnabled(false)
                 .spamhausEnabled(false)
                 .abusechRansomEnabled(false)
+                .minemeldEnabled(false)
                 .build();
     }
 
@@ -72,6 +78,8 @@ public static abstract class Builder {
         public abstract Builder spamhausEnabled(boolean spamhausEnabled);
 
         public abstract Builder abusechRansomEnabled(boolean abusechRansomEnabled);
+
+        public abstract Builder minemeldEnabled(boolean minemeldEnabled);
 
         public abstract ThreatIntelPluginConfiguration build();
     }

src/main/java/org/graylog/plugins/threatintel/ThreatIntelPluginModule.java

@@ -6,6 +6,7 @@
 import com.google.inject.multibindings.MapBinder;
 import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
 import org.graylog.plugins.threatintel.adapters.abusech.AbuseChRansomAdapter;
+import org.graylog.plugins.threatintel.adapters.minemeld.MineMeldBlockListAdapter;
 import org.graylog.plugins.threatintel.adapters.otx.OTXDataAdapter;
 import org.graylog.plugins.threatintel.functions.DomainFunctions;
 import org.graylog.plugins.threatintel.functions.IPFunctions;
@@ -15,6 +16,8 @@
 import org.graylog.plugins.threatintel.functions.GenericLookupResult;
 import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomDomainLookupFunction;
 import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomIpLookupFunction;
+import org.graylog.plugins.threatintel.functions.minemeld.MineMeldDomainLookupFunction;
+import org.graylog.plugins.threatintel.functions.minemeld.MineMeldIpLookupFunction;
 import org.graylog.plugins.threatintel.functions.global.GlobalDomainLookupFunction;
 import org.graylog.plugins.threatintel.functions.global.GlobalIpLookupFunction;
 import org.graylog.plugins.threatintel.functions.otx.OTXDomainLookupFunction;
@@ -57,6 +60,11 @@ protected void configure() {
         // abuse.ch Ransomware
         addMessageProcessorFunction(AbuseChRansomDomainLookupFunction.NAME, AbuseChRansomDomainLookupFunction.class);
         addMessageProcessorFunction(AbuseChRansomIpLookupFunction.NAME, AbuseChRansomIpLookupFunction.class);
+
+
+        // MineMeld Threat Feeds
+        addMessageProcessorFunction(MineMeldDomainLookupFunction.NAME, MineMeldDomainLookupFunction.class);
+        addMessageProcessorFunction(MineMeldIpLookupFunction.NAME, MineMeldIpLookupFunction.class);
 
         // Global/combined lookup
         addMessageProcessorFunction(GlobalIpLookupFunction.NAME, GlobalIpLookupFunction.class);
@@ -69,6 +77,7 @@ protected void configure() {
         addMessageProcessorFunction(PrivateNetLookupFunction.NAME, PrivateNetLookupFunction.class);
 
         installLookupDataAdapter(AbuseChRansomAdapter.NAME, AbuseChRansomAdapter.class, AbuseChRansomAdapter.Factory.class, AbuseChRansomAdapter.Config.class);
+        installLookupDataAdapter(MineMeldBlockListAdapter.NAME, MineMeldBlockListAdapter.class, MineMeldBlockListAdapter.Factory.class, MineMeldBlockListAdapter.Config.class);
         installLookupDataAdapter(SpamhausEDROPDataAdapter.NAME, SpamhausEDROPDataAdapter.class, SpamhausEDROPDataAdapter.Factory.class, SpamhausEDROPDataAdapter.Config.class);
         installLookupDataAdapter(TorExitNodeDataAdapter.NAME, TorExitNodeDataAdapter.class, TorExitNodeDataAdapter.Factory.class, TorExitNodeDataAdapter.Config.class);
         installLookupDataAdapter(WhoisDataAdapter.NAME, WhoisDataAdapter.class, WhoisDataAdapter.Factory.class, WhoisDataAdapter.Config.class);
@@ -79,6 +88,8 @@ protected void configure() {
 
         addDomainFunction("abusech_ransomware", AbuseChRansomDomainLookupFunction.class);
         addIPFunction("abusech_ransomware", AbuseChRansomIpLookupFunction.class);
+        addDomainFunction("minemeld", MineMeldDomainLookupFunction.class);
+        addIPFunction("minemeld", MineMeldIpLookupFunction.class);
         addIPFunction("spamhaus", SpamhausIpLookupFunction.class);
         addIPFunction("tor", TorExitNodeLookupFunction.class);
     }

src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/BlocklistType.java

@@ -0,0 +1,47 @@
+package org.graylog.plugins.threatintel.adapters.minemeld;
+
+import com.google.common.base.MoreObjects;
+
+public enum BlocklistType {
+
+	//These URLs need to be changed to your minemeld instance output urls. 
+	//To find the url follow these steps:
+
+	// 1) Log into minemeld
+	// 2) Click on Nodes
+	// 3) Find the output you want to configure to utilize with this plugin.
+	// 4) Copy the feed base URL.
+
+    // Example https://hostname/feeds/OutPutURL
+
+    DOMAINS("https://FEEDBASEURL-FOR-DOMAINS", true),
+    URLS("https://FEEDBASEURL-FOR-URLS", true),
+	//keep the ?tr=1 on the IP list type in order to output into CIDR per documentation here: 
+	//https://live.paloaltonetworks.com/t5/MineMeld-Articles/Parameters-for-the-output-feeds/ta-p/146170
+    IPS("https://FEEDBASEURL-FOR-IPS?tr=1", false); 
+
+    private final String url;
+    private final boolean caseInsensitive;
+
+    BlocklistType(String url, boolean caseInsensitive) {
+        this.url = url;
+        this.caseInsensitive = caseInsensitive;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public boolean isCaseInsensitive() {
+        return caseInsensitive;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+                .add("url", url)
+                .add("caseInsensitive", caseInsensitive)
+                .toString();
+    }
+
+}

src/main/java/org/graylog/plugins/threatintel/adapters/minemeld/MineMeldBlockListAdapter.java

@@ -0,0 +1,207 @@
+package org.graylog.plugins.threatintel.adapters.minemeld;
+
+import com.codahale.metrics.MetricRegistry;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import org.graylog.autovalue.WithBeanGetter;
+import org.graylog.plugins.threatintel.PluginConfigService;
+import org.graylog.plugins.threatintel.tools.AdapterDisabledException;
+import org.graylog2.lookup.adapters.dsvhttp.DSVParser;
+import org.graylog2.lookup.adapters.dsvhttp.HTTPFileRetriever;
+import org.graylog2.plugin.lookup.LookupCachePurge;
+import org.graylog2.plugin.lookup.LookupDataAdapter;
+import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
+import org.graylog2.plugin.lookup.LookupResult;
+import org.joda.time.Duration;
+import org.joda.time.Period;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+import javax.validation.constraints.Min;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class MineMeldBlockListAdapter extends LookupDataAdapter {
+    private static final Logger LOG = LoggerFactory.getLogger(MineMeldBlockListAdapter.class);
+
+    // MineMeld updates miners at different 
+    private static final int REFRESH_INTERVAL = Period.minutes(5).toStandardSeconds().getSeconds() / 2;
+
+    public static final String NAME = "minemeldblocklist";
+    private static final LookupResult TRUE_RESULT = LookupResult.single(true);
+
+    private final HTTPFileRetriever httpFileRetriever;
+    private final PluginConfigService pluginConfigService;
+    private final AtomicReference<Set<String>> lookupRef = new AtomicReference<>(Collections.emptySet());
+    private final DSVParser dsvParser;
+    private final BlocklistType blocklistType;
+
+    @Inject
+    public MineMeldBlockListAdapter(@Assisted("id") String id,
+                                @Assisted("name") String name,
+                                @Assisted LookupDataAdapterConfiguration config,
+                                MetricRegistry metricRegistry,
+                                HTTPFileRetriever httpFileRetriever,
+                                PluginConfigService pluginConfigService) {
+        super(id, name, config, metricRegistry);
+        this.httpFileRetriever = httpFileRetriever;
+        this.pluginConfigService = pluginConfigService;
+        blocklistType = ((Config) getConfig()).blocklistType();
+        dsvParser = new DSVParser(
+                "#",
+                "\n",
+                ",",
+                "\"",
+                true,
+                blocklistType.isCaseInsensitive(),
+                0,
+                Optional.of(0)
+        );
+    }
+
+    @Override
+    public void doStart() throws Exception {
+        if (!pluginConfigService.config().getCurrent().minemeldEnabled()) {
+            throw new AdapterDisabledException("MineMeldAdapter service is disabled, not starting adapter. To enable it please go to System / Configurations.");
+        }
+        final Config config = ((Config) getConfig());
+        LOG.debug("Starting MineMeldAdapter data adapter for blocklist {}", config.blocklistType());
+        if (config.refreshInterval() < 1) {
+            throw new IllegalStateException("Check interval setting cannot be smaller than 1");
+        }
+
+        loadData();
+    }
+
+    @Override
+    protected void doStop() throws Exception {
+        // nothing to do
+    }
+
+    @Override
+    public Duration refreshInterval() {
+        if (!pluginConfigService.config().getCurrent().minemeldEnabled()) {
+            return Duration.ZERO;
+        }
+        return Duration.standardSeconds(((Config) getConfig()).refreshInterval());
+    }
+
+    @Override
+    protected void doRefresh(LookupCachePurge cachePurge) throws Exception {
+        if (!pluginConfigService.config().getCurrent().minemeldEnabled()) {
+            throw new AdapterDisabledException("MineMeldAdapter service is disabled, not refreshing adapter. To enable it please go to System / Configurations.");
+        }
+        loadData();
+        cachePurge.purgeAll();
+    }
+
+    private void loadData() throws IOException {
+        final Optional<String> response = httpFileRetriever.fetchFileIfNotModified(blocklistType.getUrl());
+
+        response.ifPresent(body -> {
+            final Map<String, String> map = dsvParser.parse(body);
+            lookupRef.set(map.keySet());
+        });
+    }
+
+    @Override
+    protected LookupResult doGet(Object key) {
+        return lookupRef.get().contains(key.toString())
+                ? TRUE_RESULT
+                : LookupResult.empty();
+    }
+
+    @Override
+    public void set(Object key, Object value) {
+        // not supported
+    }
+
+    public interface Factory extends LookupDataAdapter.Factory<MineMeldBlockListAdapter> {
+        @Override
+        MineMeldBlockListAdapter create(@Assisted("id") String id,
+                                    @Assisted("name") String name,
+                                    LookupDataAdapterConfiguration configuration);
+
+        @Override
+        Descriptor getDescriptor();
+    }
+
+    public static class Descriptor extends LookupDataAdapter.Descriptor<Config> {
+
+        public Descriptor() {
+            super(NAME, Config.class);
+        }
+
+        @Override
+        public Config defaultConfiguration() {
+            return Config.builder()
+                    .type(NAME)
+                    .refreshInterval(REFRESH_INTERVAL)
+                    .blocklistType(BlocklistType.DOMAINS)
+                    .build();
+        }
+    }
+
+    @AutoValue
+    @WithBeanGetter
+    @JsonAutoDetect
+    @JsonDeserialize(builder = AutoValue_MineMeldBlockListAdapter_Config.Builder.class)
+    @JsonTypeName(NAME)
+    public static abstract class Config implements LookupDataAdapterConfiguration {
+
+        public static Builder builder() {
+            return new AutoValue_MineMeldBlockListAdapter_Config.Builder();
+        }
+
+        @Override
+        @JsonProperty(TYPE_FIELD)
+        public abstract String type();
+
+        @JsonProperty("refresh_interval")
+        @Min(150) // see REFRESH_INTERVAL
+        public abstract long refreshInterval();
+
+        @Nullable
+        @JsonProperty("refresh_interval_unit")
+        public abstract TimeUnit refreshIntervalUnit();
+
+        @JsonProperty("blocklist_type")
+        public abstract BlocklistType blocklistType();
+
+        @Override
+        public Optional<Multimap<String, String>> validate() {
+            final ArrayListMultimap<String, String> errors = ArrayListMultimap.create();
+
+            return errors.isEmpty() ? Optional.empty() : Optional.of(errors);
+        }
+
+        @AutoValue.Builder
+        public abstract static class Builder {
+            @JsonProperty(TYPE_FIELD)
+            public abstract Builder type(String type);
+
+            @JsonProperty("refresh_interval")
+            public abstract Builder refreshInterval(long refreshInterval);
+
+            @JsonProperty("blocklist_type")
+            public abstract Builder blocklistType(BlocklistType blocklistType);
+
+            @JsonProperty("refresh_interval_unit")
+            public abstract Builder refreshIntervalUnit(@Nullable TimeUnit refreshIntervalUnit);
+
+            public abstract Config build();
+        }
+    }
+}

src/main/java/org/graylog/plugins/threatintel/functions/global/GlobalDomainLookupFunction.java

@@ -9,6 +9,7 @@
 import org.graylog.plugins.threatintel.functions.DomainFunctions;
 import org.graylog.plugins.threatintel.functions.GenericLookupResult;
 import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomDomainLookupFunction;
+import org.graylog.plugins.threatintel.functions.minemeld.MineMeldDomainLookupFunction;
 import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction;
 import org.graylog2.plugin.cluster.ClusterConfigService;
 import org.slf4j.Logger;
@@ -64,9 +65,21 @@ boolean isEnabled(LookupTableFunction<? extends GenericLookupResult> function) {
         if (function.getClass().equals(AbuseChRansomDomainLookupFunction.class)) {
             return configuration.abusechRansomEnabled();
         }
+        else if (function.getClass().equals(MineMeldDomainLookupFunction.class)) {
+            return configuration.minemeldEnabled();
+        }
         return true;
     }
-
+/*
+    @Override
+    boolean isEnabled(LookupTableFunction<? extends GenericLookupResult> function) {
+        final ThreatIntelPluginConfiguration configuration = this.threatIntelPluginConfiguration();
+        if (function.getClass().equals(MineMeldDomainLookupFunction.class)) {
+            return configuration.minemeldEnabled();
+        }
+        return true;
+    }
+  */  
     @Override
     public FunctionDescriptor<GlobalLookupResult> descriptor() {
         return FunctionDescriptor.<GlobalLookupResult>builder()