track popularity of authors and resources

SpiGetOrg · Jan 6, 2017 · dc52bee · dc52bee
1 parent 1193142
commit dc52bee
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 4 deletions.
diff --git a/src/main/java/org/spiget/logparser/LogLine.java b/src/main/java/org/spiget/logparser/LogLine.java
@@ -15,6 +15,7 @@ public class LogLine {
 	private Date   time;
 	private String method;
 	private String url;
+	private String rawPath;
 	private String path;
 	private String parameters;
 	private String apiVersion;
@@ -44,6 +45,7 @@ public LogLine update(String address, String time, String method, String url, St
 			this.path = url;
 			this.parameters = "";
 		}
+		this.rawPath = this.path;
 
 		String[] versionSplit = url.split("/");
 		apiVersion = versionSplit[1];

diff --git a/src/main/java/org/spiget/logparser/Main.java b/src/main/java/org/spiget/logparser/Main.java
@@ -11,6 +11,7 @@ public static void main(String[] args) throws IOException, ParseException {
 		parser.downloadLogs();
 		parser.parse();
 
+		parser.cleanUpData();
 		parser.saveToDatabase();
 
 		parser.cleanup();

diff --git a/src/main/java/org/spiget/logparser/SpigetLogParser.java b/src/main/java/org/spiget/logparser/SpigetLogParser.java
@@ -24,6 +24,8 @@
 @Log4j2
 public class SpigetLogParser {
 
+	static final Pattern POPULARITY_PATTERN = Pattern.compile("/(authors|resources)/([0-9]+)(/.*)?");
+
 	public JsonObject     config;
 	public DatabaseClient databaseClient;
 	public Pattern        logPattern;
@@ -187,6 +189,14 @@ public void parseFile(String server, File file) throws IOException, ParseExcepti
 				globalStats.increaseMethod(logLine.getMethod());
 				globalStats.increaseServer(server);
 
+				Matcher matcher = POPULARITY_PATTERN.matcher(logLine.getRawPath());
+				if (matcher.find()) {
+					String name = matcher.group(1);
+					String target = matcher.group(2);
+
+					globalStats.increasePopularity(name, target);
+				}
+
 				// Version-Specific
 				Stats versionStats = this.versionStats.get(logLine.getApiVersion());
 				if (versionStats == null) {
@@ -320,6 +330,21 @@ public LogLine simplifyPath(LogLine line) {
 		return line;
 	}
 
+	public void cleanUpData() {
+		// Remove all small popularity entries
+		int minPopularity = config.get("log").getAsJsonObject().get("minPopularity").getAsInt();
+		for (String key : globalStats.getPopularity().keySet()) {
+			Map<String, Integer> map = globalStats.getPopularity().get(key);
+			Iterator<Map.Entry<String, Integer>> iterator = map.entrySet().iterator();
+			while (iterator.hasNext()) {
+				Map.Entry<String, Integer> entry = iterator.next();
+				if (entry.getValue() < minPopularity) {
+					iterator.remove();
+				}
+			}
+		}
+	}
+
 	public void saveToDatabase() {
 		Gson gson = new Gson();
 

diff --git a/src/main/java/org/spiget/logparser/Stats.java b/src/main/java/org/spiget/logparser/Stats.java
@@ -10,10 +10,11 @@ public class Stats {
 
 	public int total;
 	public int unique;
-	public Map<String, Integer> userAgents = new HashMap<>();
-	public Map<String, Integer> paths      = new HashMap<>();
-	public Map<String, Integer> methods    = new HashMap<>();
-	public Map<String, Integer> servers    = new HashMap<>();
+	public Map<String, Integer>              userAgents = new HashMap<>();
+	public Map<String, Integer>              paths      = new HashMap<>();
+	public Map<String, Integer>              methods    = new HashMap<>();
+	public Map<String, Integer>              servers    = new HashMap<>();
+	public Map<String, Map<String, Integer>> popularity = new HashMap<>();
 
 	public void increaseUserAgent(String userAgent) {
 		increaseMap(userAgents, userAgent);
@@ -31,6 +32,17 @@ public void increaseServer(String server) {
 		increaseMap(servers, server);
 	}
 
+	public void increasePopularity(String name, String target) {
+		Map<String, Integer> map;
+		if (popularity.containsKey(name)) {
+			map = popularity.get(name);
+		} else {
+			map = new HashMap<>();
+		}
+		increaseMap(map, target);
+		popularity.put(name, map);
+	}
+
 	void increaseMap(Map<String, Integer> map, String key) {
 		if (map.containsKey(key)) {
 			map.put(key, map.get(key) + 1);

diff --git a/src/main/resources/config.json b/src/main/resources/config.json
@@ -21,6 +21,7 @@
       "token": "12345",
       "pass": "54321"
     },
+    "minPopularity": 3,
     "format": {
       "date": "yyyyMMdd",
       "truncateRegex": [