Process information from Artifact POM files

src/main/scala/de/upb/cs/swt/delphi/crawler/discovery/maven/MavenDiscoveryProcess.scala

@@ -28,8 +28,8 @@ import de.upb.cs.swt.delphi.crawler.{AppLogging, Configuration}
 import de.upb.cs.swt.delphi.crawler.control.Phase
 import de.upb.cs.swt.delphi.crawler.control.Phase.Phase
 import de.upb.cs.swt.delphi.crawler.tools.ActorStreamIntegrationSignals.{Ack, StreamCompleted, StreamFailure, StreamInitialized}
-import de.upb.cs.swt.delphi.crawler.preprocessing.{MavenArtifact, MavenDownloadActor}
-import de.upb.cs.swt.delphi.crawler.processing.{HermesActor, HermesResults}
+import de.upb.cs.swt.delphi.crawler.preprocessing.{MavenArtifact, MavenArtifactMetadata, MavenDownloadActor}
+import de.upb.cs.swt.delphi.crawler.processing.{HermesActor, HermesResults, PomFileReadActor}
 import de.upb.cs.swt.delphi.crawler.storage.ArtifactExistsQuery
 import de.upb.cs.swt.delphi.crawler.tools.NotYetImplementedException
 
@@ -57,6 +57,7 @@ class MavenDiscoveryProcess(configuration: Configuration, elasticPool: ActorRef)
   private val seen = mutable.HashSet[MavenIdentifier]()
 
   val downloaderPool = system.actorOf(SmallestMailboxPool(8).props(MavenDownloadActor.props))
+  val pomReaderPool = system.actorOf(SmallestMailboxPool(8).props(PomFileReadActor.props(configuration)))
   val hermesPool = system.actorOf(SmallestMailboxPool(configuration.hermesActorPoolSize).props(HermesActor.props()))
 
   override def phase: Phase = Phase.Discovery
@@ -92,6 +93,8 @@ class MavenDiscoveryProcess(configuration: Configuration, elasticPool: ActorRef)
 
     val finalizer =
       preprocessing
+        .mapAsync(8)(artifact => (pomReaderPool ? artifact).mapTo[MavenArtifact])
+        .alsoTo(createSinkFromActorRef[MavenArtifact](elasticPool))
         .mapAsync(configuration.hermesActorPoolSize)(artifact => (hermesPool ? artifact).mapTo[Try[HermesResults]])
         .filter(results => results.isSuccess)
         .map(results => results.get)

src/main/scala/de/upb/cs/swt/delphi/crawler/preprocessing/MavenArtifact.scala

@@ -17,5 +17,26 @@
 package de.upb.cs.swt.delphi.crawler.preprocessing
 
 import de.upb.cs.swt.delphi.crawler.discovery.maven.MavenIdentifier
+import org.joda.time.DateTime
 
-case class MavenArtifact(identifier : MavenIdentifier, jarFile: JarFile, pomFile: PomFile)
+case class MavenArtifact(identifier : MavenIdentifier, jarFile: JarFile, pomFile: PomFile,
+                         publicationDate: Option[DateTime], metadata: Option[MavenArtifactMetadata])
+
+case class MavenArtifactMetadata(name: String,
+                                 description: String,
+                                 developers: List[String],
+                                 licenses: List[ArtifactLicense],
+                                 issueManagement: Option[IssueManagementData],
+                                 dependencies: Set[ArtifactDependency],
+                                 parent:Option[MavenIdentifier],
+                                 packaging: String)
+
+case class IssueManagementData(system: String, url: String)
+case class ArtifactLicense(name: String, url:String)
+case class ArtifactDependency(identifier: MavenIdentifier, scope: Option[String])
+
+object MavenArtifact{
+  def withMetadata(artifact: MavenArtifact, metadata: MavenArtifactMetadata): MavenArtifact = {
+    MavenArtifact(artifact.identifier, artifact.jarFile, artifact.pomFile, artifact.publicationDate, Some(metadata))
+  }
+}

src/main/scala/de/upb/cs/swt/delphi/crawler/preprocessing/MavenDownloadActor.scala

@@ -16,11 +16,14 @@
 
 package de.upb.cs.swt.delphi.crawler.preprocessing
 
+import java.util.Locale
+
 import akka.actor.{Actor, ActorLogging, ActorSystem, Props}
 import de.upb.cs.swt.delphi.crawler.discovery.maven.MavenIdentifier
 import de.upb.cs.swt.delphi.crawler.tools.HttpDownloader
+import org.joda.time.format.DateTimeFormat
 
-import scala.util.{Failure, Success}
+import scala.util.{Failure, Success, Try}
 
 class MavenDownloadActor extends Actor with ActorLogging {
   override def receive: Receive = {
@@ -30,14 +33,25 @@ class MavenDownloadActor extends Actor with ActorLogging {
       val downloader = new HttpDownloader
 
       val jarStream = downloader.downloadFromUri(m.toJarLocation.toString())
-      val pomStream = downloader.downloadFromUri(m.toPomLocation.toString())
+      val pomResponse = downloader.downloadFromUriWithHeaders(m.toPomLocation.toString())
 
       jarStream match {
         case Success(jar) => {
-          pomStream match {
-            case Success(pom) => {
+          pomResponse match {
+            case Success((pomStream, pomHeaders)) => {
               log.info(s"Downloaded $m")
-              sender() ! Success(MavenArtifact(m, JarFile(jar, m.toJarLocation.toURL), PomFile(pom)))
+
+              // Extract and parse publication date from header
+              val datePattern = DateTimeFormat.forPattern("E, dd MMM yyyy HH:mm:ss zzz").withLocale(Locale.ENGLISH)
+              val pomPublicationDate = pomHeaders.find( _.lowercaseName().equals("last-modified") )
+                .map( header => Try(datePattern.parseDateTime(header.value())) ) match {
+                case Some(Success(date)) => Some(date)
+                case Some(Failure(x)) => x.printStackTrace(); None
+                case _ => None
+              }
+
+              sender() ! Success(MavenArtifact(m, JarFile(jar, m.toJarLocation.toURL), PomFile(pomStream),
+                pomPublicationDate, None))
             }
             case Failure(e) => {
               // TODO: push error to actor