Add creation time field to GoogleMediaItem and populate the upload ti…

…me for the media models. (#1306)

* First changelist description

* Fixes upload time bug. Upload time wasn't being populated

* remove unused code

* remove en secrets

* remove useless changelist

* adressed the comments on the changelist

* fix tests

* Verify creationTime is not null and can be parsed. Skip item if metadata is corrupted.

* Removed nullException check and addressed comments.

* cleanup and syntax.

* added env.secrets to gitignore

* Update excceptions to ParseException

* removed not needed logging

* remove gitignore

* removed end.secrets file

* remove ws

* Added logging and addressed comments

* Syntax formating and refactoring

* removed extra new lines

* Add errorlogging for failed media items

* Inlined errorDetail

* Logged errors and addressed merge issues

* Catching exception in a single block

* Added error logging to GoogleVideoExporter

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/GoogleTransferExtension.java

@@ -149,7 +149,7 @@ public void initialize(ExtensionContext context) {
     exporterBuilder.put(TASKS, new GoogleTasksExporter(credentialFactory, monitor));
     exporterBuilder.put(
         PHOTOS, new GooglePhotosExporter(credentialFactory, jobStore, jsonFactory, monitor));
-    exporterBuilder.put(VIDEOS, new GoogleVideosExporter(credentialFactory, jsonFactory));
+    exporterBuilder.put(VIDEOS, new GoogleVideosExporter(credentialFactory, jobStore, jsonFactory, monitor));
     exporterBuilder.put(
         MEDIA, new GoogleMediaExporter(credentialFactory, jobStore, jsonFactory, monitor, /* photosInterface= */ null, idempotentImportExecutor, enableRetrying));
     exporterBuilder.put(MUSIC, new GoogleMusicExporter(credentialFactory, jsonFactory, monitor));

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java

@@ -29,13 +29,15 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 import javax.annotation.Nullable;
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
+import org.datatransferproject.datatransfer.google.common.GoogleErrorLogger;
 import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
 import org.datatransferproject.datatransfer.google.mediaModels.GoogleAlbum;
 import org.datatransferproject.datatransfer.google.mediaModels.GoogleMediaItem;
@@ -63,6 +65,7 @@
 import org.datatransferproject.types.common.models.photos.PhotosContainerResource;
 import org.datatransferproject.types.common.models.videos.VideoModel;
 import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData;
+import org.datatransferproject.types.transfer.errors.ErrorDetail;
 
 public class GoogleMediaExporter implements Exporter<TokensAndUrlAuthData, MediaContainerResource> {
 
@@ -149,13 +152,13 @@ public ExportResult<MediaContainerResource> export(
       // if ExportInformation is a photos container, this is a request to only export the contents
       // in that container instead of the whole user library
       return exportPhotosContainer(
-          (PhotosContainerResource) exportInformation.get().getContainerResource(), authData);
+          (PhotosContainerResource) exportInformation.get().getContainerResource(), authData, jobId);
     } else if (exportInformation.get().getContainerResource() instanceof MediaContainerResource) {
       // if ExportInformation is a media container, this is a request to only export the contents
       // in that container instead of the whole user library (this is to support backwards
       // compatibility with the GooglePhotosExporter)
       return exportMediaContainer(
-          (MediaContainerResource) exportInformation.get().getContainerResource(), authData);
+          (MediaContainerResource) exportInformation.get().getContainerResource(), authData, jobId);
     }
 
     /*
@@ -198,7 +201,7 @@ public ExportResult<MediaContainerResource> export(
 
   /* Maintain this for backwards compatability, so that we can pull out the album information */
   private ExportResult<MediaContainerResource> exportPhotosContainer(
-      PhotosContainerResource container, TokensAndUrlAuthData authData)
+      PhotosContainerResource container, TokensAndUrlAuthData authData, UUID jobId)
       throws IOException, InvalidTokenException, PermissionDeniedException {
     ImmutableList.Builder<MediaAlbum> albumBuilder = ImmutableList.builder();
     ImmutableList.Builder<PhotoModel> photosBuilder = ImmutableList.builder();
@@ -215,15 +218,26 @@ private ExportResult<MediaContainerResource> exportPhotosContainer(
       subResources.add(new IdOnlyContainerResource(googleAlbum.getId()));
     }
 
+    ImmutableList.Builder<ErrorDetail> errors = ImmutableList.builder();
     for (PhotoModel photo : container.getPhotos()) {
       GoogleMediaItem googleMediaItem =
           getGoogleMediaItem(photo.getIdempotentId(), photo.getDataId(), photo.getName(), authData);
       if (googleMediaItem == null) {
         continue;
       }
 
-      photosBuilder.add(GoogleMediaItem.convertToPhotoModel(Optional.empty(), googleMediaItem));
+      try {
+        photosBuilder.add(GoogleMediaItem.convertToPhotoModel(Optional.empty(), googleMediaItem));
+      } catch(ParseException e) {
+        monitor.info(() -> "Parse exception occurred while converting photo, skipping this item. "
+            + "Failure message : %s ", e.getMessage());
+
+        errors.add(GoogleErrorLogger.createErrorDetail(
+            googleMediaItem.getId(), googleMediaItem.getFilename(), e, /* canSkip= */ true));
+      }
     }
+    // Log all the errors in 1 commit to DataStore
+    GoogleErrorLogger.logFailedItemErrors(jobStore, jobId, errors.build());
 
     MediaContainerResource mediaContainerResource =
         new MediaContainerResource(albumBuilder.build(), photosBuilder.build(), null);
@@ -234,12 +248,11 @@ private ExportResult<MediaContainerResource> exportPhotosContainer(
 
   /* Maintain this for backwards compatability, so that we can pull out the album information */
   private ExportResult<MediaContainerResource> exportMediaContainer(
-      MediaContainerResource container, TokensAndUrlAuthData authData)
+      MediaContainerResource container, TokensAndUrlAuthData authData, UUID jobId)
       throws IOException, InvalidTokenException, PermissionDeniedException {
     ImmutableList.Builder<MediaAlbum> albumBuilder = ImmutableList.builder();
     ImmutableList.Builder<PhotoModel> photosBuilder = ImmutableList.builder();
     ImmutableList.Builder<VideoModel> videosBuilder = ImmutableList.builder();
-
     List<IdOnlyContainerResource> subResources = new ArrayList<>();
 
     for (MediaAlbum album : container.getAlbums()) {
@@ -253,14 +266,23 @@ private ExportResult<MediaContainerResource> exportMediaContainer(
       subResources.add(new IdOnlyContainerResource(googleAlbum.getId()));
     }
 
+    ImmutableList.Builder<ErrorDetail> errors = ImmutableList.builder();
     for (PhotoModel photo : container.getPhotos()) {
       GoogleMediaItem photoMediaItem =
           getGoogleMediaItem(photo.getIdempotentId(), photo.getDataId(), photo.getName(), authData);
       if (photoMediaItem == null) {
         continue;
       }
 
-      photosBuilder.add(GoogleMediaItem.convertToPhotoModel(Optional.empty(), photoMediaItem));
+      try {
+        photosBuilder.add(GoogleMediaItem.convertToPhotoModel(Optional.empty(), photoMediaItem));
+      } catch(ParseException e) {
+        monitor.info(() -> "Parse exception occurred while converting photo, skipping this item. "
+            + "Failure message : %s ", e.getMessage());
+
+        errors.add(GoogleErrorLogger.createErrorDetail(
+            photoMediaItem.getId(), photoMediaItem.getFilename(), e, /* canSkip= */ true));
+      }
     }
 
     for (VideoModel video : container.getVideos()) {
@@ -270,9 +292,20 @@ private ExportResult<MediaContainerResource> exportMediaContainer(
         continue;
       }
 
-      videosBuilder.add(GoogleMediaItem.convertToVideoModel(Optional.empty(), videoMediaItem));
+      try {
+        videosBuilder.add(GoogleMediaItem.convertToVideoModel(Optional.empty(), videoMediaItem));
+      } catch(ParseException e) {
+        monitor.info(() -> "Parse exception occurred while converting video, skipping this item. "
+            + "Failure message : %s ", e.getMessage());
+
+        errors.add(GoogleErrorLogger.createErrorDetail(
+            videoMediaItem.getId(), videoMediaItem.getFilename(), e, /* canSkip= */ true));
+      }
     }
 
+    // Log all the errors in 1 commit to DataStore
+    GoogleErrorLogger.logFailedItemErrors(jobStore, jobId, errors.build());
+
     MediaContainerResource mediaContainerResource =
         new MediaContainerResource(
             albumBuilder.build(), photosBuilder.build(), videosBuilder.build());
@@ -435,6 +468,7 @@ private MediaContainerResource convertMediaListToResource(
       stream.close();
     }
 
+    ImmutableList.Builder<ErrorDetail> errors = ImmutableList.builder();
     for (GoogleMediaItem mediaItem : mediaItems) {
       boolean shouldUpload = albumId.isPresent();
 
@@ -444,21 +478,41 @@ private MediaContainerResource convertMediaListToResource(
 
       if (mediaItem.isPhoto()) {
         if (shouldUpload) {
-          PhotoModel photoModel = GoogleMediaItem.convertToPhotoModel(albumId, mediaItem);
-          photos.add(photoModel);
-
-          monitor.debug(
-              () -> format("%s: Google exporting photo: %s", jobId, photoModel.getDataId()));
+          try {
+            PhotoModel photoModel = GoogleMediaItem.convertToPhotoModel(albumId, mediaItem);
+            photos.add(photoModel);
+
+            monitor.debug(
+                () -> String.format("%s: Google exporting photo: %s", jobId, photoModel.getDataId()));
+          } catch(ParseException e) {
+            monitor.info(() -> "Parse exception occurred while converting photo, skipping this item. "
+                + "Failure message : %s ", e.getMessage());
+
+            errors.add(GoogleErrorLogger.createErrorDetail(
+                mediaItem.getId(), mediaItem.getFilename(), e, /* canSkip= */ true));
+          }
         }
       } else if (mediaItem.isVideo()) {
         if (shouldUpload) {
-          VideoModel videoModel = GoogleMediaItem.convertToVideoModel(albumId, mediaItem);
-          videos.add(videoModel);
-          monitor.debug(
-              () -> format("%s: Google exporting video: %s", jobId, videoModel.getDataId()));
+          try {
+            VideoModel videoModel = GoogleMediaItem.convertToVideoModel(albumId, mediaItem);
+            videos.add(videoModel);
+
+            monitor.debug(
+                () -> String.format("%s: Google exporting video: %s", jobId, videoModel.getDataId()));
+          } catch(ParseException e) {
+            monitor.info(() -> "Parse exception occurred while converting video, skipping this item. "
+                + "Failure message : %s ", e.getMessage());
+
+            errors.add(GoogleErrorLogger.createErrorDetail(
+                mediaItem.getId(), mediaItem.getFilename(), e, /* canSkip= */ true));
+          }
         }
       }
     }
+
+    // Log all the errors in 1 commit to DataStore
+    GoogleErrorLogger.logFailedItemErrors(jobStore, jobId, errors.build());
     return new MediaContainerResource(null  /*albums*/, photos, videos);
   }
 

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/mediaModels/GoogleMediaItem.java

@@ -22,11 +22,15 @@
 import java.io.File;
 import java.io.Serializable;
 import java.nio.file.Files;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.Optional;
 import org.datatransferproject.types.common.models.photos.PhotoModel;
 import org.datatransferproject.types.common.models.videos.VideoModel;
 import org.apache.tika.Tika;
+import com.google.common.base.Strings;
+
 
 /** Media item returned by queries to the Google Photos API. Represents what is stored by Google. */
 public class GoogleMediaItem implements Serializable {
@@ -35,6 +39,7 @@ public class GoogleMediaItem implements Serializable {
   private final static String DEFAULT_VIDEO_MIMETYPE = "video/mp4";
   // If Tika cannot detect the mimetype, it returns the binary mimetype. This can be considered null
   private final static String DEFAULT_BINARY_MIMETYPE = "application/octet-stream";
+  private final static SimpleDateFormat CREATION_TIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
 
   @JsonProperty("id")
   private String id;
@@ -58,6 +63,8 @@ public class GoogleMediaItem implements Serializable {
   private String productUrl;
 
   @JsonProperty("uploadedTime")
+  // TODO akshaysinghh - rename the field to creationTime since creation time is what all the
+  //  services use to display the photos timeline, instead of uploadTime.
   private Date uploadedTime;
 
   public boolean isPhoto() {
@@ -80,7 +87,7 @@ public String getFetchableUrl() {
   }
 
   public static VideoModel convertToVideoModel(
-      Optional<String> albumId, GoogleMediaItem mediaItem) {
+      Optional<String> albumId, GoogleMediaItem mediaItem) throws ParseException{
     Preconditions.checkArgument(mediaItem.isVideo());
 
     return new VideoModel(
@@ -91,11 +98,11 @@ public static VideoModel convertToVideoModel(
         mediaItem.getId(),
         albumId.orElse(null),
         false /*inTempStore*/,
-        mediaItem.getUploadedTime());
+        getCreationTime(mediaItem));
   }
 
-  public static PhotoModel convertToPhotoModel(
-      Optional<String> albumId, GoogleMediaItem mediaItem) {
+  public static PhotoModel convertToPhotoModel (
+      Optional<String> albumId, GoogleMediaItem mediaItem) throws ParseException{
     Preconditions.checkArgument(mediaItem.isPhoto());
 
     return new PhotoModel(
@@ -107,7 +114,18 @@ public static PhotoModel convertToPhotoModel(
         albumId.orElse(null),
         false  /*inTempStore*/,
         null  /*sha1*/,
-        mediaItem.getUploadedTime());
+        getCreationTime(mediaItem));
+  }
+
+  private static Date getCreationTime(GoogleMediaItem mediaItem) throws ParseException  {
+    // cannot be empty or null based. Verified the backend code.
+    try {
+      return CREATION_TIME_FORMAT.parse(mediaItem.getMediaMetadata().getCreationTime());
+    } catch (ParseException parseException) {
+      throw new ParseException(String.format("Failed to parse the string %s to get creationTime for "
+              + "MediaItem %s .", mediaItem.getId(), mediaItem.getMediaMetadata().getCreationTime()),
+          parseException.getErrorOffset());
+    }
   }
 
   private static String getMimeType(GoogleMediaItem mediaItem) {

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/mediaModels/MediaMetadata.java

@@ -18,16 +18,23 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.io.Serializable;
+import java.util.Date;
 
 /** Metadata about a {@code MediaItem}. */
 public class MediaMetadata implements Serializable {
 
+  //Time when the media item was first created (not when it was uploaded to Google Photos).
+  @JsonProperty("creationTime")
+  private String creationTime;
   @JsonProperty("photo")
   private Photo photo;
 
   @JsonProperty("video")
   private Video video;
 
+  public String getCreationTime() {
+    return creationTime;
+  }
   public Photo getPhoto() {
     return photo;
   }
@@ -36,6 +43,10 @@ public Video getVideo() {
     return video;
   }
 
+  public void setCreationTime(String creationTime) {
+    this.creationTime = creationTime;
+  }
+
   public void setPhoto(Photo photo) {
     this.photo = photo;
   }