Retry listMediaItems in GoogleMediaExporter

dtinit · Jan 8, 2024 · 945fe90 · 945fe90
1 parent a81eaed
commit 945fe90
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 8 deletions.
diff --git a/...a/org/datatransferproject/datatransfer/google/common/FailedToListMediaItemsException.java b/...a/org/datatransferproject/datatransfer/google/common/FailedToListMediaItemsException.java
@@ -0,0 +1,7 @@
+package org.datatransferproject.datatransfer.google.common;
+
+public class FailedToListMediaItemsException extends Exception {
+    public FailedToListMediaItemsException(String message, Exception cause) {
+        super(message, cause);
+    }
+}
diff --git a/.../src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java b/.../src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java
@@ -38,6 +38,7 @@
 
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException;
+import org.datatransferproject.datatransfer.google.common.FailedToListMediaItemsException;
 import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
 import org.datatransferproject.datatransfer.google.common.GoogleErrorLogger;
 import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
@@ -144,7 +145,7 @@ private static String createCacheKey() {
   @Override
   public ExportResult<MediaContainerResource> export(
       UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation)
-          throws UploadErrorException, FailedToListAlbumsException, InvalidTokenException, PermissionDeniedException, IOException {
+          throws UploadErrorException, FailedToListAlbumsException, InvalidTokenException, PermissionDeniedException, IOException, FailedToListMediaItemsException {
     if (!exportInformation.isPresent()) {
       // Make list of photos contained in albums so they are not exported twice later on
       populateContainedMediaList(jobId, authData);
@@ -371,15 +372,14 @@ ExportResult<MediaContainerResource> exportMedia(
       Optional<IdOnlyContainerResource> albumData,
       Optional<PaginationData> paginationData,
       UUID jobId)
-      throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
+      throws IOException, FailedToListMediaItemsException {
     Optional<String> albumId = Optional.empty();
     if (albumData.isPresent()) {
       albumId = Optional.of(albumData.get().getId());
     }
     Optional<String> paginationToken = getPhotosPaginationToken(paginationData);
 
-    MediaItemSearchResponse mediaItemSearchResponse =
-        getOrCreatePhotosInterface(authData).listMediaItems(albumId, paginationToken);
+    MediaItemSearchResponse mediaItemSearchResponse = listMediaItems(jobId, authData, albumId, paginationToken);
 
     PaginationData nextPageData = null;
     if (!Strings.isNullOrEmpty(mediaItemSearchResponse.getNextPageToken())) {
@@ -586,6 +586,36 @@ private AlbumListResponse listAlbums(UUID jobId, TokensAndUrlAuthData authData,
     }
   }
 
+  /**
+   * Tries to call PhotosInterface.ListMediaItems, and retries on failure. If unsuccessful, throws a
+   * FailedToListMediaItemsException.
+   */
+  private MediaItemSearchResponse listMediaItems(
+          UUID jobId,
+          TokensAndUrlAuthData authData,
+          Optional<String> albumId,
+          Optional<String> pageToken
+  )
+          throws FailedToListMediaItemsException {
+    if (retryingExecutor == null || !enableRetrying) {
+      try {
+        return getOrCreatePhotosInterface(authData).listMediaItems(albumId, pageToken);
+      } catch (IOException | InvalidTokenException | PermissionDeniedException | UploadErrorException e) {
+        throw new FailedToListMediaItemsException(e.getMessage(), e);
+      }
+    }
+
+    try {
+      return retryingExecutor.executeOrThrowException(
+              format("%s: listMediaItems(albumId=%s, page=%s)", jobId, albumId, pageToken),
+              format("listMediaItems(albumId=%s, page=%s)", albumId, pageToken),
+              () -> getOrCreatePhotosInterface(authData).listMediaItems(albumId, pageToken)
+      );
+    } catch (Exception e) {
+      throw new FailedToListMediaItemsException(e.getMessage(), e);
+    }
+  }
+
   private synchronized GooglePhotosInterface getOrCreatePhotosInterface(
       TokensAndUrlAuthData authData) {
     return photosInterface == null ? makePhotosInterface(authData) : photosInterface;

diff --git a/.../test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java b/.../test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java
@@ -43,6 +43,7 @@
 import java.util.stream.Collectors;
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException;
+import org.datatransferproject.datatransfer.google.common.FailedToListMediaItemsException;
 import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
 import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
 import org.datatransferproject.datatransfer.google.mediaModels.GoogleAlbum;
@@ -209,7 +210,7 @@ public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException
 
   @Test
   public void exportPhotoFirstSet()
-      throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
+          throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListMediaItemsException {
     setUpSingleAlbum();
     when(albumListResponse.getNextPageToken()).thenReturn(null);
     GoogleMediaItem mediaItem = setUpSinglePhoto("some://fake/gphotoapi/uri", "some-upstream-generated-photo-id");
@@ -249,7 +250,7 @@ public void exportPhotoFirstSet()
 
   @Test
   public void exportVideoFirstSet()
-      throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
+          throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListMediaItemsException {
     setUpSingleAlbum();
     when(albumListResponse.getNextPageToken()).thenReturn(null);
     GoogleMediaItem mediaItem = setUpSingleVideo(
@@ -290,7 +291,7 @@ public void exportVideoFirstSet()
 
   @Test
   public void exportPhotoSubsequentSet()
-      throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
+          throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListMediaItemsException {
     setUpSingleAlbum();
     when(albumListResponse.getNextPageToken()).thenReturn(null);
     GoogleMediaItem mediaItem = setUpSinglePhoto(
@@ -354,7 +355,7 @@ public void populateContainedMediaList()
   photos are exported.
   */
   public void onlyExportAlbumlessPhoto()
-      throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
+          throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListMediaItemsException {
     // Set up - two photos will be returned by a media item search without an album id, but one of
     // them will have already been put into the list of contained photos
     String containedPhotoUri = "contained photo uri";