From a81eaed0ba1ecbbdd14ff4a4e844a27a09fdd537 Mon Sep 17 00:00:00 2001 From: osamahan999 <35288172+osamahan999@users.noreply.github.com> Date: Mon, 8 Jan 2024 11:26:48 -0800 Subject: [PATCH] Add retrying for listAlbums in GoogleMediaExporter (#1324) * Add retrying for listALbums in GoogleMediaExporter * fix comment --- .../common/FailedToListAlbumsException.java | 11 +++ .../google/media/GoogleMediaExporter.java | 76 +++++++++++++------ .../google/media/GoogleMediaExporterTest.java | 9 ++- 3 files changed, 68 insertions(+), 28 deletions(-) create mode 100644 extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java diff --git a/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java b/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java new file mode 100644 index 000000000..78985a2e7 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java @@ -0,0 +1,11 @@ +package org.datatransferproject.datatransfer.google.common; + +/** + * FailedToListAlbumsException is thrown when we try to call PhotosInterface.listAlbums and are + * unsuccessful. + */ +public class FailedToListAlbumsException extends Exception { + public FailedToListAlbumsException(String message, Exception cause) { + super(message, cause); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java b/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java index bec3c675c..a3d70e27f 100644 --- a/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java +++ b/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java @@ -35,7 +35,9 @@ import java.util.Optional; import java.util.UUID; import javax.annotation.Nullable; + import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException; import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory; import org.datatransferproject.datatransfer.google.common.GoogleErrorLogger; import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse; @@ -44,7 +46,6 @@ import org.datatransferproject.datatransfer.google.mediaModels.MediaItemSearchResponse; import org.datatransferproject.datatransfer.google.photos.GooglePhotosInterface; import org.datatransferproject.spi.cloud.storage.JobStore; -import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; import org.datatransferproject.spi.transfer.provider.ExportResult; import org.datatransferproject.spi.transfer.provider.ExportResult.ResultType; @@ -143,7 +144,7 @@ private static String createCacheKey() { @Override public ExportResult export( UUID jobId, TokensAndUrlAuthData authData, Optional exportInformation) - throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException { + throws UploadErrorException, FailedToListAlbumsException, InvalidTokenException, PermissionDeniedException, IOException { if (!exportInformation.isPresent()) { // Make list of photos contained in albums so they are not exported twice later on populateContainedMediaList(jobId, authData); @@ -321,7 +322,7 @@ private ExportResult exportMediaContainer( @VisibleForTesting ExportResult exportAlbums( TokensAndUrlAuthData authData, Optional paginationData, UUID jobId) - throws IOException, InvalidTokenException, PermissionDeniedException { + throws FailedToListAlbumsException { Optional paginationToken = Optional.empty(); if (paginationData.isPresent()) { String token = ((StringPaginationToken) paginationData.get()).getToken(); @@ -330,9 +331,7 @@ ExportResult exportAlbums( paginationToken = Optional.of(token.substring(ALBUM_TOKEN_PREFIX.length())); } - AlbumListResponse albumListResponse; - - albumListResponse = getOrCreatePhotosInterface(authData).listAlbums(paginationToken); + AlbumListResponse albumListResponse = listAlbums(jobId, authData, paginationToken); PaginationData nextPageData; String token = albumListResponse.getNextPageToken(); @@ -406,7 +405,7 @@ ExportResult exportMedia( /** Method for storing a list of all photos that are already contained in albums */ void populateContainedMediaList(UUID jobId, TokensAndUrlAuthData authData) - throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException { + throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException { // This method is only called once at the beginning of the transfer, so we can start by // initializing a new TempMediaData to be store in the job store. TempMediaData tempMediaData = new TempMediaData(jobId); @@ -415,25 +414,29 @@ void populateContainedMediaList(UUID jobId, TokensAndUrlAuthData authData) AlbumListResponse albumListResponse; MediaItemSearchResponse containedMediaSearchResponse; do { - albumListResponse = - getOrCreatePhotosInterface(authData).listAlbums(Optional.ofNullable(albumToken)); - if (albumListResponse.getAlbums() != null) { - for (GoogleAlbum album : albumListResponse.getAlbums()) { - String albumId = album.getId(); - String photoToken = null; - do { - containedMediaSearchResponse = - getOrCreatePhotosInterface(authData) - .listMediaItems(Optional.of(albumId), Optional.ofNullable(photoToken)); - if (containedMediaSearchResponse.getMediaItems() != null) { - for (GoogleMediaItem mediaItem : containedMediaSearchResponse.getMediaItems()) { - tempMediaData.addContainedPhotoId(mediaItem.getId()); - } + albumListResponse = listAlbums(jobId, authData, Optional.ofNullable(albumToken)); + albumToken = albumListResponse.getNextPageToken(); + if (albumListResponse.getAlbums() == null) { + continue; + } + + for (GoogleAlbum album : albumListResponse.getAlbums()) { + String albumId = album.getId(); + String photoToken = null; + + do { + containedMediaSearchResponse = + getOrCreatePhotosInterface(authData) + .listMediaItems(Optional.of(albumId), Optional.ofNullable(photoToken)); + if (containedMediaSearchResponse.getMediaItems() != null) { + for (GoogleMediaItem mediaItem : containedMediaSearchResponse.getMediaItems()) { + tempMediaData.addContainedPhotoId(mediaItem.getId()); } - photoToken = containedMediaSearchResponse.getNextPageToken(); - } while (photoToken != null); - } + } + photoToken = containedMediaSearchResponse.getNextPageToken(); + } while (photoToken != null); } + albumToken = albumListResponse.getNextPageToken(); } while (albumToken != null); @@ -558,6 +561,31 @@ GoogleMediaItem getGoogleMediaItem(String photoIdempotentId, String photoDataId, return null; } + /** + * Tries to call PhotosInterface.listAlbums, and retries on failure. If unsuccessful, throws a + * FailedToListAlbumsException. + */ + private AlbumListResponse listAlbums(UUID jobId, TokensAndUrlAuthData authData, Optional albumToken) + throws FailedToListAlbumsException { + if (retryingExecutor == null || !enableRetrying) { + try { + return getOrCreatePhotosInterface(authData).listAlbums(albumToken); + } catch (IOException | InvalidTokenException | PermissionDeniedException e) { + throw new FailedToListAlbumsException(e.getMessage(), e); + } + } + + try { + return retryingExecutor.executeOrThrowException( + format("%s: listAlbums(page=%s)", jobId, albumToken), + format("listAlbums(page=%s)", albumToken), + () -> getOrCreatePhotosInterface(authData).listAlbums(albumToken) + ); + } catch (Exception e) { + throw new FailedToListAlbumsException(e.getMessage(), e); + } + } + private synchronized GooglePhotosInterface getOrCreatePhotosInterface( TokensAndUrlAuthData authData) { return photosInterface == null ? makePhotosInterface(authData) : photosInterface; diff --git a/extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java b/extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java index d8d457c88..b05599913 100644 --- a/extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java +++ b/extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java @@ -42,6 +42,7 @@ import java.util.UUID; import java.util.stream.Collectors; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException; import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory; import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse; import org.datatransferproject.datatransfer.google.mediaModels.GoogleAlbum; @@ -145,7 +146,7 @@ public void setup() } @Test - public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException { + public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException, FailedToListAlbumsException { setUpSingleAlbum(); when(albumListResponse.getNextPageToken()).thenReturn(ALBUM_TOKEN); @@ -183,7 +184,7 @@ public void exportAlbumFirstSet() throws IOException, InvalidTokenException, Per } @Test - public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException { + public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException, FailedToListAlbumsException { setUpSingleAlbum(); when(albumListResponse.getNextPageToken()).thenReturn(null); @@ -319,7 +320,7 @@ public void exportPhotoSubsequentSet() @Test public void populateContainedMediaList() - throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException { + throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException { // Set up an album with two photos setUpSingleAlbum(); when(albumListResponse.getNextPageToken()).thenReturn(null); @@ -397,7 +398,7 @@ public void testGetGoogleMediaItemSucceeds() throws IOException, InvalidTokenExc } @Test - public void testExportPhotosContainer_photosRetrying() throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException { + public void testExportPhotosContainer_photosRetrying() throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException { String photoIdToFail1 = "photo3"; String photoIdToFail2 = "photo5";