Skip to content

Commit

Permalink
Add retrying for listAlbums in GoogleMediaExporter (#1324)
Browse files Browse the repository at this point in the history 
* Add retrying for listALbums in GoogleMediaExporter

* fix comment
  • Loading branch information
@osamahan999
osamahan999 authored Jan 8, 2024
1 parent d911ff5 commit a81eaed
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 28 deletions.
11 changes: 11 additions & 0 deletions .../java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package org.datatransferproject.datatransfer.google.common;

/**
* FailedToListAlbumsException is thrown when we try to call PhotosInterface.listAlbums and are
* unsuccessful.
*/
public class FailedToListAlbumsException extends Exception {
public FailedToListAlbumsException(String message, Exception cause) {
super(message, cause);
}
}
76 changes: 52 additions & 24 deletions .../src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@
import java.util.Optional;
import java.util.UUID;
import javax.annotation.Nullable;

import org.datatransferproject.api.launcher.Monitor;
import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException;
import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
import org.datatransferproject.datatransfer.google.common.GoogleErrorLogger;
import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
Expand All @@ -44,7 +46,6 @@
import org.datatransferproject.datatransfer.google.mediaModels.MediaItemSearchResponse;
import org.datatransferproject.datatransfer.google.photos.GooglePhotosInterface;
import org.datatransferproject.spi.cloud.storage.JobStore;
import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore;
import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor;
import org.datatransferproject.spi.transfer.provider.ExportResult;
import org.datatransferproject.spi.transfer.provider.ExportResult.ResultType;
Expand Down Expand Up @@ -143,7 +144,7 @@ private static String createCacheKey() {
@Override
public ExportResult<MediaContainerResource> export(
UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation)
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
throws UploadErrorException, FailedToListAlbumsException, InvalidTokenException, PermissionDeniedException, IOException {
if (!exportInformation.isPresent()) {
// Make list of photos contained in albums so they are not exported twice later on
populateContainedMediaList(jobId, authData);
Expand Down Expand Up @@ -321,7 +322,7 @@ private ExportResult<MediaContainerResource> exportMediaContainer(
@VisibleForTesting
ExportResult<MediaContainerResource> exportAlbums(
TokensAndUrlAuthData authData, Optional<PaginationData> paginationData, UUID jobId)
throws IOException, InvalidTokenException, PermissionDeniedException {
throws FailedToListAlbumsException {
Optional<String> paginationToken = Optional.empty();
if (paginationData.isPresent()) {
String token = ((StringPaginationToken) paginationData.get()).getToken();
Expand All @@ -330,9 +331,7 @@ ExportResult<MediaContainerResource> exportAlbums(
paginationToken = Optional.of(token.substring(ALBUM_TOKEN_PREFIX.length()));
}

AlbumListResponse albumListResponse;

albumListResponse = getOrCreatePhotosInterface(authData).listAlbums(paginationToken);
AlbumListResponse albumListResponse = listAlbums(jobId, authData, paginationToken);

PaginationData nextPageData;
String token = albumListResponse.getNextPageToken();
Expand Down Expand Up @@ -406,7 +405,7 @@ ExportResult<MediaContainerResource> exportMedia(

/** Method for storing a list of all photos that are already contained in albums */
void populateContainedMediaList(UUID jobId, TokensAndUrlAuthData authData)
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException {
// This method is only called once at the beginning of the transfer, so we can start by
// initializing a new TempMediaData to be store in the job store.
TempMediaData tempMediaData = new TempMediaData(jobId);
Expand All @@ -415,25 +414,29 @@ void populateContainedMediaList(UUID jobId, TokensAndUrlAuthData authData)
AlbumListResponse albumListResponse;
MediaItemSearchResponse containedMediaSearchResponse;
do {
albumListResponse =
getOrCreatePhotosInterface(authData).listAlbums(Optional.ofNullable(albumToken));
if (albumListResponse.getAlbums() != null) {
for (GoogleAlbum album : albumListResponse.getAlbums()) {
String albumId = album.getId();
String photoToken = null;
do {
containedMediaSearchResponse =
getOrCreatePhotosInterface(authData)
.listMediaItems(Optional.of(albumId), Optional.ofNullable(photoToken));
if (containedMediaSearchResponse.getMediaItems() != null) {
for (GoogleMediaItem mediaItem : containedMediaSearchResponse.getMediaItems()) {
tempMediaData.addContainedPhotoId(mediaItem.getId());
}
albumListResponse = listAlbums(jobId, authData, Optional.ofNullable(albumToken));
albumToken = albumListResponse.getNextPageToken();
if (albumListResponse.getAlbums() == null) {
continue;
}

for (GoogleAlbum album : albumListResponse.getAlbums()) {
String albumId = album.getId();
String photoToken = null;

do {
containedMediaSearchResponse =
getOrCreatePhotosInterface(authData)
.listMediaItems(Optional.of(albumId), Optional.ofNullable(photoToken));
if (containedMediaSearchResponse.getMediaItems() != null) {
for (GoogleMediaItem mediaItem : containedMediaSearchResponse.getMediaItems()) {
tempMediaData.addContainedPhotoId(mediaItem.getId());
}
photoToken = containedMediaSearchResponse.getNextPageToken();
} while (photoToken != null);
}
}
photoToken = containedMediaSearchResponse.getNextPageToken();
} while (photoToken != null);
}

albumToken = albumListResponse.getNextPageToken();
} while (albumToken != null);

Expand Down Expand Up @@ -558,6 +561,31 @@ GoogleMediaItem getGoogleMediaItem(String photoIdempotentId, String photoDataId,
return null;
}

/**
* Tries to call PhotosInterface.listAlbums, and retries on failure. If unsuccessful, throws a
* FailedToListAlbumsException.
*/
private AlbumListResponse listAlbums(UUID jobId, TokensAndUrlAuthData authData, Optional<String> albumToken)
throws FailedToListAlbumsException {
if (retryingExecutor == null || !enableRetrying) {
try {
return getOrCreatePhotosInterface(authData).listAlbums(albumToken);
} catch (IOException | InvalidTokenException | PermissionDeniedException e) {
throw new FailedToListAlbumsException(e.getMessage(), e);
}
}

try {
return retryingExecutor.executeOrThrowException(
format("%s: listAlbums(page=%s)", jobId, albumToken),
format("listAlbums(page=%s)", albumToken),
() -> getOrCreatePhotosInterface(authData).listAlbums(albumToken)
);
} catch (Exception e) {
throw new FailedToListAlbumsException(e.getMessage(), e);
}
}

private synchronized GooglePhotosInterface getOrCreatePhotosInterface(
TokensAndUrlAuthData authData) {
return photosInterface == null ? makePhotosInterface(authData) : photosInterface;
Expand Down
9 changes: 5 additions & 4 deletions .../test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import java.util.UUID;
import java.util.stream.Collectors;
import org.datatransferproject.api.launcher.Monitor;
import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException;
import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
import org.datatransferproject.datatransfer.google.mediaModels.GoogleAlbum;
Expand Down Expand Up @@ -145,7 +146,7 @@ public void setup()
}

@Test
public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException {
public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException, FailedToListAlbumsException {
setUpSingleAlbum();
when(albumListResponse.getNextPageToken()).thenReturn(ALBUM_TOKEN);

Expand Down Expand Up @@ -183,7 +184,7 @@ public void exportAlbumFirstSet() throws IOException, InvalidTokenException, Per
}

@Test
public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException {
public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException, FailedToListAlbumsException {
setUpSingleAlbum();
when(albumListResponse.getNextPageToken()).thenReturn(null);

Expand Down Expand Up @@ -319,7 +320,7 @@ public void exportPhotoSubsequentSet()

@Test
public void populateContainedMediaList()
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException {
// Set up an album with two photos
setUpSingleAlbum();
when(albumListResponse.getNextPageToken()).thenReturn(null);
Expand Down Expand Up @@ -397,7 +398,7 @@ public void testGetGoogleMediaItemSucceeds() throws IOException, InvalidTokenExc
}

@Test
public void testExportPhotosContainer_photosRetrying() throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
public void testExportPhotosContainer_photosRetrying() throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException {
String photoIdToFail1 = "photo3";
String photoIdToFail2 = "photo5";

Expand Down

0 comments on commit a81eaed

Please sign in to comment.