From af136b5c1dc89b6da6def4725c37af46271c3b02 Mon Sep 17 00:00:00 2001 From: dvitiiuk Date: Tue, 26 Nov 2019 15:07:58 +0200 Subject: [PATCH 1/2] PLUGIN-68. Integration tests for Google Drive plugins (SaaS source). --- .../app/etl/google/drive/GoogleDriveTest.java | 832 ++++++++++++++++++ .../google/drive/UserCredentialsTestBase.java | 52 ++ pom.xml | 6 + 3 files changed, 890 insertions(+) create mode 100644 integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java create mode 100644 integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java diff --git a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java new file mode 100644 index 000000000..5c442a8a9 --- /dev/null +++ b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java @@ -0,0 +1,832 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.app.etl.google.drive; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.File; +import com.google.api.services.drive.model.FileList; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.cdap.cdap.api.artifact.ArtifactScope; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.common.ArtifactNotFoundException; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.utils.Tasks; +import io.cdap.cdap.datapipeline.SmartWorkflow; +import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.proto.ArtifactSelectorConfig; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.ProgramRunStatus; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.artifact.PluginSummary; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.WorkflowManager; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Tests reading to and writing from Google Drive within a sandbox cluster. + */ +public class GoogleDriveTest extends UserCredentialsTestBase { + + @Rule + public TestName testName = new TestName(); + + protected static final ArtifactSelectorConfig GOOGLE_DRIVE_ARTIFACT = + new ArtifactSelectorConfig("SYSTEM", "google-drive-plugins", "[0.0.0, 100.0.0)"); + protected static final ArtifactSelectorConfig FILE_ARTIFACT = + new ArtifactSelectorConfig("SYSTEM", "core-plugins", "[0.0.0, 100.0.0)"); + private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); + private static final String GOOGLE_DRIVE_PLUGIN_NAME = "GoogleDrive"; + private static final String FILE_PLUGIN_NAME = "File"; + private static final int GENERATED_NAME_LENGTH = 16; + private static final String TEXT_PLAIN_MIME = "text/plain"; + private static final String TEXT_CSV_MIME = "text/csv"; + private static final String UNDEFINED_MIME = "application/octet-stream"; + + private static final String TEST_TEXT_FILE_NAME = "textFile"; + private static final String TEST_DOC_FILE_NAME = "docFile"; + private static final String TEST_SHEET_FILE_NAME = "sheetFile"; + private static final String TEST_TEXT_FILE_CONTENT = "text file content"; + private static final String TEST_DOC_FILE_CONTENT = "Google Document file content"; + private static final String TEST_SHEET_FILE_CONTENT = "a,b,c\r\n,d,e"; + private static final String DRIVE_SOURCE_STAGE_NAME = "GoogleDriveSource"; + private static final String DRIVE_SINK_STAGE_NAME = "GoogleDriveSink"; + private static final String FILE_SOURCE_STAGE_NAME = "FileSource"; + private static final String FILE_SINK_STAGE_NAME = "FileSink"; + public static final String TMP_FOLDER_NAME = "googleDriveTestFolder"; + + private static Drive service; + private String sourceFolderId; + private String sinkFolderId; + private String testTextFileId; + private String testDocFileId; + private String testSheetFileId; + private Path tmpFolder; + + @BeforeClass + public static void setupDrive() throws GeneralSecurityException, IOException { + final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + + GoogleCredential credential = new GoogleCredential.Builder() + .setTransport(HTTP_TRANSPORT) + .setJsonFactory(JSON_FACTORY) + .setClientSecrets(getClientId(), + getClientSecret()) + .build(); + credential.setRefreshToken(getRefreshToken()); + + service = new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).build(); + } + + @Before + public void testClassSetup() throws IOException { + ImmutableList.of(ImmutableList.of(GOOGLE_DRIVE_PLUGIN_NAME, BatchSource.PLUGIN_TYPE, "cdap-data-pipeline"), + ImmutableList.of(GOOGLE_DRIVE_PLUGIN_NAME, BatchSink.PLUGIN_TYPE, "cdap-data-pipeline")) + .forEach((pluginInfo) -> checkPluginExists(pluginInfo.get(0), pluginInfo.get(1), pluginInfo.get(2))); + + String sourceFolderName = RandomStringUtils.randomAlphanumeric(16); + String sinkFolderName = RandomStringUtils.randomAlphanumeric(16); + + sourceFolderId = createFolder(service, sourceFolderName); + sinkFolderId = createFolder(service, sinkFolderName); + + testTextFileId = createFile(service, TEST_TEXT_FILE_CONTENT.getBytes(), TEST_TEXT_FILE_NAME, + TEXT_PLAIN_MIME, null, sourceFolderId); + testDocFileId = createFile(service, TEST_DOC_FILE_CONTENT.getBytes(), TEST_DOC_FILE_NAME, + "application/vnd.google-apps.document", TEXT_PLAIN_MIME, sourceFolderId); + testSheetFileId = createFile(service, TEST_SHEET_FILE_CONTENT.getBytes(), TEST_SHEET_FILE_NAME, + "application/vnd.google-apps.spreadsheet", TEXT_CSV_MIME, sourceFolderId); + tmpFolder = createFileSystemFolder(TMP_FOLDER_NAME); + } + + @After + public void removeFolders() throws IOException { + removeFile(service, testTextFileId); + removeFile(service, testDocFileId); + removeFile(service, testSheetFileId); + removeFile(service, sourceFolderId); + removeFile(service, sinkFolderId); + + Files.walk(tmpFolder) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(java.io.File::delete); + } + + @Test + public void testBinaryOnly() throws Exception { + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "binary"); + } + }; + Map sinkProps = getDriveSinkMinimalDefaultConfigs(); + + DeploymentDetails deploymentDetails = + deployGoogleDriveApplication(sourceProps, sinkProps, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(1, destFiles.size()); + + File textFile = destFiles.get(0); + + Assert.assertEquals(UNDEFINED_MIME, textFile.getMimeType()); + Assert.assertNotEquals(TEST_TEXT_FILE_NAME, textFile.getName()); + Assert.assertEquals(GENERATED_NAME_LENGTH, textFile.getName().length()); + + String content = getFileContent(textFile.getId()); + Assert.assertEquals(TEST_TEXT_FILE_CONTENT, content); + } + + @Test + public void testDocFileOnly() throws Exception { + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "documents"); + } + }; + Map sinkProps = getDriveSinkMinimalDefaultConfigs(); + + DeploymentDetails deploymentDetails = + deployGoogleDriveApplication(sourceProps, sinkProps, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(1, destFiles.size()); + + File docFile = destFiles.get(0); + + Assert.assertEquals(UNDEFINED_MIME, docFile.getMimeType()); + Assert.assertNotEquals(TEST_TEXT_FILE_NAME, docFile.getName()); + Assert.assertEquals(GENERATED_NAME_LENGTH, docFile.getName().length()); + + String content = getFileContent(docFile.getId()); + // check BOM + Assert.assertEquals('\uFEFF', content.charAt(0)); + Assert.assertEquals(TEST_DOC_FILE_CONTENT, content.replace("\uFEFF", "")); + } + + @Test + public void testAllFileTypes() throws Exception { + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "binary,documents,spreadsheets"); + } + }; + Map sinkProps = getDriveSinkMinimalDefaultConfigs(); + + DeploymentDetails deploymentDetails = + deployGoogleDriveApplication(sourceProps, sinkProps, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 3); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(3, destFiles.size()); + + destFiles.forEach(file -> { + Assert.assertEquals(UNDEFINED_MIME, file.getMimeType()); + Assert.assertNotEquals(TEST_TEXT_FILE_NAME, file.getName()); + Assert.assertNotEquals(TEST_DOC_FILE_NAME, file.getName()); + Assert.assertNotEquals(TEST_SHEET_FILE_NAME, file.getName()); + Assert.assertEquals(GENERATED_NAME_LENGTH, file.getName().length()); + }); + } + + @Test + public void testAllFileTypesNamed() throws Exception { + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "binary,documents,spreadsheets"); + put("fileMetadataProperties", "name"); + } + }; + Map sinkProps = new HashMap() { + { + putAll(getDriveSinkMinimalDefaultConfigs()); + put("schemaNameFieldName", "name"); + } + }; + + DeploymentDetails deploymentDetails = + deployGoogleDriveApplication(sourceProps, sinkProps, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 3); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(3, destFiles.size()); + + destFiles.forEach(file -> { + Assert.assertEquals(UNDEFINED_MIME, file.getMimeType()); + Assert.assertNotNull(file.getName()); + try { + String fileName = file.getName(); + String content = getFileContent(file.getId()); + switch (fileName) { + case TEST_TEXT_FILE_NAME: + Assert.assertEquals(TEST_TEXT_FILE_CONTENT, content); + break; + case TEST_DOC_FILE_NAME: + // check BOM + Assert.assertEquals('\uFEFF', content.charAt(0)); + Assert.assertEquals(TEST_DOC_FILE_CONTENT, content.replace("\uFEFF", "")); + break; + case TEST_SHEET_FILE_NAME: + Assert.assertEquals(TEST_SHEET_FILE_CONTENT, content); + break; + default: + Assert.fail(String.format("Invalid file name after pipeline completion: '%s', content: '%s'", + fileName, content)); + } + } catch (IOException e) { + Assert.fail(String.format("Exception during test results check: '%s'", e.getMessage())); + } + }); + } + + @Test + public void testAllFileTypesNamedAndMimed() throws Exception { + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "binary,documents,spreadsheets"); + put("fileMetadataProperties", "name,mimeType"); + } + }; + Map sinkProps = new HashMap() { + { + putAll(getDriveSinkMinimalDefaultConfigs()); + put("schemaNameFieldName", "name"); + put("schemaMimeFieldName", "mimeType"); + } + }; + + DeploymentDetails deploymentDetails = + deployGoogleDriveApplication(sourceProps, sinkProps, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 3); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(3, destFiles.size()); + + destFiles.forEach(file -> { + Assert.assertNotNull(file.getName()); + String fileName = null; + String mimeType = null; + try { + fileName = file.getName(); + mimeType = file.getMimeType(); + String content = getFileContent(file.getId()); + switch (fileName) { + case TEST_TEXT_FILE_NAME: + Assert.assertEquals(TEST_TEXT_FILE_CONTENT, content); + Assert.assertEquals(TEXT_PLAIN_MIME, mimeType); + break; + case TEST_DOC_FILE_NAME: + // check BOM + Assert.assertEquals('\uFEFF', content.charAt(0)); + Assert.assertEquals(TEST_DOC_FILE_CONTENT, content.replace("\uFEFF", "")); + Assert.assertEquals(TEXT_PLAIN_MIME, mimeType); + break; + case TEST_SHEET_FILE_NAME: + Assert.assertEquals(TEST_SHEET_FILE_CONTENT, content); + Assert.assertEquals(TEXT_CSV_MIME, mimeType); + break; + default: + Assert.fail( + String.format("Invalid file name after pipeline completion: '%s', content: '%s', mime type: '%s'", + fileName, content, mimeType)); + } + } catch (IOException e) { + Assert.fail(String.format("Exception during test results check: '%s', file name '%s', mimeType '%s'", + e.getMessage(), + fileName == null ? "unknown" : fileName, + mimeType == null ? "unknown" : mimeType)); + } + }); + } + + @Test + public void testPartitionSize() throws Exception { + int testMaxPartitionSize = 10; + + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "binary,documents,spreadsheets"); + put("fileMetadataProperties", "name,mimeType"); + put("maxPartitionSize", Integer.toString(testMaxPartitionSize)); + } + }; + Map sinkProps = new HashMap() { + { + putAll(getDriveSinkMinimalDefaultConfigs()); + put("schemaNameFieldName", "name"); + put("schemaMimeFieldName", "mimeType"); + } + }; + + DeploymentDetails deploymentDetails = + deployGoogleDriveApplication(sourceProps, sinkProps, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 4); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(4, destFiles.size()); + + // flags to check partitioning work + boolean firstTextPart = false; + boolean secondTextPart = false; + List parts = new ArrayList<>(); + + // Document and Sheets don't support partitioning + for (File file : destFiles) { + Assert.assertNotNull(file.getName()); + try { + String fileName = file.getName(); + String mimeType = file.getMimeType(); + String content = getFileContent(file.getId()); + switch (fileName) { + case TEST_TEXT_FILE_NAME: + Assert.assertNotEquals(TEST_TEXT_FILE_CONTENT, content); + Assert.assertEquals(TEXT_PLAIN_MIME, mimeType); + parts.add(content); + if (content.equals(TEST_TEXT_FILE_CONTENT.substring(0, testMaxPartitionSize))) { + firstTextPart = true; + } + if (content.equals(TEST_TEXT_FILE_CONTENT.substring(testMaxPartitionSize))) { + secondTextPart = true; + } + break; + case TEST_DOC_FILE_NAME: + // check BOM + Assert.assertEquals('\uFEFF', content.charAt(0)); + Assert.assertEquals(TEST_DOC_FILE_CONTENT, content.replace("\uFEFF", "")); + Assert.assertEquals(TEXT_PLAIN_MIME, mimeType); + break; + case TEST_SHEET_FILE_NAME: + Assert.assertEquals(TEST_SHEET_FILE_CONTENT, content); + Assert.assertEquals(TEXT_CSV_MIME, mimeType); + break; + default: + Assert.fail( + String.format("Invalid file name after pipeline completion: '%s', content: '%s', mime type: '%s'", + fileName, content, mimeType)); + } + } catch (IOException e) { + Assert.fail(String.format("Exception during test results check: '%s'", e.getMessage())); + } + } + Assert.assertTrue(String.format("Text file was separated incorrectly: '%s'", parts.toString()), firstTextPart); + Assert.assertTrue(String.format("Text file was separated incorrectly: '%s'", parts.toString()), secondTextPart); + } + + @Test + public void testWithFileSource() throws Exception { + // create test file + createFileSystemTextFile(tmpFolder, TEST_TEXT_FILE_NAME, TEST_TEXT_FILE_CONTENT); + + Map sourceProps = getFileSourceMinimalDefaultConfigs(); + Map sinkProps = getDriveSinkMinimalDefaultConfigs(); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, FILE_SOURCE_STAGE_NAME, DRIVE_SINK_STAGE_NAME, + FILE_PLUGIN_NAME, GOOGLE_DRIVE_PLUGIN_NAME, FILE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + List destFiles = getFiles(sinkFolderId); + Assert.assertEquals(1, destFiles.size()); + + File textFile = destFiles.get(0); + + Assert.assertEquals(UNDEFINED_MIME, textFile.getMimeType()); + Assert.assertNotEquals(TEST_TEXT_FILE_NAME, textFile.getName()); + Assert.assertEquals(GENERATED_NAME_LENGTH, textFile.getName().length()); + + String content = getFileContent(textFile.getId()); + Assert.assertEquals(TEST_TEXT_FILE_CONTENT, content); + } + + @Test + public void testWithFileSink() throws Exception { + int testMaxPartitionSize = 50; + String testFileName = "Image.png"; + String testFileMime = "image/png"; + byte[] testPNGContent = new byte[]{-119, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 5, 0, + 0, 0, 5, 8, 2, 0, 0, 0, 2, 13, -79, -78, 0, 0, 0, 9, 112, 72, 89, 115, 0, 0, 11, 19, 0, 0, 11, 19, 1, 0, -102, + -100, 24, 0, 0, 0, 7, 116, 73, 77, 69, 7, -29, 10, 18, 13, 43, 15, 2, -77, 55, -110, 0, 0, 0, 25, 116, 69, 88, + 116, 67, 111, 109, 109, 101, 110, 116, 0, 67, 114, 101, 97, 116, 101, 100, 32, 119, 105, 116, 104, 32, 71, 73, + 77, 80, 87, -127, 14, 23, 0, 0, 0, 40, 73, 68, 65, 84, 8, -41, 93, -117, 65, 10, 0, 48, 12, -62, -30, -1, 31, + -99, 29, 108, -95, -52, -125, 72, -44, -88, 73, 84, 0, 8, -85, 65, 106, 83, -3, 44, -5, -6, -6, 7, -62, -105, 32, + -23, 115, 33, -2, -49, 0, 0, 0, 0, 73, 69, 78, 68, -82, 66, 96, -126}; + int contentLength = testPNGContent.length; + + // create png file with metadata in Google Drive + // size: 174 bytes + createFile(service, testPNGContent, testFileName, "image/png", null, sourceFolderId); + + Map sourceProps = new HashMap() { + { + putAll(getDriveSourceMinimalDefaultConfigs()); + put("fileTypesToPull", "binary,documents,spreadsheets"); + put("bodyFormat", "bytes"); + put("filter", String.format("name='%s'", testFileName)); + put("fileMetadataProperties", "name,mimeType,size,imageMediaMetadata.width," + + "imageMediaMetadata.height,imageMediaMetadata.rotation"); + put("maxPartitionSize", Integer.toString(testMaxPartitionSize)); + } + }; + Map sinkProps = getFileSinkMinimalDefaultConfigs(); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, DRIVE_SOURCE_STAGE_NAME, FILE_SINK_STAGE_NAME, + GOOGLE_DRIVE_PLUGIN_NAME, FILE_PLUGIN_NAME, GOOGLE_DRIVE_ARTIFACT, FILE_ARTIFACT, + GOOGLE_DRIVE_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 4); + + Assert.assertTrue(Files.isDirectory(tmpFolder)); + + List allDeploysResults = Files.list(tmpFolder).collect(Collectors.toList()); + Assert.assertEquals(1, allDeploysResults.size()); + + Path deployResult = allDeploysResults.get(0); + Assert.assertTrue(Files.isDirectory(deployResult)); + Assert.assertEquals(1, + Files.list(deployResult).filter(p -> p.getFileName().toString().equals("_SUCCESS")).count()); + + List destFiles = + Files.list(deployResult).filter(p -> p.getFileName().toString().startsWith("part")).collect(Collectors.toList()); + Assert.assertEquals(4, destFiles.size()); + + JsonParser jsonParser = new JsonParser(); + + Map partitionedContent = new HashMap<>(); + for (Path destFile : destFiles) { + List fileLines = null; + try { + fileLines = Files.readAllLines(destFile); + } catch (IOException e) { + Assert.fail(String.format("Exception during reading file '%s': '%s'", destFile.toString(), e.getMessage())); + } + String fileContent = String.join(",", fileLines); + JsonElement rootElement = jsonParser.parse(fileContent); + Assert.assertTrue(rootElement.isJsonObject()); + + JsonObject rootObject = rootElement.getAsJsonObject(); + + // Entries: name, mimeType, size, imageMediaMetadata, offset, body + Assert.assertEquals(6, rootObject.entrySet().size()); + Assert.assertEquals(testFileName, rootObject.get("name").getAsString()); + Assert.assertEquals(testFileMime, rootObject.get("mimeType").getAsString()); + Assert.assertEquals(contentLength, rootObject.get("size").getAsInt()); + + JsonObject imageMediaMetadataObject = rootObject.get("imageMediaMetadata").getAsJsonObject(); + + // Image metadata entries: width, height, rotation + Assert.assertEquals(3, imageMediaMetadataObject.entrySet().size()); + Assert.assertEquals(5, imageMediaMetadataObject.get("width").getAsInt()); + Assert.assertEquals(5, imageMediaMetadataObject.get("height").getAsInt()); + Assert.assertEquals(0, imageMediaMetadataObject.get("rotation").getAsInt()); + + // collect bodies for next check + int resultOffset = rootObject.get("offset").getAsInt(); + JsonArray bytes = rootObject.get("body").getAsJsonArray(); + byte[] resultBody = new byte[bytes.size()]; + for (int i = 0; i < bytes.size(); i++) { + resultBody[i] = bytes.get(i).getAsByte(); + } + partitionedContent.put(resultOffset, resultBody); + } + byte[] assembledContent = new byte[contentLength]; + ByteBuffer buffer = ByteBuffer.wrap(assembledContent); + for (Map.Entry part : partitionedContent.entrySet()) { + buffer.position(part.getKey()); + buffer.put(part.getValue()); + } + Assert.assertArrayEquals(testPNGContent, buffer.array()); + } + + private Map getDriveSourceMinimalDefaultConfigs() { + return new HashMap() { + { + put("referenceName", "ref"); + put("directoryIdentifier", sourceFolderId); + put("modificationDateRange", "lifetime"); + put("fileTypesToPull", "binary"); + put("maxPartitionSize", "0"); + put("bodyFormat", "bytes"); + put("docsExportingFormat", "text/plain"); + put("sheetsExportingFormat", "text/csv"); + put("drawingsExportingFormat", "image/svg+xml"); + put("presentationsExportingFormat", "text/plain"); + put("authType", "oAuth2"); + put("clientId", getClientId()); + put("clientSecret", getClientSecret()); + put("refreshToken", getRefreshToken()); + put("maxRetryCount", "8"); + put("maxRetryWait", "200"); + put("maxRetryJitterWait", "100"); + } + }; + } + + private Map getFileSourceMinimalDefaultConfigs() { + Set schemaFields = new HashSet<>(); + schemaFields.add(Schema.Field.of("body", Schema.nullableOf(Schema.of(Schema.Type.BYTES)))); + Schema fileSchema = Schema.recordOf( + "blob", + schemaFields); + return new HashMap() { + { + put("path", tmpFolder.toString()); + put("referenceName", "fileref"); + put("format", "blob"); + put("schema", fileSchema.toString()); + } + }; + } + + private Map getDriveSinkMinimalDefaultConfigs() { + return new HashMap() { + { + put("referenceName", "refd"); + put("directoryIdentifier", sinkFolderId); + put("schemaBodyFieldName", "body"); + put("authType", "oAuth2"); + put("clientId", getClientId()); + put("clientSecret", getClientSecret()); + put("refreshToken", getRefreshToken()); + put("maxRetryCount", "8"); + put("maxRetryWait", "200"); + put("maxRetryJitterWait", "100"); + } + }; + } + + private Map getFileSinkMinimalDefaultConfigs() { + return new HashMap() { + { + put("suffix", "yyyy-MM-dd-HH-mm"); + put("path", tmpFolder.toString()); + put("referenceName", "fileref"); + put("format", "json"); + } + }; + } + + protected void startWorkFlow(ApplicationManager appManager, ProgramRunStatus expectedStatus) throws Exception { + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.startAndWaitForRun(expectedStatus, 5, TimeUnit.MINUTES); + } + + private void checkRowsNumber(DeploymentDetails deploymentDetails, int expectedCount) throws Exception { + ApplicationId appId = deploymentDetails.getAppId(); + Map tags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, appId.getNamespace(), + Constants.Metrics.Tag.APP, appId.getEntityName()); + checkMetric(tags, "user." + deploymentDetails.getSource().getName() + ".records.out", + expectedCount, 10); + checkMetric(tags, "user." + deploymentDetails.getSink().getName() + ".records.in", + expectedCount, 10); + } + + private void checkPluginExists(String pluginName, String pluginType, String artifact) { + Preconditions.checkNotNull(pluginName); + Preconditions.checkNotNull(pluginType); + Preconditions.checkNotNull(artifact); + + try { + Tasks.waitFor(true, () -> { + try { + final ArtifactId artifactId = TEST_NAMESPACE.artifact(artifact, version); + List plugins = + artifactClient.getPluginSummaries(artifactId, pluginType, ArtifactScope.SYSTEM); + return plugins.stream().anyMatch(pluginSummary -> pluginName.equals(pluginSummary.getName())); + } catch (ArtifactNotFoundException e) { + // happens if the relevant artifact(s) were not added yet + return false; + } + }, 5, TimeUnit.MINUTES, 3, TimeUnit.SECONDS); + } catch (Exception e) { + Throwables.throwIfUnchecked(e); + throw new RuntimeException(e); + } + } + + private DeploymentDetails deployGoogleDriveApplication(Map sourceProperties, + Map sinkProperties, + String applicationName) throws Exception { + return deployApplication(sourceProperties, sinkProperties, + DRIVE_SOURCE_STAGE_NAME, DRIVE_SINK_STAGE_NAME, + GOOGLE_DRIVE_PLUGIN_NAME, GOOGLE_DRIVE_PLUGIN_NAME, + GOOGLE_DRIVE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, applicationName); + } + + private DeploymentDetails deployApplication(Map sourceProperties, Map sinkProperties, + String sourceStageName, String sinkStageName, + String sourcePluginName, String sinkPluginName, + ArtifactSelectorConfig sourceArtifact, + ArtifactSelectorConfig sinkArtifact, String applicationName) + throws Exception { + ETLStage source = new ETLStage(sourceStageName, + new ETLPlugin(sourcePluginName, + BatchSource.PLUGIN_TYPE, + sourceProperties, + sourceArtifact)); + ETLStage sink = new ETLStage(sinkStageName, new ETLPlugin(sinkPluginName, + BatchSink.PLUGIN_TYPE, + sinkProperties, + sinkArtifact)); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(source) + .addStage(sink) + .addConnection(source.getName(), sink.getName()) + .build(); + + AppRequest appRequest = getBatchAppRequestV2(etlConfig); + ApplicationId appId = TEST_NAMESPACE.app(applicationName); + ApplicationManager applicationManager = deployApplication(appId, appRequest); + return new DeploymentDetails(source, sink, appId, applicationManager); + } + + private static String createFile(Drive service, byte[] content, String name, String mime, String subMime, + String folderId) throws IOException { + File fileToWrite = new File(); + fileToWrite.setName(name); + fileToWrite.setParents(Collections.singletonList(folderId)); + fileToWrite.setMimeType(mime); + ByteArrayContent fileContent = new ByteArrayContent(subMime, content); + + File file = service.files().create(fileToWrite, fileContent) + .setFields("id, parents, mimeType") + .execute(); + return file.getId(); + } + + private static String createFolder(Drive service, String folderName) throws IOException { + File fileMetadata = new File(); + fileMetadata.setName(folderName); + fileMetadata.setMimeType("application/vnd.google-apps.folder"); + + File createdFolder = service.files().create(fileMetadata).setFields("id").execute(); + return createdFolder.getId(); + } + + private static void removeFile(Drive service, String fileId) throws IOException { + service.files().delete(fileId).execute(); + } + + private static List getFiles(String parentFolderId) { + try { + List files = new ArrayList<>(); + String nextToken = ""; + Drive.Files.List request = service.files().list() + .setQ(String.format("'%s' in parents", parentFolderId)) + .setFields("nextPageToken, files(id, name, size, mimeType)"); + while (nextToken != null) { + FileList result = request.execute(); + files.addAll(result.getFiles()); + nextToken = result.getNextPageToken(); + request.setPageToken(nextToken); + } + return files; + } catch (IOException e) { + throw new RuntimeException("Issue during retrieving summary for files.", e); + } + } + + private static String getFileContent(String fileId) throws IOException { + OutputStream outputStream = new ByteArrayOutputStream(); + Drive.Files.Get get = service.files().get(fileId); + + get.executeMediaAndDownloadTo(outputStream); + return ((ByteArrayOutputStream) outputStream).toString(); + } + + private static Path createFileSystemFolder(String path) throws IOException { + return Files.createTempDirectory(path); + } + + private static void createFileSystemTextFile(Path dirPath, String name, String content) throws IOException { + Path createdFile = Files.createTempFile(dirPath, name, null); + Files.write(createdFile, content.getBytes()); + } + + private class DeploymentDetails { + + private final ApplicationId appId; + private final ETLStage source; + private final ETLStage sink; + private final ApplicationManager appManager; + + DeploymentDetails(ETLStage source, ETLStage sink, ApplicationId appId, ApplicationManager appManager) { + this.appId = appId; + this.source = source; + this.sink = sink; + this.appManager = appManager; + } + + public ApplicationId getAppId() { + return appId; + } + + public ETLStage getSource() { + return source; + } + + public ETLStage getSink() { + return sink; + } + + public ApplicationManager getAppManager() { + return appManager; + } + } +} diff --git a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java new file mode 100644 index 000000000..adb10e5b1 --- /dev/null +++ b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.app.etl.google.drive; + +import org.junit.BeforeClass; + +import io.cdap.cdap.app.etl.ETLTestBase; + +/** + * An abstract class used for running integration tests with Google OAuth2 user account credentials. + */ +public abstract class UserCredentialsTestBase extends ETLTestBase { + private static String clientId; + private static String clientSecret; + private static String refreshToken; + + @BeforeClass + public static void userCredentialsSetup() { + clientId = System.getProperty("google.application.clientId"); + clientSecret = System.getProperty("google.application.clientSecret"); + refreshToken = System.getProperty("google.application.refreshToken"); + if (clientId == null || clientSecret== null || refreshToken == null) { + throw new IllegalArgumentException("Invalid user credential parameters"); + } + } + + public static String getClientId() { + return clientId; + } + + public static String getClientSecret() { + return clientSecret; + } + + public static String getRefreshToken() { + return refreshToken; + } +} diff --git a/pom.xml b/pom.xml index 28c285bdc..413aba7a6 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ ${cdap.version} 0.12.0 + v3-rev173-1.25.0 4.12 0.8.2.2 1.1.1.7 @@ -99,6 +100,11 @@ google-cloud-bigquery 1.36.0 + + com.google.apis + google-api-services-drive + ${drive-api.version} + com.google.guava guava From 1216144d39a82ee501428e0761b5c30d1fcc74ca Mon Sep 17 00:00:00 2001 From: dvitiiuk Date: Fri, 29 Nov 2019 14:47:15 +0200 Subject: [PATCH 2/2] PLUGIN-74. Integration tests for Google Sheets plugins (SaaS source). --- .../cdap/app/etl/google/GoogleBaseTest.java | 213 ++++ .../{drive => }/UserCredentialsTestBase.java | 2 +- .../app/etl/google/drive/GoogleDriveTest.java | 194 +--- .../etl/google/sheets/GoogleSheetsTest.java | 974 ++++++++++++++++++ pom.xml | 6 + 5 files changed, 1204 insertions(+), 185 deletions(-) create mode 100644 integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/GoogleBaseTest.java rename integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/{drive => }/UserCredentialsTestBase.java (97%) create mode 100644 integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/sheets/GoogleSheetsTest.java diff --git a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/GoogleBaseTest.java b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/GoogleBaseTest.java new file mode 100644 index 000000000..b5653e92e --- /dev/null +++ b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/GoogleBaseTest.java @@ -0,0 +1,213 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.app.etl.google; + +import com.google.api.client.http.ByteArrayContent; +import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.File; +import com.google.api.services.drive.model.FileList; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.artifact.ArtifactScope; +import io.cdap.cdap.common.ArtifactNotFoundException; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.utils.Tasks; +import io.cdap.cdap.datapipeline.SmartWorkflow; +import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.proto.ArtifactSelectorConfig; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.ProgramRunStatus; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.artifact.PluginSummary; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.WorkflowManager; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Base class for tests, that contain all general methods. + */ +public class GoogleBaseTest extends UserCredentialsTestBase { + protected static final String FILE_SOURCE_STAGE_NAME = "FileSource"; + protected static final String FILE_SINK_STAGE_NAME = "FileSink"; + + protected void checkPluginExists(String pluginName, String pluginType, String artifact) { + Preconditions.checkNotNull(pluginName); + Preconditions.checkNotNull(pluginType); + Preconditions.checkNotNull(artifact); + + try { + Tasks.waitFor(true, () -> { + try { + final ArtifactId artifactId = TEST_NAMESPACE.artifact(artifact, version); + List plugins = + artifactClient.getPluginSummaries(artifactId, pluginType, ArtifactScope.SYSTEM); + return plugins.stream().anyMatch(pluginSummary -> pluginName.equals(pluginSummary.getName())); + } catch (ArtifactNotFoundException e) { + // happens if the relevant artifact(s) were not added yet + return false; + } + }, 5, TimeUnit.MINUTES, 3, TimeUnit.SECONDS); + } catch (Exception e) { + Throwables.throwIfUnchecked(e); + throw new RuntimeException(e); + } + } + + protected static void startWorkFlow(ApplicationManager appManager, ProgramRunStatus expectedStatus) throws Exception { + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.startAndWaitForRun(expectedStatus, 5, TimeUnit.MINUTES); + } + + protected void checkRowsNumber(DeploymentDetails deploymentDetails, int expectedCount) throws Exception { + ApplicationId appId = deploymentDetails.getAppId(); + Map tags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, appId.getNamespace(), + Constants.Metrics.Tag.APP, appId.getEntityName()); + checkMetric(tags, "user." + deploymentDetails.getSource().getName() + ".records.out", + expectedCount, 10); + checkMetric(tags, "user." + deploymentDetails.getSink().getName() + ".records.in", + expectedCount, 10); + } + + protected Path createFileSystemFolder(String path) throws IOException { + return Files.createTempDirectory(path); + } + + protected static String createFolder(Drive service, String folderName) throws IOException { + File fileMetadata = new File(); + fileMetadata.setName(folderName); + fileMetadata.setMimeType("application/vnd.google-apps.folder"); + + File createdFolder = service.files().create(fileMetadata).setFields("id").execute(); + return createdFolder.getId(); + } + + protected static String createFile(Drive service, byte[] content, String name, String mime, String subMime, + String folderId) throws IOException { + File fileToWrite = new File(); + fileToWrite.setName(name); + fileToWrite.setParents(Collections.singletonList(folderId)); + fileToWrite.setMimeType(mime); + ByteArrayContent fileContent = new ByteArrayContent(subMime, content); + + File file = service.files().create(fileToWrite, fileContent) + .setFields("id, parents, mimeType") + .execute(); + return file.getId(); + } + + protected static void removeFile(Drive service, String fileId) throws IOException { + service.files().delete(fileId).execute(); + } + + protected static List getFiles(Drive drive, String parentFolderId) { + try { + List files = new ArrayList<>(); + String nextToken = ""; + Drive.Files.List request = drive.files().list() + .setQ(String.format("'%s' in parents", parentFolderId)) + .setFields("nextPageToken, files(id, name, size, mimeType)"); + while (nextToken != null) { + FileList result = request.execute(); + files.addAll(result.getFiles()); + nextToken = result.getNextPageToken(); + request.setPageToken(nextToken); + } + return files; + } catch (IOException e) { + throw new RuntimeException("Issue during retrieving summary for files.", e); + } + } + + protected static void createFileSystemTextFile(Path dirPath, String name, String content) throws IOException { + Path createdFile = Files.createTempFile(dirPath, name, null); + Files.write(createdFile, content.getBytes()); + } + + protected DeploymentDetails deployApplication(Map sourceProperties, + Map sinkProperties, + String sourceStageName, String sinkStageName, + String sourcePluginName, String sinkPluginName, + ArtifactSelectorConfig sourceArtifact, + ArtifactSelectorConfig sinkArtifact, + String applicationName) throws Exception { + ETLStage source = new ETLStage(sourceStageName, + new ETLPlugin(sourcePluginName, + BatchSource.PLUGIN_TYPE, + sourceProperties, + sourceArtifact)); + ETLStage sink = new ETLStage(sinkStageName, new ETLPlugin(sinkPluginName, + BatchSink.PLUGIN_TYPE, + sinkProperties, + sinkArtifact)); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(source) + .addStage(sink) + .addConnection(source.getName(), sink.getName()) + .build(); + + AppRequest appRequest = getBatchAppRequestV2(etlConfig); + ApplicationId appId = TEST_NAMESPACE.app(applicationName); + ApplicationManager applicationManager = deployApplication(appId, appRequest); + return new DeploymentDetails(source, sink, appId, applicationManager); + } + + protected static class DeploymentDetails { + + private final ApplicationId appId; + private final ETLStage source; + private final ETLStage sink; + private final ApplicationManager appManager; + + public DeploymentDetails(ETLStage source, ETLStage sink, ApplicationId appId, ApplicationManager appManager) { + this.appId = appId; + this.source = source; + this.sink = sink; + this.appManager = appManager; + } + + public ApplicationId getAppId() { + return appId; + } + + public ETLStage getSource() { + return source; + } + + public ETLStage getSink() { + return sink; + } + + public ApplicationManager getAppManager() { + return appManager; + } + } +} diff --git a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/UserCredentialsTestBase.java similarity index 97% rename from integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java rename to integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/UserCredentialsTestBase.java index adb10e5b1..e385e5304 100644 --- a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/UserCredentialsTestBase.java +++ b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/UserCredentialsTestBase.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.cdap.app.etl.google.drive; +package io.cdap.cdap.app.etl.google; import org.junit.BeforeClass; diff --git a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java index 5c442a8a9..1d06ed043 100644 --- a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java +++ b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/drive/GoogleDriveTest.java @@ -18,40 +18,22 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.drive.Drive; import com.google.api.services.drive.model.File; -import com.google.api.services.drive.model.FileList; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import io.cdap.cdap.api.artifact.ArtifactScope; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.common.ArtifactNotFoundException; -import io.cdap.cdap.common.conf.Constants; -import io.cdap.cdap.common.utils.Tasks; -import io.cdap.cdap.datapipeline.SmartWorkflow; +import io.cdap.cdap.app.etl.google.GoogleBaseTest; import io.cdap.cdap.etl.api.batch.BatchSink; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.proto.ArtifactSelectorConfig; -import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; -import io.cdap.cdap.etl.proto.v2.ETLPlugin; -import io.cdap.cdap.etl.proto.v2.ETLStage; import io.cdap.cdap.proto.ProgramRunStatus; -import io.cdap.cdap.proto.artifact.AppRequest; -import io.cdap.cdap.proto.artifact.PluginSummary; -import io.cdap.cdap.proto.id.ApplicationId; -import io.cdap.cdap.proto.id.ArtifactId; -import io.cdap.cdap.test.ApplicationManager; -import io.cdap.cdap.test.WorkflowManager; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; @@ -69,20 +51,18 @@ import java.nio.file.Path; import java.security.GeneralSecurityException; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** * Tests reading to and writing from Google Drive within a sandbox cluster. */ -public class GoogleDriveTest extends UserCredentialsTestBase { +public class GoogleDriveTest extends GoogleBaseTest { @Rule public TestName testName = new TestName(); @@ -107,8 +87,6 @@ public class GoogleDriveTest extends UserCredentialsTestBase { private static final String TEST_SHEET_FILE_CONTENT = "a,b,c\r\n,d,e"; private static final String DRIVE_SOURCE_STAGE_NAME = "GoogleDriveSource"; private static final String DRIVE_SINK_STAGE_NAME = "GoogleDriveSink"; - private static final String FILE_SOURCE_STAGE_NAME = "FileSource"; - private static final String FILE_SINK_STAGE_NAME = "FileSink"; public static final String TMP_FOLDER_NAME = "googleDriveTestFolder"; private static Drive service; @@ -187,7 +165,7 @@ public void testBinaryOnly() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 1); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(1, destFiles.size()); File textFile = destFiles.get(0); @@ -218,7 +196,7 @@ public void testDocFileOnly() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 1); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(1, destFiles.size()); File docFile = destFiles.get(0); @@ -251,7 +229,7 @@ public void testAllFileTypes() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 3); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(3, destFiles.size()); destFiles.forEach(file -> { @@ -287,7 +265,7 @@ public void testAllFileTypesNamed() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 3); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(3, destFiles.size()); destFiles.forEach(file -> { @@ -343,7 +321,7 @@ public void testAllFileTypesNamedAndMimed() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 3); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(3, destFiles.size()); destFiles.forEach(file -> { @@ -411,7 +389,7 @@ public void testPartitionSize() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 4); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(4, destFiles.size()); // flags to check partitioning work @@ -478,7 +456,7 @@ public void testWithFileSource() throws Exception { // check number of rows in and out checkRowsNumber(deploymentDetails, 1); - List destFiles = getFiles(sinkFolderId); + List destFiles = getFiles(service, sinkFolderId); Assert.assertEquals(1, destFiles.size()); File textFile = destFiles.get(0); @@ -661,44 +639,6 @@ private Map getFileSinkMinimalDefaultConfigs() { }; } - protected void startWorkFlow(ApplicationManager appManager, ProgramRunStatus expectedStatus) throws Exception { - WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); - workflowManager.startAndWaitForRun(expectedStatus, 5, TimeUnit.MINUTES); - } - - private void checkRowsNumber(DeploymentDetails deploymentDetails, int expectedCount) throws Exception { - ApplicationId appId = deploymentDetails.getAppId(); - Map tags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, appId.getNamespace(), - Constants.Metrics.Tag.APP, appId.getEntityName()); - checkMetric(tags, "user." + deploymentDetails.getSource().getName() + ".records.out", - expectedCount, 10); - checkMetric(tags, "user." + deploymentDetails.getSink().getName() + ".records.in", - expectedCount, 10); - } - - private void checkPluginExists(String pluginName, String pluginType, String artifact) { - Preconditions.checkNotNull(pluginName); - Preconditions.checkNotNull(pluginType); - Preconditions.checkNotNull(artifact); - - try { - Tasks.waitFor(true, () -> { - try { - final ArtifactId artifactId = TEST_NAMESPACE.artifact(artifact, version); - List plugins = - artifactClient.getPluginSummaries(artifactId, pluginType, ArtifactScope.SYSTEM); - return plugins.stream().anyMatch(pluginSummary -> pluginName.equals(pluginSummary.getName())); - } catch (ArtifactNotFoundException e) { - // happens if the relevant artifact(s) were not added yet - return false; - } - }, 5, TimeUnit.MINUTES, 3, TimeUnit.SECONDS); - } catch (Exception e) { - Throwables.throwIfUnchecked(e); - throw new RuntimeException(e); - } - } - private DeploymentDetails deployGoogleDriveApplication(Map sourceProperties, Map sinkProperties, String applicationName) throws Exception { @@ -708,125 +648,11 @@ private DeploymentDetails deployGoogleDriveApplication(Map sourc GOOGLE_DRIVE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, applicationName); } - private DeploymentDetails deployApplication(Map sourceProperties, Map sinkProperties, - String sourceStageName, String sinkStageName, - String sourcePluginName, String sinkPluginName, - ArtifactSelectorConfig sourceArtifact, - ArtifactSelectorConfig sinkArtifact, String applicationName) - throws Exception { - ETLStage source = new ETLStage(sourceStageName, - new ETLPlugin(sourcePluginName, - BatchSource.PLUGIN_TYPE, - sourceProperties, - sourceArtifact)); - ETLStage sink = new ETLStage(sinkStageName, new ETLPlugin(sinkPluginName, - BatchSink.PLUGIN_TYPE, - sinkProperties, - sinkArtifact)); - - ETLBatchConfig etlConfig = ETLBatchConfig.builder() - .addStage(source) - .addStage(sink) - .addConnection(source.getName(), sink.getName()) - .build(); - - AppRequest appRequest = getBatchAppRequestV2(etlConfig); - ApplicationId appId = TEST_NAMESPACE.app(applicationName); - ApplicationManager applicationManager = deployApplication(appId, appRequest); - return new DeploymentDetails(source, sink, appId, applicationManager); - } - - private static String createFile(Drive service, byte[] content, String name, String mime, String subMime, - String folderId) throws IOException { - File fileToWrite = new File(); - fileToWrite.setName(name); - fileToWrite.setParents(Collections.singletonList(folderId)); - fileToWrite.setMimeType(mime); - ByteArrayContent fileContent = new ByteArrayContent(subMime, content); - - File file = service.files().create(fileToWrite, fileContent) - .setFields("id, parents, mimeType") - .execute(); - return file.getId(); - } - - private static String createFolder(Drive service, String folderName) throws IOException { - File fileMetadata = new File(); - fileMetadata.setName(folderName); - fileMetadata.setMimeType("application/vnd.google-apps.folder"); - - File createdFolder = service.files().create(fileMetadata).setFields("id").execute(); - return createdFolder.getId(); - } - - private static void removeFile(Drive service, String fileId) throws IOException { - service.files().delete(fileId).execute(); - } - - private static List getFiles(String parentFolderId) { - try { - List files = new ArrayList<>(); - String nextToken = ""; - Drive.Files.List request = service.files().list() - .setQ(String.format("'%s' in parents", parentFolderId)) - .setFields("nextPageToken, files(id, name, size, mimeType)"); - while (nextToken != null) { - FileList result = request.execute(); - files.addAll(result.getFiles()); - nextToken = result.getNextPageToken(); - request.setPageToken(nextToken); - } - return files; - } catch (IOException e) { - throw new RuntimeException("Issue during retrieving summary for files.", e); - } - } - - private static String getFileContent(String fileId) throws IOException { + private String getFileContent(String fileId) throws IOException { OutputStream outputStream = new ByteArrayOutputStream(); Drive.Files.Get get = service.files().get(fileId); get.executeMediaAndDownloadTo(outputStream); return ((ByteArrayOutputStream) outputStream).toString(); } - - private static Path createFileSystemFolder(String path) throws IOException { - return Files.createTempDirectory(path); - } - - private static void createFileSystemTextFile(Path dirPath, String name, String content) throws IOException { - Path createdFile = Files.createTempFile(dirPath, name, null); - Files.write(createdFile, content.getBytes()); - } - - private class DeploymentDetails { - - private final ApplicationId appId; - private final ETLStage source; - private final ETLStage sink; - private final ApplicationManager appManager; - - DeploymentDetails(ETLStage source, ETLStage sink, ApplicationId appId, ApplicationManager appManager) { - this.appId = appId; - this.source = source; - this.sink = sink; - this.appManager = appManager; - } - - public ApplicationId getAppId() { - return appId; - } - - public ETLStage getSource() { - return source; - } - - public ETLStage getSink() { - return sink; - } - - public ApplicationManager getAppManager() { - return appManager; - } - } } diff --git a/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/sheets/GoogleSheetsTest.java b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/sheets/GoogleSheetsTest.java new file mode 100644 index 000000000..20d9213c7 --- /dev/null +++ b/integration-test-remote/src/test/java/io/cdap/cdap/app/etl/google/sheets/GoogleSheetsTest.java @@ -0,0 +1,974 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.app.etl.google.sheets; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.File; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.model.AppendCellsRequest; +import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetRequest; +import com.google.api.services.sheets.v4.model.CellData; +import com.google.api.services.sheets.v4.model.CellFormat; +import com.google.api.services.sheets.v4.model.ExtendedValue; +import com.google.api.services.sheets.v4.model.GridRange; +import com.google.api.services.sheets.v4.model.NumberFormat; +import com.google.api.services.sheets.v4.model.Request; +import com.google.api.services.sheets.v4.model.RowData; +import com.google.api.services.sheets.v4.model.Sheet; +import com.google.api.services.sheets.v4.model.Spreadsheet; +import com.google.common.collect.ImmutableList; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.app.etl.google.GoogleBaseTest; +import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.proto.ArtifactSelectorConfig; +import io.cdap.cdap.proto.ProgramRunStatus; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Tests reading to and writing from Google Sheets within a sandbox cluster. + */ +public class GoogleSheetsTest extends GoogleBaseTest { + + @Rule + public TestName testName = new TestName(); + + protected static final ArtifactSelectorConfig GOOGLE_DRIVE_ARTIFACT = + new ArtifactSelectorConfig("SYSTEM", "google-drive-plugins", "[0.0.0, 100.0.0)"); + protected static final ArtifactSelectorConfig FILE_ARTIFACT = + new ArtifactSelectorConfig("SYSTEM", "core-plugins", "[0.0.0, 100.0.0)"); + private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); + private static final String GOOGLE_SHEETS_PLUGIN_NAME = "GoogleSheets"; + private static final String FILE_PLUGIN_NAME = "File"; + private static final String TEST_SHEET_FILE_NAME = "sheetFile"; + private static final String TEXT_CSV_MIME = "text/csv"; + private static final String DEFAULT_SPREADSHEET_NAME = "sp0"; + private static final String DEFAULT_SHEET_NAME = "s0"; + private static final String TEST_TEXT_FILE_NAME = "textFile"; + private static final String SHEETS_SOURCE_STAGE_NAME = "GoogleSheetsSource"; + private static final String SHEETS_SINK_STAGE_NAME = "GoogleSheetsSink"; + + public static final String TMP_FOLDER_NAME = "googleSheetsTestFolder"; + + private static Drive driveService; + private static Sheets sheetsService; + private String sourceFolderId; + private String sinkFolderId; + private String testSourceFileId; + private Path tmpFolder; + + @BeforeClass + public static void setupDrive() throws GeneralSecurityException, IOException { + final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + + GoogleCredential credential = new GoogleCredential.Builder() + .setTransport(HTTP_TRANSPORT) + .setJsonFactory(JSON_FACTORY) + .setClientSecrets(getClientId(), + getClientSecret()) + .build(); + credential.setRefreshToken(getRefreshToken()); + + driveService = new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).build(); + sheetsService = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).build(); + } + + @Before + public void testClassSetup() throws IOException { + ImmutableList.of(ImmutableList.of(GOOGLE_SHEETS_PLUGIN_NAME, BatchSource.PLUGIN_TYPE, "cdap-data-pipeline"), + ImmutableList.of(GOOGLE_SHEETS_PLUGIN_NAME, BatchSink.PLUGIN_TYPE, "cdap-data-pipeline")) + .forEach((pluginInfo) -> checkPluginExists(pluginInfo.get(0), pluginInfo.get(1), pluginInfo.get(2))); + + String sourceFolderName = RandomStringUtils.randomAlphanumeric(16); + String sinkFolderName = RandomStringUtils.randomAlphanumeric(16); + + sourceFolderId = createFolder(driveService, sourceFolderName); + sinkFolderId = createFolder(driveService, sinkFolderName); + + testSourceFileId = createFile(driveService, "".getBytes(), TEST_SHEET_FILE_NAME, + "application/vnd.google-apps.spreadsheet", TEXT_CSV_MIME, sourceFolderId); + tmpFolder = createFileSystemFolder(TMP_FOLDER_NAME); + } + + + @After + public void removeFolders() throws IOException { + if (testSourceFileId != null) { + removeFile(driveService, testSourceFileId); + } + if (sourceFolderId != null) { + removeFile(driveService, sourceFolderId); + } + if (sinkFolderId != null) { + removeFile(driveService, sinkFolderId); + } + + Files.walk(tmpFolder) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(java.io.File::delete); + } + + @Test + public void testSourceFileAllRecords() throws Exception { + final int recordsToRead = 15; + final int populatedRows = 10; + Map sourceProps = new HashMap() { + { + putAll(getSheetsSourceMinimalDefaultConfigs()); + put("lastDataRow", String.valueOf(recordsToRead)); + put("skipEmptyData", "false"); + } + }; + Map sinkProps = getFileSinkMinimalDefaultConfigs(); + + // populate the sheet with simple rows + populateSpreadSheetWithSimpleRows(sheetsService, testSourceFileId, generateSimpleRows(populatedRows, 5)); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, SHEETS_SOURCE_STAGE_NAME, FILE_SINK_STAGE_NAME, + GOOGLE_SHEETS_PLUGIN_NAME, FILE_PLUGIN_NAME, GOOGLE_DRIVE_ARTIFACT, FILE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, recordsToRead); + + Assert.assertTrue(Files.isDirectory(tmpFolder)); + + List allDeploysResults = Files.list(tmpFolder).collect(Collectors.toList()); + Assert.assertEquals(1, allDeploysResults.size()); + + Path deployResult = allDeploysResults.get(0); + Assert.assertTrue(Files.isDirectory(deployResult)); + Assert.assertEquals(1, + Files.list(deployResult).filter(p -> p.getFileName().toString().equals("_SUCCESS")).count()); + + List destFiles = + Files.list(deployResult).filter(p -> p.getFileName().toString().startsWith("part")).collect(Collectors.toList()); + Assert.assertEquals(1, destFiles.size()); + + Path destFile = destFiles.get(0); + List fileLines = null; + try { + fileLines = Files.readAllLines(destFile); + } catch (IOException e) { + Assert.fail(String.format("Exception during reading file '%s': %s", destFile.toString(), e.getMessage())); + } + + Assert.assertEquals(recordsToRead, fileLines.size()); + Assert.assertEquals(populatedRows, getNonNullRowsCount(fileLines)); + } + + @Test + public void testSourceFileNonEmptyRecords() throws Exception { + final int recordsToRead = 15; + final int populatedRows = 10; + Map sourceProps = new HashMap() { + { + putAll(getSheetsSourceMinimalDefaultConfigs()); + put("lastDataRow", String.valueOf(recordsToRead)); + } + }; + Map sinkProps = getFileSinkMinimalDefaultConfigs(); + + // populate the sheet with simple rows + populateSpreadSheetWithSimpleRows(sheetsService, testSourceFileId, generateSimpleRows(populatedRows, 5)); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, SHEETS_SOURCE_STAGE_NAME, FILE_SINK_STAGE_NAME, + GOOGLE_SHEETS_PLUGIN_NAME, FILE_PLUGIN_NAME, GOOGLE_DRIVE_ARTIFACT, FILE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, populatedRows); + + Assert.assertTrue(Files.isDirectory(tmpFolder)); + + List allDeploysResults = Files.list(tmpFolder).collect(Collectors.toList()); + Assert.assertEquals(1, allDeploysResults.size()); + + Path deployResult = allDeploysResults.get(0); + Assert.assertTrue(Files.isDirectory(deployResult)); + Assert.assertEquals(1, + Files.list(deployResult).filter(p -> p.getFileName().toString().equals("_SUCCESS")).count()); + + List destFiles = + Files.list(deployResult).filter(p -> p.getFileName().toString().startsWith("part")).collect(Collectors.toList()); + Assert.assertEquals(1, destFiles.size()); + + Path destFile = destFiles.get(0); + List fileLines = null; + try { + fileLines = Files.readAllLines(destFile); + } catch (IOException e) { + Assert.fail(String.format("Exception during reading file '%s': %s", destFile.toString(), e.getMessage())); + } + + Assert.assertEquals(populatedRows, fileLines.size()); + Assert.assertEquals(populatedRows, getNonNullRowsCount(fileLines)); + } + + @Test + public void testSourceSinkSingleFile() throws Exception { + final int recordsPerName = 2; + final int columnsNumber = 5; + final List names = Arrays.asList("name1", "name2"); + Map sourceProps = new HashMap() { + { + putAll(getSheetsSourceMinimalDefaultConfigs()); + put("lastDataRow", String.valueOf(recordsPerName * names.size())); + } + }; + Map sinkProps = getSheetsSinkMinimalDefaultConfigs(); + + // populate the sheet with simple rows + populateSpreadSheetWithSimpleRows(sheetsService, testSourceFileId, + generateRowsWithNames(recordsPerName, columnsNumber, names)); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, SHEETS_SOURCE_STAGE_NAME, SHEETS_SINK_STAGE_NAME, + GOOGLE_SHEETS_PLUGIN_NAME, GOOGLE_SHEETS_PLUGIN_NAME, + GOOGLE_DRIVE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, recordsPerName * names.size()); + + List resultFiles = getFiles(driveService, sinkFolderId); + Assert.assertEquals(1, resultFiles.size()); + + File file = resultFiles.get(0); + Assert.assertEquals(DEFAULT_SPREADSHEET_NAME, file.getName()); + + String fileId = file.getId(); + Spreadsheet spreadsheet = getSpreadsheet(fileId); + + Assert.assertEquals(DEFAULT_SPREADSHEET_NAME, spreadsheet.getProperties().getTitle()); + Assert.assertNotNull(spreadsheet.getSheets()); + Assert.assertEquals(1, spreadsheet.getSheets().size()); + Assert.assertEquals(DEFAULT_SHEET_NAME, spreadsheet.getSheets().get(0).getProperties().getTitle()); + Assert.assertEquals(recordsPerName * names.size(), + spreadsheet.getSheets().get(0).getData().get(0).getRowData().size()); + for (RowData rowData : spreadsheet.getSheets().get(0).getData().get(0).getRowData()) { + Assert.assertEquals(columnsNumber, rowData.getValues().size()); + } + } + + @Test + public void testSourceSinkSeparateFiles() throws Exception { + final int recordsPerName = 2; + final int columnsNumber = 5; + final List names = Arrays.asList("name1", "name2"); + Map sourceProps = new HashMap() { + { + putAll(getSheetsSourceMinimalDefaultConfigs()); + put("lastDataRow", String.valueOf(recordsPerName * names.size())); + } + }; + Map sinkProps = new HashMap() { + { + putAll(getSheetsSinkMinimalDefaultConfigs()); + + // set first column value as file name + put("schemaSpreadsheetNameFieldName", "A"); + } + }; + + // populate the sheet with simple rows + populateSpreadSheetWithSimpleRows(sheetsService, testSourceFileId, + generateRowsWithNames(recordsPerName, columnsNumber, names)); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, SHEETS_SOURCE_STAGE_NAME, SHEETS_SINK_STAGE_NAME, + GOOGLE_SHEETS_PLUGIN_NAME, GOOGLE_SHEETS_PLUGIN_NAME, + GOOGLE_DRIVE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, recordsPerName * names.size()); + + List resultFiles = getFiles(driveService, sinkFolderId); + Assert.assertEquals(2, resultFiles.size()); + + for (File file : resultFiles) { + Assert.assertTrue(names.contains(file.getName())); + + String fileId = file.getId(); + Spreadsheet spreadsheet = getSpreadsheet(fileId); + + Assert.assertTrue(names.contains(spreadsheet.getProperties().getTitle())); + Assert.assertNotNull(spreadsheet.getSheets()); + Assert.assertEquals(1, spreadsheet.getSheets().size()); + Assert.assertEquals(DEFAULT_SHEET_NAME, spreadsheet.getSheets().get(0).getProperties().getTitle()); + Assert.assertEquals(recordsPerName, + spreadsheet.getSheets().get(0).getData().get(0).getRowData().size()); + for (RowData rowData : spreadsheet.getSheets().get(0).getData().get(0).getRowData()) { + Assert.assertEquals(columnsNumber - 1, rowData.getValues().size()); + } + } + } + + @Test + public void testFileSinkMergeCells() throws Exception { + // create test file + createFileSystemTextFile(tmpFolder, TEST_TEXT_FILE_NAME, + "{\"name\": \"test\",\"array\":[true,false]}"); + + Set schemaFields = new HashSet<>(); + schemaFields.add(Schema.Field.of("name", Schema.of(Schema.Type.STRING))); + schemaFields.add(Schema.Field.of("array", Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN)))); + Schema fileSchema = Schema.recordOf( + "blob", + schemaFields); + Map sourceProps = new HashMap() { + { + putAll(getFileSourceMinimalDefaultConfigs()); + put("schema", fileSchema.toString()); + } + }; + + Map sinkProps = new HashMap() { + { + putAll(getSheetsSinkMinimalDefaultConfigs()); + put("mergeDataCells", "true"); + } + }; + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, FILE_SOURCE_STAGE_NAME, SHEETS_SINK_STAGE_NAME, + FILE_PLUGIN_NAME, GOOGLE_SHEETS_PLUGIN_NAME, FILE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + List resultFiles = getFiles(driveService, sinkFolderId); + Assert.assertEquals(1, resultFiles.size()); + + File file = resultFiles.get(0); + String fileId = file.getId(); + Spreadsheet spreadsheet = getSpreadsheet(fileId); + + // check spreadSheet and sheet names + Assert.assertEquals(DEFAULT_SPREADSHEET_NAME, spreadsheet.getProperties().getTitle()); + Assert.assertNotNull(spreadsheet.getSheets()); + Assert.assertEquals(1, spreadsheet.getSheets().size()); + Sheet sheet = spreadsheet.getSheets().get(0); + Assert.assertEquals(DEFAULT_SHEET_NAME, sheet.getProperties().getTitle()); + + // check merges + Assert.assertNotNull(sheet.getMerges()); + Assert.assertEquals(1, sheet.getMerges().size()); + + // check data cells + List rows = sheet.getData().get(0).getRowData(); + Assert.assertEquals(2, rows.size()); + if (rows.get(0).getValues().get(0).getUserEnteredValue().getStringValue() != null) { + Assert.assertEquals("test", rows.get(0).getValues().get(0).getUserEnteredValue().getStringValue()); + Assert.assertEquals(true, rows.get(0).getValues().get(1).getUserEnteredValue().getBoolValue()); + Assert.assertEquals(false, rows.get(1).getValues().get(1).getUserEnteredValue().getBoolValue()); + + Assert.assertEquals(new GridRange().setStartRowIndex(0).setEndRowIndex(2) + .setStartColumnIndex(0).setEndColumnIndex(1).setSheetId(sheet.getProperties().getSheetId()), + sheet.getMerges().get(0)); + } else if (rows.get(0).getValues().get(0).getUserEnteredValue().getBoolValue() != null) { + Assert.assertEquals("test", rows.get(0).getValues().get(1).getUserEnteredValue().getStringValue()); + Assert.assertEquals(true, rows.get(0).getValues().get(0).getUserEnteredValue().getBoolValue()); + Assert.assertEquals(false, rows.get(1).getValues().get(0).getUserEnteredValue().getBoolValue()); + + Assert.assertEquals(new GridRange().setStartRowIndex(0).setEndRowIndex(2) + .setStartColumnIndex(1).setEndColumnIndex(2).setSheetId(sheet.getProperties().getSheetId()), + sheet.getMerges().get(0)); + } else { + Assert.fail("Invalid value for first cell of the first row in the result spreadSheet."); + } + } + + @Test + public void testFileSinkFlattenCells() throws Exception { + // create test file + createFileSystemTextFile(tmpFolder, TEST_TEXT_FILE_NAME, + "{\"name\": \"test\",\"array\":[true,false]}"); + + Set schemaFields = new HashSet<>(); + schemaFields.add(Schema.Field.of("name", Schema.of(Schema.Type.STRING))); + schemaFields.add(Schema.Field.of("array", Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN)))); + Schema fileSchema = Schema.recordOf("blob", schemaFields); + Map sourceProps = new HashMap() { + { + putAll(getFileSourceMinimalDefaultConfigs()); + put("schema", fileSchema.toString()); + } + }; + + Map sinkProps = getSheetsSinkMinimalDefaultConfigs(); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, FILE_SOURCE_STAGE_NAME, SHEETS_SINK_STAGE_NAME, + FILE_PLUGIN_NAME, GOOGLE_SHEETS_PLUGIN_NAME, FILE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + List resultFiles = getFiles(driveService, sinkFolderId); + Assert.assertEquals(1, resultFiles.size()); + + File file = resultFiles.get(0); + String fileId = file.getId(); + Spreadsheet spreadsheet = getSpreadsheet(fileId); + + // check spreadSheet and sheet names + Assert.assertEquals(DEFAULT_SPREADSHEET_NAME, spreadsheet.getProperties().getTitle()); + Assert.assertNotNull(spreadsheet.getSheets()); + Assert.assertEquals(1, spreadsheet.getSheets().size()); + Sheet sheet = spreadsheet.getSheets().get(0); + Assert.assertEquals(DEFAULT_SHEET_NAME, sheet.getProperties().getTitle()); + + // check cell values + List rows = sheet.getData().get(0).getRowData(); + Assert.assertEquals(2, rows.size()); + + ExtendedValue firstValueInFirstRow = rows.get(0).getValues().get(0).getUserEnteredValue(); + if (firstValueInFirstRow.getStringValue() != null) { + Assert.assertEquals("test", rows.get(0).getValues().get(0).getUserEnteredValue().getStringValue()); + Assert.assertEquals("test", rows.get(1).getValues().get(0).getUserEnteredValue().getStringValue()); + Assert.assertEquals(true, rows.get(0).getValues().get(1).getUserEnteredValue().getBoolValue()); + Assert.assertEquals(false, rows.get(1).getValues().get(1).getUserEnteredValue().getBoolValue()); + } else if (rows.get(0).getValues().get(0).getUserEnteredValue().getBoolValue() != null) { + Assert.assertEquals("test", rows.get(0).getValues().get(1).getUserEnteredValue().getStringValue()); + Assert.assertEquals("test", rows.get(1).getValues().get(1).getUserEnteredValue().getStringValue()); + Assert.assertEquals(true, rows.get(0).getValues().get(0).getUserEnteredValue().getBoolValue()); + Assert.assertEquals(false, rows.get(1).getValues().get(0).getUserEnteredValue().getBoolValue()); + } else { + Assert.fail(String.format("Invalid value '%s' for first cell of the first row in the result spreadSheet.", + firstValueInFirstRow)); + } + + // check merges + Assert.assertNull(sheet.getMerges()); + } + + @Test + public void testFileSinkRecords() throws Exception { + // create test file + createFileSystemTextFile(tmpFolder, TEST_TEXT_FILE_NAME, + "{\"record\":{\"field0\":true,\"field1\":\"test\"}}"); + + Set schemaFields = new HashSet<>(); + schemaFields.add(Schema.Field.of("record", Schema.recordOf("record", Arrays.asList( + Schema.Field.of("field0", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("field1", Schema.of(Schema.Type.STRING)) + )))); + Schema fileSchema = Schema.recordOf("blob", schemaFields); + Map sourceProps = new HashMap() { + { + putAll(getFileSourceMinimalDefaultConfigs()); + put("schema", fileSchema.toString()); + } + }; + + Map sinkProps = new HashMap() { + { + putAll(getSheetsSinkMinimalDefaultConfigs()); + put("writeSchema", "true"); + } + }; + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, FILE_SOURCE_STAGE_NAME, SHEETS_SINK_STAGE_NAME, + FILE_PLUGIN_NAME, GOOGLE_SHEETS_PLUGIN_NAME, FILE_ARTIFACT, GOOGLE_DRIVE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + List resultFiles = getFiles(driveService, sinkFolderId); + Assert.assertEquals(1, resultFiles.size()); + + File file = resultFiles.get(0); + String fileId = file.getId(); + Spreadsheet spreadsheet = getSpreadsheet(fileId); + + // check spreadSheet and sheet names + Assert.assertEquals(DEFAULT_SPREADSHEET_NAME, spreadsheet.getProperties().getTitle()); + Assert.assertNotNull(spreadsheet.getSheets()); + Assert.assertEquals(1, spreadsheet.getSheets().size()); + Sheet sheet = spreadsheet.getSheets().get(0); + Assert.assertEquals(DEFAULT_SHEET_NAME, sheet.getProperties().getTitle()); + + // check cell values + List rows = sheet.getData().get(0).getRowData(); + // two rows for header and single for data + Assert.assertEquals(3, rows.size()); + Assert.assertEquals("record", rows.get(0).getValues().get(0).getUserEnteredValue().getStringValue()); + Assert.assertNotNull(sheet.getMerges()); + Assert.assertEquals(1, sheet.getMerges().size()); + Assert.assertEquals(new GridRange().setStartRowIndex(0).setEndRowIndex(1) + .setStartColumnIndex(0).setEndColumnIndex(2).setSheetId(sheet.getProperties().getSheetId()), + sheet.getMerges().get(0)); + + ExtendedValue firstSubHeaderValue = rows.get(1).getValues().get(0).getUserEnteredValue(); + if ("field0".equals(firstSubHeaderValue.getStringValue())) { + Assert.assertEquals("field1", rows.get(1).getValues().get(1).getUserEnteredValue().getStringValue()); + Assert.assertEquals("test", rows.get(2).getValues().get(1).getUserEnteredValue().getStringValue()); + Assert.assertEquals(true, rows.get(2).getValues().get(0).getUserEnteredValue().getBoolValue()); + } else if ("field1".equals(firstSubHeaderValue.getStringValue())) { + Assert.assertEquals("field0", rows.get(1).getValues().get(1).getUserEnteredValue().getStringValue()); + Assert.assertEquals("test", rows.get(2).getValues().get(0).getUserEnteredValue().getStringValue()); + Assert.assertEquals(true, rows.get(2).getValues().get(1).getUserEnteredValue().getBoolValue()); + } else { + Assert.fail(String.format("Invalid value '%s' for first sub-column name in the result spreadSheet.", + firstSubHeaderValue)); + } + } + + @Test + public void testMetadata() throws Exception { + final String metadataKey1 = "metadataKey1"; + final String metadataKey2 = "metadataKey2"; + final String metadataKey3 = "metadataKey3"; + final String metadataValue1 = "metadataValue1"; + final String metadataValue2 = "metadataValue2"; + final String metadataValue3 = "metadataValue3"; + final String header1 = "header1"; + final String header2 = "header2"; + final String data1 = "data1"; + final String data2 = "data2"; + final String metadataFieldName = "customMetadataField"; + Map sourceProps = new HashMap() { + { + putAll(getSheetsSourceMinimalDefaultConfigs()); + put("extractMetadata", String.valueOf(true)); + put("firstHeaderRow", String.valueOf(1)); + put("lastHeaderRow", String.valueOf(2)); + put("firstFooterRow", String.valueOf(5)); + put("lastFooterRow", String.valueOf(5)); + put("metadataFieldName", metadataFieldName); + put("metadataCells", "A1:B1,A2:B2,A5:B5"); + + put("columnNamesSelection", "customRowAsColumns"); + put("customColumnNamesRow", String.valueOf(3)); + put("lastDataRow", String.valueOf(4)); + } + }; + Map sinkProps = getFileSinkMinimalDefaultConfigs(); + + // populate the sheet with simple rows + populateSpreadSheetWithSimpleRows(sheetsService, testSourceFileId, + generateMetadataRows(metadataKey1, metadataValue1, metadataKey2, metadataValue2, + metadataKey3, metadataValue3, header1, header2, + data1, data2)); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, SHEETS_SOURCE_STAGE_NAME, FILE_SINK_STAGE_NAME, + GOOGLE_SHEETS_PLUGIN_NAME, FILE_PLUGIN_NAME, + GOOGLE_DRIVE_ARTIFACT, FILE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + Assert.assertTrue(Files.isDirectory(tmpFolder)); + + List allDeploysResults = Files.list(tmpFolder).collect(Collectors.toList()); + Assert.assertEquals(1, allDeploysResults.size()); + + Path deployResult = allDeploysResults.get(0); + Assert.assertTrue(Files.isDirectory(deployResult)); + Assert.assertEquals(1, + Files.list(deployResult).filter(p -> p.getFileName().toString().equals("_SUCCESS")).count()); + + List destFiles = + Files.list(deployResult).filter(p -> p.getFileName().toString().startsWith("part")).collect(Collectors.toList()); + Assert.assertEquals(1, destFiles.size()); + + Path destFile = destFiles.get(0); + List fileLines = null; + try { + fileLines = Files.readAllLines(destFile); + } catch (IOException e) { + Assert.fail(String.format("Exception during reading file '%s': %s", destFile.toString(), e.getMessage())); + } + + Assert.assertEquals(1, fileLines.size()); + + String resultLine = fileLines.get(0); + + JsonParser jsonParser = new JsonParser(); + JsonElement rootElement = jsonParser.parse(resultLine); + Assert.assertTrue(rootElement.isJsonObject()); + + JsonObject rootObject = rootElement.getAsJsonObject(); + + Assert.assertEquals(3, rootObject.entrySet().size()); + Assert.assertEquals(data1, rootObject.get(header1).getAsString()); + Assert.assertEquals(data2, rootObject.get(header2).getAsString()); + + JsonObject metadataObject = rootObject.get(metadataFieldName).getAsJsonObject(); + + // Image metadata entries: width, height, rotation + Assert.assertEquals(3, metadataObject.entrySet().size()); + Assert.assertEquals(metadataValue1, metadataObject.get(metadataKey1).getAsString()); + Assert.assertEquals(metadataValue2, metadataObject.get(metadataKey2).getAsString()); + Assert.assertEquals(metadataValue3, metadataObject.get(metadataKey3).getAsString()); + } + + @Test + public void testFormatting() throws Exception { + final Double simpleNumberValue = 12d; + final Double numberOfDays = 1d; + final Double partOfDay = 0.5; + final Double percent = 3d; + final Double currency = 54d; + Map sourceProps = new HashMap() { + { + putAll(getSheetsSourceMinimalDefaultConfigs()); + } + }; + Map sinkProps = getFileSinkMinimalDefaultConfigs(); + + // populate the sheet with simple rows + populateSpreadSheetWithSimpleRows(sheetsService, testSourceFileId, + generateFormattedRows(simpleNumberValue, numberOfDays, partOfDay, percent, + currency)); + + DeploymentDetails deploymentDetails = + deployApplication(sourceProps, sinkProps, SHEETS_SOURCE_STAGE_NAME, FILE_SINK_STAGE_NAME, + GOOGLE_SHEETS_PLUGIN_NAME, FILE_PLUGIN_NAME, + GOOGLE_DRIVE_ARTIFACT, FILE_ARTIFACT, + GOOGLE_SHEETS_PLUGIN_NAME + "-" + testName.getMethodName()); + startWorkFlow(deploymentDetails.getAppManager(), ProgramRunStatus.COMPLETED); + + // check number of rows in and out + checkRowsNumber(deploymentDetails, 1); + + Assert.assertTrue(Files.isDirectory(tmpFolder)); + + List allDeploysResults = Files.list(tmpFolder).collect(Collectors.toList()); + Assert.assertEquals(1, allDeploysResults.size()); + + Path deployResult = allDeploysResults.get(0); + Assert.assertTrue(Files.isDirectory(deployResult)); + Assert.assertEquals(1, + Files.list(deployResult).filter(p -> p.getFileName().toString().equals("_SUCCESS")).count()); + + List destFiles = + Files.list(deployResult).filter(p -> p.getFileName().toString().startsWith("part")).collect(Collectors.toList()); + Assert.assertEquals(1, destFiles.size()); + + Path destFile = destFiles.get(0); + List fileLines = null; + try { + fileLines = Files.readAllLines(destFile); + } catch (IOException e) { + Assert.fail(String.format("Exception during reading file '%s': %s", destFile.toString(), e.getMessage())); + } + + Assert.assertEquals(1, fileLines.size()); + + String resultLine = fileLines.get(0); + + JsonParser jsonParser = new JsonParser(); + JsonElement rootElement = jsonParser.parse(resultLine); + Assert.assertTrue(rootElement.isJsonObject()); + + JsonObject rootObject = rootElement.getAsJsonObject(); + + Assert.assertEquals(5, rootObject.entrySet().size()); + Assert.assertEquals("12", rootObject.get("A").getAsString()); + Assert.assertEquals("31.12.1899", rootObject.get("B").getAsString()); + Assert.assertEquals("12:00:00", rootObject.get("C").getAsString()); + Assert.assertEquals("300%", rootObject.get("D").getAsString()); + Assert.assertEquals("$54", rootObject.get("E").getAsString()); + } + + private int getNonNullRowsCount(List fileLines) { + JsonParser jsonParser = new JsonParser(); + int notNullCount = 0; + for (String line : fileLines) { + JsonElement rootElement = jsonParser.parse(line); + Assert.assertTrue(rootElement.isJsonObject()); + + JsonObject rootObject = rootElement.getAsJsonObject(); + + Assert.assertEquals(5, rootObject.entrySet().size()); + if (!rootObject.get("A").isJsonNull()) { + notNullCount++; + } + } + return notNullCount; + } + + private Map getSheetsSourceMinimalDefaultConfigs() { + return new HashMap() { + { + put("referenceName", "ref"); + put("directoryIdentifier", sourceFolderId); + put("modificationDateRange", "lifetime"); + put("sheetsToPull", "all"); + + put("extractMetadata", "false"); + put("metadataFieldName", "metadata"); + put("formatting", "formattedValues"); + put("skipEmptyData", "true"); + put("addNameFields", "false"); + put("columnNamesSelection", "noColumnNames"); + put("lastDataColumn", "5"); + put("lastDataRow", "5"); + put("readBufferSize", "5"); + + put("authType", "oAuth2"); + put("clientId", getClientId()); + put("clientSecret", getClientSecret()); + put("refreshToken", getRefreshToken()); + put("maxRetryCount", "8"); + put("maxRetryWait", "200"); + put("maxRetryJitterWait", "100"); + } + }; + } + + private Map getFileSourceMinimalDefaultConfigs() { + Set schemaFields = new HashSet<>(); + schemaFields.add(Schema.Field.of("body", Schema.nullableOf(Schema.of(Schema.Type.BYTES)))); + Schema fileSchema = Schema.recordOf( + "blob", + schemaFields); + return new HashMap() { + { + put("path", tmpFolder.toString()); + put("referenceName", "fileref"); + put("format", "json"); + put("schema", fileSchema.toString()); + } + }; + } + + private Map getSheetsSinkMinimalDefaultConfigs() { + return new HashMap() { + { + put("referenceName", "refd"); + put("directoryIdentifier", sinkFolderId); + put("spreadsheetName", DEFAULT_SPREADSHEET_NAME); + put("sheetName", DEFAULT_SHEET_NAME); + put("writeSchema", "false"); + put("threadsNumber", "1"); + put("maxBufferSize", "10"); + put("recordsQueueLength", "100"); + put("maxFlushInterval", "10"); + put("flushExecutionTimeout", "100"); + put("minPageExtensionSize", "100"); + put("mergeDataCells", "false"); + put("skipNameFields", "true"); + + put("authType", "oAuth2"); + put("clientId", getClientId()); + put("clientSecret", getClientSecret()); + put("refreshToken", getRefreshToken()); + put("maxRetryCount", "8"); + put("maxRetryWait", "200"); + put("maxRetryJitterWait", "100"); + } + }; + } + + private Map getFileSinkMinimalDefaultConfigs() { + return new HashMap() { + { + put("suffix", "yyyy-MM-dd-HH-mm"); + put("path", tmpFolder.toString()); + put("referenceName", "fileref"); + put("format", "json"); + } + }; + } + + private static Spreadsheet getSpreadsheet(String spreadSheetId) throws IOException { + Sheets.Spreadsheets.Get request = sheetsService.spreadsheets().get(spreadSheetId); + request.setIncludeGridData(true); + return request.execute(); + } + + private static List generateSimpleRows(int numberOfRows, int numberOfColumns) { + List rows = new ArrayList<>(); + for (int i = 0; i < numberOfRows; i++) { + List row = new ArrayList<>(); + for (int j = 0; j < numberOfColumns; j++) { + row.add(new CellData().setUserEnteredValue(new ExtendedValue().setStringValue(i + "" + j))); + } + rows.add(new RowData().setValues(row)); + } + return rows; + } + + private static List generateRowsWithNames(int numberOfRowsPerName, int numberOfColumns, List names) { + List rows = new ArrayList<>(); + for (int i = 0; i < numberOfRowsPerName; i++) { + for (String name : names) { + List row = new ArrayList<>(); + for (int j = 0; j < numberOfColumns; j++) { + row.add(new CellData().setUserEnteredValue(new ExtendedValue().setStringValue(name))); + } + rows.add(new RowData().setValues(row)); + } + } + return rows; + } + + /** + * | metadataKey1 | metadataValue1 | + * | metadataKey2 | metadataValue2 | + * | header1 | header2 | + * | data 1 | data2 | + * | metadataKey3 | metadataValue3 | + * + * @return list of populated rows. + */ + private static List generateMetadataRows(String metadataKey1, String metadataValue1, + String metadataKey2, String metadataValue2, + String metadataKey3, String metadataValue3, + String header1, String header2, + String data1, String data2) { + List rows = new ImmutableList.Builder() + .add(new RowData().setValues(new ImmutableList.Builder() + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(metadataKey1))) + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(metadataValue1))) + .build())) + .add(new RowData().setValues(new ImmutableList.Builder() + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(metadataKey2))) + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(metadataValue2))) + .build())) + .add(new RowData().setValues(new ImmutableList.Builder() + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(header1))) + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(header2))) + .build())) + .add(new RowData().setValues(new ImmutableList.Builder() + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(data1))) + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(data2))) + .build())) + .add(new RowData().setValues(new ImmutableList.Builder() + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(metadataKey3))) + .add(new CellData().setUserEnteredValue( + new ExtendedValue().setStringValue(metadataValue3))) + .build())) + .build(); + return rows; + } + + /** + * | 2 | 12.09.2018 | 06:45:32 | 150% | $45 | + * + * @return list of populated rows. + */ + private static List generateFormattedRows(Double simpleNumberValue, Double numberOfDays, + Double partOfDay, Double percent, + Double currency) { + List rows = new ImmutableList.Builder() + .add(new RowData().setValues(new ImmutableList.Builder() + .add(new CellData() + .setUserEnteredValue(new ExtendedValue().setNumberValue(simpleNumberValue))) + .add(new CellData() + .setUserEnteredValue(new ExtendedValue().setNumberValue(numberOfDays)) + .setUserEnteredFormat(new CellFormat().setNumberFormat( + new NumberFormat().setType("DATE").setPattern("dd.MM.yyyy")))) + .add(new CellData() + .setUserEnteredValue(new ExtendedValue().setNumberValue(partOfDay)) + .setUserEnteredFormat(new CellFormat().setNumberFormat( + new NumberFormat().setType("TIME").setPattern("HH:mm:ss")))) + .add(new CellData() + .setUserEnteredValue(new ExtendedValue().setNumberValue(percent)) + .setUserEnteredFormat(new CellFormat().setNumberFormat( + new NumberFormat().setType("PERCENT").setPattern("0%")))) + .add(new CellData() + .setUserEnteredValue(new ExtendedValue().setNumberValue(currency)) + .setUserEnteredFormat(new CellFormat().setNumberFormat( + new NumberFormat().setType("CURRENCY").setPattern("[$$]#0")))) + .build())) + .build(); + return rows; + } + + private static void populateSpreadSheetWithSimpleRows(Sheets sheetsService, String fileId, + List rowData) throws IOException { + BatchUpdateSpreadsheetRequest requestBody = new BatchUpdateSpreadsheetRequest(); + requestBody.setRequests(new ArrayList<>()); + + AppendCellsRequest appendCellsRequest = new AppendCellsRequest(); + appendCellsRequest.setFields("*"); + appendCellsRequest.setSheetId(getFirstAvailableSheetId(sheetsService, fileId)); + appendCellsRequest.setRows(rowData); + + requestBody.getRequests().add(new Request().setAppendCells(appendCellsRequest)); + + Sheets.Spreadsheets.BatchUpdate request = + sheetsService.spreadsheets().batchUpdate(fileId, requestBody); + + request.execute(); + } + + private static Integer getFirstAvailableSheetId(Sheets sheetsService, String fileId) throws IOException { + Spreadsheet spreadsheet = sheetsService.spreadsheets().get(fileId).execute(); + return spreadsheet.getSheets().get(0).getProperties().getSheetId(); + } +} diff --git a/pom.xml b/pom.xml index 413aba7a6..c2bcac323 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 0.8.2.2 1.1.1.7 0.1.0-SNAPSHOT + v4-rev581-1.25.0 true @@ -105,6 +106,11 @@ google-api-services-drive ${drive-api.version} + + com.google.apis + google-api-services-sheets + ${sheets-api.version} + com.google.guava guava