Merge pull request #150 from CBIIT/INS-1475

INS-1475

Author: JoonLeeNIH

src/main/java/gov/nih/nci/bento_ri/model/PrivateESDataFetcher.java

@@ -6,10 +6,13 @@
 import gov.nih.nci.bento.service.ESService;
 import gov.nih.nci.bento.utility.TypeChecker;
 import gov.nih.nci.bento_ri.service.InsESService;
+
 import graphql.schema.idl.RuntimeWiring;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.client.Request;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import com.google.gson.JsonArray;
@@ -27,13 +30,17 @@ public class PrivateESDataFetcher extends AbstractPrivateESDataFetcher {
     private final YamlQueryFactory yamlQueryFactory;
     private InsESService insEsService;
 
+    @Value("${aws.cloudfront.url:#{environment.AWS_CLOUDFRONT_URL}}")
+    private String cloudFrontUrl;
+
     // parameters used in queries
     final String PAGE_SIZE = "first";
     final String OFFSET = "offset";
     final String ORDER_BY = "order_by";
     final String SORT_DIRECTION = "sort_direction";
 
     final String DATASETS_END_POINT = "/datasets/_search";
+    final String FILES_END_POINT = "/files/_search";
     final String GRANTS_END_POINT = "/grants/_search";
     final String PROGRAMS_END_POINT = "/programs/_search";
     final String PROJECTS_END_POINT = "/projects/_search";
@@ -127,6 +134,10 @@ public RuntimeWiring buildRuntimeWiring() throws IOException {
                             Map<String, Object> args = env.getArguments();
                             return datasetDetails(args);
                         })
+                        .dataFetcher("getDatasetFiles", env -> {
+                            Map<String, Object> args = env.getArguments();
+                            return getDatasetFiles(args);
+                        })
                         .dataFetcher("programDetails", env -> {
                             Map<String, Object> args = env.getArguments();
                             return programDetails(args);
@@ -826,6 +837,83 @@ private Map<String, Object> datasetDetails(Map<String, Object> params) throws IO
         return dataset;
     }
 
+    /**
+     * Gets the files for a single Dataset record
+     *
+     * @param params The filters applied
+     * @return A list of the Dataset's files
+     * @throws IOException
+     */
+    private List<Map<String, Object>> getDatasetFiles(Map<String, Object> params) throws IOException {
+        List<Map<String, Object>> files;
+
+        final String[][] PROPERTIES = new String[][]{
+            new String[]{"dataset_source_id", "dataset_source_id"},
+            new String[]{"file_url_repo_prefix", "file_url_repo_prefix"},
+            new String[]{"file_id", "file_id"},
+            new String[]{"file_name", "file_name"},
+            new String[]{"file_type", "file_type"},
+            new String[]{"file_url", "file_url"},
+            new String[]{"access_level", "access_level"}
+        };
+
+        Map<String, String> mapping = Map.ofEntries(
+            Map.entry("dataset_source_id", "dataset_source_id.sort"),
+            Map.entry("file_url_repo_prefix", "file_url_repo_prefix.sort"),
+            Map.entry("file_id", "file_id.sort"),
+            Map.entry("file_name", "file_name.sort"),
+            Map.entry("file_type", "file_type.sort"),
+            Map.entry("file_url", "file_url.sort"),
+            Map.entry("access_level", "access_level.sort")
+        );
+
+        // Create a new params object rather than mutating the input
+        Map<String, Object> queryParams = new HashMap<>(params);
+
+        // Rename 'accessTypes' to 'access_level'
+        queryParams.put("access_level.search", queryParams.remove("accessTypes"));
+
+        // Turn dataset_source_id into a list
+        queryParams.put("dataset_source_id.search", List.of(queryParams.remove("dataset_source_id")));
+
+        // Turn filters into lowercase
+        for (String key : queryParams.keySet()) {
+            @SuppressWarnings("unchecked")
+            List<String> list = (List<String>) queryParams.get(key);
+            List<String> lowerCaseList = new ArrayList<String>();
+            list.forEach(s -> lowerCaseList.add(s.toLowerCase()));
+            queryParams.put(key, lowerCaseList);
+        }
+
+        // Add missing sorting params if not present
+        queryParams.putIfAbsent(ORDER_BY, "file_name");
+        queryParams.putIfAbsent(SORT_DIRECTION, "ASC");
+        queryParams.putIfAbsent(PAGE_SIZE, InsESService.MAX_ES_SIZE); // use MAX_ES_SIZE for maximum files per request
+        queryParams.putIfAbsent(OFFSET, 0);
+
+        files = overview(FILES_END_POINT, queryParams, PROPERTIES, "file_name", mapping, REGULAR_PARAMS, "nested_filters", "files");
+
+        // Form URLs for download
+        for (Map<String, Object> file : files) {
+            String prefix = (String) file.get("file_url_repo_prefix");
+            String fileUrl = (String) file.get("file_url"); // Required property, guaranteed by ETL
+
+            // Compose downloadUrl, handling extra slashes
+            String url = cloudFrontUrl.endsWith("/") ? cloudFrontUrl.substring(0, cloudFrontUrl.length() - 1) : cloudFrontUrl;
+            url += "/" + (prefix != null ? prefix : "");
+            if (!url.endsWith("/") && !fileUrl.startsWith("/")) {
+                url += "/";
+            }
+            url += fileUrl;
+
+            file.put("downloadUrl", url);
+            file.remove("file_url_repo_prefix");
+            file.remove("file_url");
+        }
+
+        return files;
+    }
+
     /**
      * Gets the details for a single Program record
      *

src/main/resources/application.properties

@@ -12,9 +12,9 @@ spring.mvc.view.suffix=.jsp
 error.redirect_url=http://localhost/error.html
 
 #Neo4j
-neo4j.url = bolt://10.208.1.7:7687
-neo4j.user = neo4j1
-neo4j.password = DJA8JU@1oQVk9
+#neo4j.url=
+#neo4j.user=
+#neo4j.password=
 
 #GraphQL
 graphql.schema=graphql/ins.graphql
@@ -36,7 +36,7 @@ redis.port=6379
 redis.ttl=-1
 
 # Elasticsearch Filtering
-es.host=vpc-ccdi-sandbox-hub-opensearch-2qghwrh6n4r2t25m56ryso2lay.us-east-1.es.amazonaws.com
+#es.host=
 es.port=443
 es.scheme=https
 es.filter.enabled = true
@@ -45,5 +45,8 @@ es.sign.requests = false
 es.service_name = es
 es.region = us-east-1
 
+# CloudFront URL
+#aws.cloudfront.url=
+
 #Testing
 test.queries_file=placeholder

src/main/resources/application.properties.j2

@@ -47,5 +47,8 @@ es.sign.requests = true
 es.service_name = es
 es.region = us-east-1
 
+# CloudFront URL
+#aws.cloudfront.url=
+
 #Testing
 test.queries_file=placeholder

src/main/resources/graphql/ins-private-es.graphql

@@ -152,6 +152,14 @@ type ProgramInfo {
     num_subjects: Int
 }
 
+type DatasetFile {
+    access_level: String!
+    downloadUrl: String!
+    file_id: String!
+    file_name: String!
+    file_type: String
+}
+
 type DatasetDetails {
     dataset_maximum_age_at_baseline: Int
     dataset_minimum_age_at_baseline: Int
@@ -380,6 +388,7 @@ type QueryType {
 
     programInfo: [ProgramInfo]
     datasetDetails(dataset_source_id: String):  DatasetDetails
+    getDatasetFiles(dataset_source_id: String!, accessTypes: [String!]!): [DatasetFile]
     programDetails(program_id: String):  ProgramDetails
     projectDetails(project_id: String):  ProjectDetails
     subjectDetail(subject_id: String): SubjectDetail