Legend ingest fix

legend-engine-xts-ingest/legend-engine-xt-ingest-protocol/pom.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.finos.legend.engine</groupId>
+        <artifactId>legend-engine-xts-ingest</artifactId>
+        <version>4.67.1-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>legend-engine-xt-ingest-protocol</artifactId>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-executionPlan-dependencies</artifactId>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <configuration>
+                    <addResourcesToClasspath>true</addResourcesToClasspath>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>generate metamodel</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                        <configuration>
+                            <includePluginDependencies>true</includePluginDependencies>
+                            <mainClass>org.finos.legend.engine.protocol.generation.GenerateMetaClasses</mainClass>
+                            <arguments>
+                                <argument>core_ingest_metamodel.protocol.json</argument>
+                                <argument>${project.build.directory}/generated-sources/</argument>
+                            </arguments>
+                            <classpathScope>test</classpathScope>
+                        </configuration>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-protocol-generation</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-ingest-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                </dependencies>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.basedir}/target/generated-sources/</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

legend-engine-xts-ingest/legend-engine-xt-ingest-protocol/src/main/resources/core_ingest_metamodel.protocol.json

@@ -0,0 +1,7 @@
+{
+  "purePackage": "meta::external::ingest::metamodel",
+  "javaPackage": "org.finos.legend.engine.protocol.ingest.metamodel",
+  "elementsToBeExcluded": [
+    "meta::external::ingest::metamodel::IngestConfig"
+  ]
+}

legend-engine-xts-ingest/legend-engine-xt-ingest-pure/pom.xml

@@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.finos.legend.engine</groupId>
+        <artifactId>legend-engine-xts-ingest</artifactId>
+        <version>4.67.1-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>legend-engine-xt-ingest-pure</artifactId>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <packaging>jar</packaging>
+    <name>Legend Engine - XT - INGEST - PAR/JAVA</name>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.finos.legend.pure</groupId>
+                <artifactId>legend-pure-maven-generation-par</artifactId>
+                <configuration>
+                    <sourceDirectory>src/main/resources</sourceDirectory>
+                    <purePlatformVersion>${legend.pure.version}</purePlatformVersion>
+                    <repositories>
+                        <repository>core_ingest</repository>
+                    </repositories>
+                    <extraRepositories>
+                        <extraRepository>${project.basedir}/src/main/resources/core_ingest.definition.json</extraRepository>
+                    </extraRepositories>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>build-pure-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-analytics-lineage-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-pure-functions-unclassified-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-functionActivator-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.pure</groupId>
+                        <artifactId>legend-pure-m2-dsl-diagram-grammar</artifactId>
+                        <version>${legend.pure.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-relationalStore-core-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                </dependencies>
+            </plugin>
+            <plugin>
+                <groupId>org.finos.legend.pure</groupId>
+                <artifactId>legend-pure-maven-generation-java</artifactId>
+                <executions>
+                    <execution>
+                        <phase>compile</phase>
+                        <goals>
+                            <goal>build-pure-compiled-jar</goal>
+                        </goals>
+                        <configuration>
+                            <generateSources>true</generateSources>
+                            <preventJavaCompilation>true</preventJavaCompilation>
+                            <generationType>modular</generationType>
+                            <useSingleDir>true</useSingleDir>
+                            <repositories>
+                                <repository>core_ingest</repository>
+                            </repositories>
+                        </configuration>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-pure-functions-unclassified-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-functionActivator-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.pure</groupId>
+                        <artifactId>legend-pure-m2-dsl-diagram-grammar</artifactId>
+                        <version>${legend.pure.version}</version>
+                    </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-relationalStore-core-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                </dependencies>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.finos.legend.pure</groupId>
+            <artifactId>legend-pure-m3-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.pure</groupId>
+            <artifactId>legend-pure-m4</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.pure</groupId>
+            <artifactId>legend-pure-runtime-java-engine-compiled</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-platform-java</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-platform-dsl-store-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-platform-store-relational-java</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-core-pure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.pure</groupId>
+            <artifactId>legend-pure-m2-store-relational-pure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-code-compiled-core</artifactId>
+        </dependency>
+
+    </dependencies>
+</project>

legend-engine-xts-ingest/legend-engine-xt-ingest-pure/src/main/java/org/finos/legend/pure/code/core/CoreIngestCodeRepositoryProvider.java

@@ -0,0 +1,29 @@
+// Copyright 2023 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.pure.code.core;
+
+import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository;
+import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider;
+import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository;
+
+public class CoreIngestCodeRepositoryProvider implements CodeRepositoryProvider
+{
+    @Override
+    public CodeRepository repository()
+    {
+        return GenericCodeRepository.build("core_ingest.definition.json");
+    }
+}
+

legend-engine-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider

@@ -0,0 +1 @@
+org.finos.legend.pure.code.core.CoreIngestCodeRepositoryProvider

legend-engine-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/core_ingest.definition.json

@@ -0,0 +1,12 @@
+{
+  "name": "core_ingest",
+  "pattern": "(meta::external::ingest)(::.*)?",
+  "dependencies": [
+    "core",
+    "platform",
+    "platform_dsl_store",
+    "platform_store_relational",
+    "core_functions_json",
+    "core_relational"
+  ]
+}

legend-engine-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/core_ingest/barrier.pure

@@ -0,0 +1,30 @@
+import meta::external::ingest::barrier::*;
+import meta::external::ingest::metamodel::*;
+
+function meta::external::ingest::barrier::isDatasetAvailable<T>(ingestConfig: IngestConfig<T>[1], prevPublishedWatermark: Watermark[1], currentStateWatermark: Watermark[1]):Boolean[1]
+{
+  let prevHighBatchId = getBatchId($ingestConfig.datasetName, $prevPublishedWatermark);
+  let currentBatchId = getBatchId($ingestConfig.datasetName, $currentStateWatermark);   
+  ($currentBatchId > $prevHighBatchId) ;                
+}
+
+function meta::external::ingest::barrier::isDatasetAvailable<T>(ingestConfig: IngestConfig<T>[1],filterFunc:Function<{T[1]->Boolean[1]}>[1], prevPublishedWatermark: Watermark[1], currentStateWatermark: Watermark[1]):Boolean[1]
+{
+  let prevHighBatchId = getBatchId($ingestConfig.datasetName, $prevPublishedWatermark);
+  let currentBatchId = getBatchId($ingestConfig.datasetName, $currentStateWatermark); 
+
+//TODO : Change to T , likely need to uplift getAll to handle T properly
+  meta::pure::functions::collection::getAll(BatchMetadata)
+                  ->filter(x | $x.datasetName == $ingestConfig.datasetName)
+                  ->filter(y | ($y.batchId > $prevHighBatchId) && ($y.batchId <= $currentBatchId))
+                  ->cast(@T)
+                  ->filter($filterFunc)
+                  ->isNotEmpty();
+}
+
+
+function <<access.private>> meta::external::ingest::barrier::getBatchId(datasetName: String[1], watermark: Watermark[1]):Integer[1]
+{
+  $watermark.datasetMarks->filter(w | $w.datasetName == $datasetName)->toOne().batchId;
+}
+

legend-engine-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/core_ingest/metamodel.pure

@@ -0,0 +1,37 @@
+import meta::external::ingest::metamodel::*;
+
+Class meta::external::ingest::metamodel::DatasetMark
+{
+  datasetName: String[1];
+  batchId: Integer[1];
+}
+
+Class meta::external::ingest::metamodel::Watermark
+{
+  watermarkId: String[1];
+  datasetMarks: DatasetMark[*];
+}
+
+Class meta::external::ingest::metamodel::IngestConfig<T>
+{ 
+  datasetName: String[1];  
+  metadataType: Class<T>[1];  
+}
+
+Class meta::external::ingest::metamodel::BatchMetadata
+{
+  datasetName: String[1];
+  batchId: Integer[1];
+  ingest_request_id: String[1];
+  ingest_definition_urn: String[1];  
+}
+
+Class meta::external::ingest::metamodel::EodBatchMetadata extends meta::external::ingest::metamodel::BatchMetadata
+{
+  endOfDay: Boolean[1];
+}
+
+Class meta::external::ingest::metamodel::BusinessDateBatchMetadata extends meta::external::ingest::metamodel::EodBatchMetadata
+{
+  businessDate: StrictDate[1];
+}