From c924108d74b4a9fbd425c601b2636e529745ac3b Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 21 Apr 2025 10:23:07 +0800 Subject: [PATCH 01/62] Add withError in Record.Builder --- .../sdk/component/api/record/Record.java | 8 ++++ .../sdk/component/api/record/Schema.java | 5 +++ .../component/api/record/SchemaProperty.java | 8 ++++ .../component/api/service/schema/Schema.java | 4 ++ .../sdk/component/api/record/SchemaTest.java | 3 ++ .../component/runtime/record/RecordImpl.java | 45 +++++++++++++++++++ .../component/runtime/record/SchemaImpl.java | 9 ++++ .../runtime/record/RecordBuilderImplTest.java | 35 +++++++++++++++ 8 files changed, 117 insertions(+) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index b9258587bdbee..d34a27e6ed7fb 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -34,6 +34,8 @@ public interface Record { + String RECORD_ERROR_SUPPORT = "talend.sdk.runtime.record.error.support"; + /** * @return the schema of this record. */ @@ -311,6 +313,10 @@ default Optional getOptionalRecord(final String name) { return ofNullable(get(Record.class, name)); } + default boolean isValid() { + return getSchema().getAllEntries().filter(entry -> !entry.isValid()).findAny().isPresent(); + } + /** * Allows to create a record with a fluent API. This is the unique recommended way to create a record. */ @@ -447,5 +453,7 @@ default Builder withInstant(Schema.Entry entry, Instant value) { Builder withRecord(String name, Record value); Builder withArray(Schema.Entry entry, Collection values); + + Builder withError(String columnName, Object value, String errorMessage, Exception exception); } } diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java index 5050102b4879e..da4b0806638be 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java @@ -232,6 +232,11 @@ interface Entry { */ boolean isMetadata(); + /** + * @return true if the value of this entry is valid; false for invalid value. + */ + boolean isValid(); + /** * @param the default value type. * diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java index 076467334ce3c..95bf16be2d181 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java @@ -37,6 +37,14 @@ public interface SchemaProperty { String ALLOW_SPECIAL_NAME = "field.special.name"; + String ENTRY_IS_ON_ERROR = "entry.on.error"; + + String ENTRY_ERROR_MESSAGE = "entry.error.message"; + + String ENTRY_ERROR_FALLBACK_VALUE = "entry.error.fallback.value"; + + String ERROR_EXCEPTION = "entry.error.exception"; + enum LogicalType { DATE("date"), diff --git a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java index 355e321ebedce..197ea0299d8f7 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java @@ -30,6 +30,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.talend.sdk.component.api.record.SchemaProperty; @Partial("This API should support nested schema but the Studio is not yet ready.\n\n" + "The cloud platform also doesn't use it yet.\n\nAlso prefer to use " @@ -163,6 +164,9 @@ public String getProp(final String property) { public Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder()"); } + + @Override + public boolean isValid() { return true; } } } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java index 0157dabdff610..11c5e325f3c9d 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java @@ -326,6 +326,9 @@ public JsonValue getJsonProp(final String name) { return Entry.super.getJsonProp(name); } + @Override + public boolean isValid() { return true; } + } class SchemaBuilder implements Schema.Builder { diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 47c77ce50c529..9c3f6f06ad3cc 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -64,6 +64,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import org.talend.sdk.component.api.record.SchemaProperty; @EqualsAndHashCode public final class RecordImpl implements Record { @@ -92,6 +93,11 @@ public T get(final Class expectedType, final String name) { return RECORD_CONVERTERS.coerce(expectedType, value, name); } + @Override + public boolean isValid() { + return false; + } + @Override // for debug purposes, don't use it for anything else public String toString() { try (final Jsonb jsonb = JsonbBuilder @@ -513,6 +519,45 @@ public Builder withArray(final Schema.Entry entry, final Collection value return append(entry, values); } + @Override + public Builder withError(String columnName, Object value, String errorMessage, Exception exception) { + final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); + if (!supportError) { + throw new IllegalArgumentException(errorMessage); + } else { + // duplicate the schema instance with a modified Entry + final Entry oldEntry = this.findExistingEntry(columnName); + final Entry updateEntry = oldEntry.toBuilder() + .withName(columnName) + .withNullable(true) + .withType(oldEntry.getType()) + .withProp(SchemaProperty.ENTRY_IS_ON_ERROR, "true") + .withProp(SchemaProperty.ENTRY_ERROR_MESSAGE, errorMessage) + .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) + .withProp(SchemaProperty.ERROR_EXCEPTION, exception == null ? "" : exception.toString()) + .build(); + final Schema.Builder builder = getErrorBuilder(providedSchema.getType()); + providedSchema.getAllEntries() +// .filter(e -> Objects.equals(providedSchema.getEntry(e.getName()), e)) + .forEach(e -> { + if (columnName.equals(e.getName())) { + builder.withEntry(updateEntry); + } else { + builder.withEntry(e); + } + }); + return getErrorBuilder(builder); + } + } + + protected Builder getErrorBuilder(Schema.Builder builder) { + return new BuilderImpl(builder.build()); + } + + protected Schema.Builder getErrorBuilder(Schema.Type type) { + return new SchemaImpl.BuilderImpl().withType(type); + } + private void assertType(final Schema.Type actual, final Schema.Type expected) { if (actual != expected) { throw new IllegalArgumentException("Expected entry type: " + expected + ", got: " + actual); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java index 035aec97f83aa..6348c4d0f69f0 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java @@ -465,6 +465,15 @@ public Map getProps() { return this.props; } + @Override + public boolean isValid() { + String property = this.getProp(SchemaProperty.ENTRY_IS_ON_ERROR); + if (property == null) { + return true; + } + return !Boolean.parseBoolean(property); + } + /** * Plain builder matching {@link Entry} structure. */ diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index dec6e7ceb3260..9cddee216ca8b 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -17,6 +17,7 @@ import static java.util.stream.Collectors.joining; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -248,6 +249,40 @@ void dateTime() { assertThrows(IllegalArgumentException.class, () -> builder2.withDateTime("date", (ZonedDateTime) null)); } + @Test + void withError() { + final Schema schema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("intValue") + .withNullable(false) + .withType(Type.INT) + .build()) + .build(); + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); + final Record record3 = builder3.withError("date", null, "date is null", null) + .withError("intValue", "string", "wrong int value", null) + .build(); + assertFalse(record3.isValid()); + final Entry entry = record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); + assertNotNull(entry); + Assertions.assertFalse(entry.isValid()); + + final Entry entry2 = record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); + assertNotNull(entry2); + Assertions.assertFalse(entry2.isValid()); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); + } + @Test void zonedDateTimeVSInstant() { final Schema schema = new SchemaImpl.BuilderImpl() From ae4cad71d6cb2d707300fa1737b068ff183d91e1 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 21 Apr 2025 18:17:30 +0800 Subject: [PATCH 02/62] Test for AvroRecord --- .../sdk/component/api/record/RecordTest.java | 3 + .../record/RecordBuilderFactoryTest.java | 3 + .../beam/spi/record/AvroRecordBuilder.java | 9 +++ .../spi/record/AvroRecordBuilderTest.java | 62 +++++++++++++++---- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java index c734c2bc50a55..c07eda9008351 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java @@ -135,6 +135,9 @@ public Map getProps() { public String getProp(String property) { return null; } + + @Override + public boolean isValid() { return true; } }; Assertions.assertEquals("value", record.get(String.class, e1)); } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java index 006ffddb6d3d2..a34a6cc58d82f 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java @@ -102,6 +102,9 @@ public String getProp(final String property) { public Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder()"); } + + @Override + public boolean isValid() { return true; } } @RequiredArgsConstructor diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java index 0ce9e2c4c21b6..92232af527616 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java @@ -35,4 +35,13 @@ public AvroRecordBuilder(final Schema providedSchema) { public Record build() { return new AvroRecord(super.build()); } + + protected Record.Builder getErrorBuilder(Schema.Builder builder) { + + return new AvroRecordBuilder(builder.build()); + } + + protected Schema.Builder getErrorBuilder(Schema.Type type) { + return new AvroSchemaBuilder().withType(type); + } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 07fb3da564bf3..7777ef456b03a 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -192,21 +192,59 @@ void avroTest() { @Test void mixedRecordTest() { - final AvroRecordBuilderFactoryProvider recordBuilderFactoryProvider = new AvroRecordBuilderFactoryProvider(); - System.setProperty("talend.component.beam.record.factory.impl", "avro"); - final RecordBuilderFactory recordBuilderFactory = recordBuilderFactoryProvider.apply("test"); - - final RecordBuilderFactory otherFactory = new RecordBuilderFactoryImpl("test"); - final Schema schema = otherFactory - .newSchemaBuilder(RECORD) - .withEntry(otherFactory.newEntryBuilder().withName("e1").withType(INT).build()) + final Schema schema0 = new AvroSchemaBuilder()// + .withType(RECORD) // + .withEntry(dataEntry1) // + .withEntryBefore("data1", meta1) // + .withEntry(dataEntry2) // + .withEntryAfter("meta1", meta2) // .build(); - final Schema arrayType = recordBuilderFactory // - .newSchemaBuilder(Schema.Type.ARRAY) // - .withElementSchema(schema) + final Record.Builder builder0 = factory.newRecordBuilder(schema0); + builder0.withInt("data1", 101) + .withString("data2", "102") + .withInt("meta1", 103) + .withString("meta2", "104"); + final Record record0 = builder0.build(); + assertEquals(101, record0.getInt("data1")); + } + + @Test + void testWithError() { + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + + org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD); + Schema.Entry nameEntry = factory + .newEntryBuilder() + .withName("name") + .withNullable(false) + .withType(Schema.Type.STRING) .build(); - Assertions.assertNotNull(arrayType); + Schema.Entry ageEntry = factory + .newEntryBuilder() + .withName("age") + .withNullable(false) + .withType(Schema.Type.INT) + .build(); + Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(ageEntry).build(); + // record 1 + Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); + Record record1 = recordBuilder.withError("name", null, "Stirng is null", null) + .withError("age", "string", "is not an int", null) + .build(); + assertFalse(record1.isValid()); + + final Schema.Entry entry = record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); + assertNotNull(entry); + Assertions.assertFalse(entry.isValid()); + + final Schema.Entry entry2 = record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); + assertNotNull(entry2); + Assertions.assertFalse(entry2.isValid()); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); } @Test From 920576501ac6df5e22f00bdac1cd9641afe9de8b Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 22 Apr 2025 15:06:11 +0800 Subject: [PATCH 03/62] fix isValid --- .../java/org/talend/sdk/component/api/record/Record.java | 4 +++- .../talend/sdk/component/api/service/schema/Schema.java | 2 +- .../talend/sdk/component/runtime/record/RecordImpl.java | 7 +------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index d34a27e6ed7fb..28a9a3dd6bb6f 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -314,7 +314,9 @@ default Optional getOptionalRecord(final String name) { } default boolean isValid() { - return getSchema().getAllEntries().filter(entry -> !entry.isValid()).findAny().isPresent(); + return !getSchema().getAllEntries() + .filter(entry -> !entry.isValid()).findAny() + .isPresent(); } /** diff --git a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java index 197ea0299d8f7..433673db4b170 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java @@ -30,7 +30,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.talend.sdk.component.api.record.SchemaProperty; @Partial("This API should support nested schema but the Studio is not yet ready.\n\n" + "The cloud platform also doesn't use it yet.\n\nAlso prefer to use " @@ -167,6 +166,7 @@ public Builder toBuilder() { @Override public boolean isValid() { return true; } + } } diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 9c3f6f06ad3cc..82b05b3ed618f 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -93,11 +93,6 @@ public T get(final Class expectedType, final String name) { return RECORD_CONVERTERS.coerce(expectedType, value, name); } - @Override - public boolean isValid() { - return false; - } - @Override // for debug purposes, don't use it for anything else public String toString() { try (final Jsonb jsonb = JsonbBuilder @@ -522,7 +517,7 @@ public Builder withArray(final Schema.Entry entry, final Collection value @Override public Builder withError(String columnName, Object value, String errorMessage, Exception exception) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); - if (!supportError) { + if (!supportError) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry From cf0a3c684eb336a7f04fcab71cb86de9ec9893a8 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 30 Apr 2025 14:19:02 +0800 Subject: [PATCH 04/62] Add sample-feature\supporterror --- .../sdk/component/api/record/Record.java | 3 +- .../beam/spi/record/AvroRecordBuilder.java | 4 +- .../spi/record/AvroRecordBuilderTest.java | 7 +- sample-parent/sample-features/pom.xml | 1 + .../sample-features/supporterror/pom.xml | 104 +++++++++ .../feature/supporterror/SVersions.java | 23 ++ .../sample/feature/supporterror/Cli.java | 212 ++++++++++++++++++ .../supporterror/SupportErrorInput.java | 131 +++++++++++ .../supporterror/SupportErrorMapper.java | 60 +++++ .../feature/supporterror/package-info.java | 21 ++ ...nent.runtime.serialization.ContainerFinder | 1 + .../src/main/resources/icons/dark/icon.svg | 5 + .../src/main/resources/icons/dark/mapper.svg | 65 ++++++ .../src/main/resources/icons/light/icon.svg | 5 + .../src/main/resources/icons/light/mapper.svg | 65 ++++++ .../feature/supporterror/Messages.properties | 27 +++ 16 files changed, 728 insertions(+), 6 deletions(-) create mode 100644 sample-parent/sample-features/supporterror/pom.xml create mode 100644 sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 28a9a3dd6bb6f..8fceaf047897a 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -315,7 +315,8 @@ default Optional getOptionalRecord(final String name) { default boolean isValid() { return !getSchema().getAllEntries() - .filter(entry -> !entry.isValid()).findAny() + .filter(entry -> !entry.isValid()) + .findAny() .isPresent(); } diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java index 92232af527616..35fd6082bd89c 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java @@ -36,12 +36,12 @@ public Record build() { return new AvroRecord(super.build()); } - protected Record.Builder getErrorBuilder(Schema.Builder builder) { + protected Record.Builder getErrorBuilder(final Schema.Builder builder) { return new AvroRecordBuilder(builder.build()); } - protected Schema.Builder getErrorBuilder(Schema.Type type) { + protected Schema.Builder getErrorBuilder(final Schema.Type type) { return new AvroSchemaBuilder().withType(type); } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 7777ef456b03a..bbfc3eda1e6b7 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -54,7 +54,6 @@ import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; -import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.record.SchemaImpl; import org.talend.sdk.component.runtime.record.SchemaImpl.EntryImpl; @@ -236,11 +235,13 @@ void testWithError() { .build(); assertFalse(record1.isValid()); - final Schema.Entry entry = record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); + final Schema.Entry entry = + record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); assertNotNull(entry); Assertions.assertFalse(entry.isValid()); - final Schema.Entry entry2 = record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); + final Schema.Entry entry2 = + record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); diff --git a/sample-parent/sample-features/pom.xml b/sample-parent/sample-features/pom.xml index 3214d3ab785d6..1b1046b48b1b5 100644 --- a/sample-parent/sample-features/pom.xml +++ b/sample-parent/sample-features/pom.xml @@ -32,6 +32,7 @@ aftergroup-lastgroup conditional-outputs checkpoint-runner + supporterror diff --git a/sample-parent/sample-features/supporterror/pom.xml b/sample-parent/sample-features/supporterror/pom.xml new file mode 100644 index 0000000000000..4f1279e001ebb --- /dev/null +++ b/sample-parent/sample-features/supporterror/pom.xml @@ -0,0 +1,104 @@ + + + + 4.0.0 + + + org.talend.sdk.component + sample-features + 1.81.0-SNAPSHOT + + + org.talend.sdk.component.sample.feature + supporterror + jar + + Component Runtime :: Sample Feature :: Supporterror + + + + org.talend.sdk.component + component-api + ${project.version} + compile + + + org.talend.sdk.component + component-runtime-manager + ${project.version} + compile + + + info.picocli + picocli + 4.7.6 + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + + + + true + org.talend.sdk.component.sample.feature.supporterror.Cli + + + + + + maven-shade-plugin + 2.1 + + + + shade + + package + + + + org.talend.sdk.component.sample.feature.supporterror.Cli + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.0.0 + + org.talend.sdk.component.sample.feature.supporterror.Cli + + + + + diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java new file mode 100644 index 0000000000000..0ad88da683be1 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java @@ -0,0 +1,23 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.talend.sdk.component.sample.feature.supporterror; + +public interface SVersions { + + String KIT_VERSION = "${project.version}"; + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java new file mode 100644 index 0000000000000..c39114ce26f44 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -0,0 +1,212 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.supporterror; + + +import static java.util.Optional.ofNullable; +import static lombok.AccessLevel.PRIVATE; +import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +import org.talend.sdk.component.api.record.Record; +import org.talend.sdk.component.api.record.Schema; +import org.talend.sdk.component.api.record.SchemaProperty; +import org.talend.sdk.component.api.service.record.RecordBuilderFactory; +import org.talend.sdk.component.dependencies.maven.Artifact; +import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; +import org.talend.sdk.component.runtime.input.Mapper; +import org.talend.sdk.component.runtime.manager.ComponentManager; +import org.talend.sdk.component.runtime.output.ProcessorImpl; +import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; +import org.talend.sdk.component.runtime.serialization.ContainerFinder; +import org.talend.sdk.component.runtime.serialization.LightContainer; + + +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = PRIVATE) +@Command(name="supportError") +public final class Cli implements Callable { + +// @Parameters(index = "0", description = "The file whose checksum to calculate.") +// private File file; + + //support errors or not. default=false + @Option(names = "-s", defaultValue = "true") + boolean support; + + @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") + File jar; + + @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") + String mapper; + + @Option(names = { "-fl", "--family" },defaultValue = "supporterror") + String family; + + static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + + SVersions.KIT_VERSION; + + @Override + public Integer call() throws Exception { + + try (final ComponentManager manager = manager(jar, GAV)) { + // final JsonObject jsonConfig = readJsonFromFile(configurationFile); + final Map configuration = new HashMap<>(); + + info("support" + String.valueOf(support)); + // create the mapper + final Mapper mpr = manager.findMapper(family, mapper, 1, configuration) + .orElseThrow(() -> new IllegalStateException( + String.format("No mapper found for: %s/%s.", family, manager))); + + final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) + .orElseThrow(() -> new IllegalStateException( + String.format("No Processor found for: %s/%s.", family, manager))); + + info("create input now."); + + List records = ((Supplier>) processor.getDelegate()).get(); + + //final SupportErrorInput input = mpr.create(); + //set the property for support or not +// input.setSupportError(String.valueOf(support)); + info("getting the record."); +// Record data = input.data(); + + for (Record data : records) { + info("Record : " + data.isValid()); + entryout(data, "date"); + entryout(data, "age"); + } + // + info("finished."); + } catch (Exception e) { + error(e); + } + + return 0; + } + + private static void entryout(final Record data, final String column) { + Schema.Entry ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny().get(); + if(ageEntry.isValid()) { + Integer age = data.get(Integer.class, column); + // process the age... + info("Record '"+ column +"': " + age); + } else{ + String errorMessage = ageEntry.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); + info("ERROR: " + errorMessage); + } + } + + public static void main(final String... args) { + int exitCode = new CommandLine(new Cli()).execute(args); + System.exit(exitCode); + } + + static final String ERROR = "[ERROR] "; + + static final String WARN = "[WARN] "; + + static final String INFO = "[INFO] "; + + static MvnCoordinateToFileConverter mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter(); + + + public static ComponentManager manager(final File jar, final String artifact) { + return new ComponentManager(findM2()) { + + final ContainerFinder containerFinder = ContainerFinder.Instance.get(); + + final ComponentManager originalMgr = contextualInstance().get(); + + { + contextualInstance().set(this); + String containerId; + if (jar != null) { + containerId = addPlugin(jar.getAbsolutePath()); + Cli.info(String.format("Manager is using plugin %s from %s.", containerId, jar)); + } else { + final String pluginPath = ofNullable(artifact) + .map(gav -> mvnCoordinateToFileConverter.toArtifact(gav)) + .map(Artifact::toPath) + .orElseThrow(() -> new IllegalArgumentException("Plugin GAV can't be empty")); + String p = findM2().resolve(pluginPath).toAbsolutePath().toString(); + containerId = addPlugin(p); + Cli.info(String.format("Manager is using plugin: %s from GAV %s.", containerId, artifact)); + } + DynamicContainerFinder.SERVICES.put(RecordBuilderFactory.class, + new RecordBuilderFactoryImpl(containerId)); + } + + @Override + public void close() { + DynamicContainerFinder.SERVICES.clear(); + super.close(); + contextualInstance().set(originalMgr); + } + }; + } + + public static class DynamicContainerFinder implements ContainerFinder { + + public static final Map LOADERS = new ConcurrentHashMap<>(); + + public static final Map, Object> SERVICES = new ConcurrentHashMap<>(); + + @Override + public LightContainer find(final String plugin) { + return new LightContainer() { + + @Override + public ClassLoader classloader() { + return LOADERS.get(plugin); + } + + @Override + public T findService(final Class key) { + return key.cast(SERVICES.get(key)); + } + }; + } + } + + public static void info(final String message) { + System.out.println(INFO + message); + } + + public static void warn(final String message) { + System.err.println(WARN + message); + } + + public static void error(final Throwable e) { + System.err.println(ERROR + e.getMessage()); + System.exit(501); + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java new file mode 100644 index 0000000000000..fafd69f1f96bc --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -0,0 +1,131 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.supporterror; + +import java.io.Serializable; +import java.util.Date; + +import javax.annotation.PostConstruct; + +import org.talend.sdk.component.api.component.Icon; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.type.DataSet; +import org.talend.sdk.component.api.configuration.type.DataStore; +import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.Producer; +import org.talend.sdk.component.api.meta.Documentation; +import org.talend.sdk.component.api.record.Record; +import org.talend.sdk.component.api.record.Schema; +import org.talend.sdk.component.runtime.record.RecordImpl; +import org.talend.sdk.component.runtime.record.SchemaImpl; + +import lombok.Data; + +@Version +@Icon(value = Icon.IconType.CUSTOM, custom = "icon") +@Documentation("Support Error Input sample processor connector.") +@Emitter(family = "supporterror", name = "supportErrorInput") +public class SupportErrorInput implements Serializable { + +// private final transient JsonBuilderFactory factory; + + private transient Schema recordSchema; + + private final transient InputConfig configuration; + + public SupportErrorInput(@Option("configuration") final InputConfig config) { + this.configuration = config; + } + + @PostConstruct + public void init() { + recordSchema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("age") + .withNullable(false) + .withType(Schema.Type.INT) + .build()) + .build(); + } + + //set support or not. + public void setSupportError(final String supportError) { + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + } + + @Producer + public Record data() { + final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); + final Record record = builder.withError("date", null, "date is null", null) + .withError("age", "string", "wrong int value", null) + .build(); + + return record; + } + + @Data + @GridLayout(value = { + @GridLayout.Row("dataset"), + }) + @Version + public static class InputConfig { + + @Option + @Documentation("Dataset.") + private Dataset dataset = new Dataset(); + } + + @DataSet + @Data + @GridLayout(value = { @GridLayout.Row("datastore") }) + public static class Dataset implements Serializable { + + @Option + @Documentation("Datastore.") + private Datastore datastore = new Datastore(); + + } + + @DataStore + @Data + @GridLayout(value = { @GridLayout.Row("name"), @GridLayout.Row("age"), + @GridLayout.Row("date")}) + public static class Datastore implements Serializable { + + @Option + @Documentation("Name prop.") + private String name = "test"; + + @Option + @Documentation("Age prop.") + private int age = 0; + + @Option + @Documentation("Date prop.") + private Date date; + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java new file mode 100644 index 0000000000000..3aa4210aae676 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.supporterror; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +import org.talend.sdk.component.api.component.Icon; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.input.Assessor; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.PartitionMapper; +import org.talend.sdk.component.api.input.PartitionSize; +import org.talend.sdk.component.api.input.Split; +import org.talend.sdk.component.api.meta.Documentation; + +@Version(1) +@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") +@PartitionMapper(name = "SupportErrorMapper", infinite = false) +@Documentation("Doc: default SupportErrorMapper documentation without Internationalization.") +public class SupportErrorMapper implements Serializable { + + private SupportErrorInput.InputConfig config; + + public SupportErrorMapper(final @Option("configin") SupportErrorInput.InputConfig config) { + this.config = config; + } + + @Assessor + public long estimateSize() { + return 1500L; + } + + @Split + public List split(@PartitionSize final int desiredNbSplits) { + return Collections.singletonList(this); + } + + @Emitter + public SupportErrorInput createSource() { + + return new SupportErrorInput(this.config); + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java new file mode 100644 index 0000000000000..ff1441932bd2a --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@Components(family = "supporterror", categories = "sample") +@Icon(value = Icon.IconType.CUSTOM, custom = "icon") +package org.talend.sdk.component.sample.feature.supporterror; + +import org.talend.sdk.component.api.component.Components; +import org.talend.sdk.component.api.component.Icon; \ No newline at end of file diff --git a/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder b/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder new file mode 100644 index 0000000000000..0d23def70f392 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder @@ -0,0 +1 @@ +org.talend.sdk.component.runtime.manager.finder.StandaloneContainerFinder diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg new file mode 100644 index 0000000000000..3351e2c0bc9e9 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg @@ -0,0 +1,5 @@ + + + + Checkpoint​Input + \ No newline at end of file diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg new file mode 100644 index 0000000000000..4f0fafff9a769 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg @@ -0,0 +1,65 @@ + + + + + + + + + + + + + diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg new file mode 100644 index 0000000000000..3351e2c0bc9e9 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg @@ -0,0 +1,5 @@ + + + + Checkpoint​Input + \ No newline at end of file diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg new file mode 100644 index 0000000000000..2958a0056838e --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg @@ -0,0 +1,65 @@ + + + + + + + + + + + + + diff --git a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties new file mode 100644 index 0000000000000..aa81e7a64c5fa --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties @@ -0,0 +1,27 @@ +# Copyright (C) 2006-2025 Talend Inc. - www.talend.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Here you can change all your configuration display names to use more explicit labels +# You can also translate your configuration by adding one file by local Messages_fr.properties for french for example +supporterror.supportErrorInput._displayName=Support Error Input +supporterror.SupportErrorMapper._displayName=Support Error Mapper +supporterror.datastore.default._displayName=default +supporterror.dataset.default._displayName=dataset +Dataset.datastore._displayName=Datastore +InputConfig.dataset._displayName=Dataset +Datastore.age._displayName=age +Datastore.date._displayName=date +Datastore.name._displayName=name +Datastore.age._placeholder= +Datastore.date._placeholder= +Datastore.name._placeholder= \ No newline at end of file From 7d049d1393a8fe026b52f6d591a8d6bb50d8c8c6 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 30 Apr 2025 15:39:16 +0800 Subject: [PATCH 05/62] worked --- .../sample/feature/supporterror/Cli.java | 49 +++++++++---------- .../supporterror/SupportErrorInput.java | 7 --- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index c39114ce26f44..608422e334bee 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -26,11 +26,9 @@ import java.io.File; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Supplier; import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; @@ -38,9 +36,8 @@ import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; -import org.talend.sdk.component.runtime.input.Mapper; +import org.talend.sdk.component.runtime.input.PartitionMapperImpl; import org.talend.sdk.component.runtime.manager.ComponentManager; -import org.talend.sdk.component.runtime.output.ProcessorImpl; import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; @@ -52,11 +49,8 @@ @Command(name="supportError") public final class Cli implements Callable { -// @Parameters(index = "0", description = "The file whose checksum to calculate.") -// private File file; - //support errors or not. default=false - @Option(names = "-s", defaultValue = "true") + @Option(names = "-s", defaultValue = "false") boolean support; @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") @@ -69,7 +63,7 @@ public final class Cli implements Callable { String family; static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" - + SVersions.KIT_VERSION; + + Versions.KIT_VERSION; @Override public Integer call() throws Exception { @@ -79,31 +73,30 @@ public Integer call() throws Exception { final Map configuration = new HashMap<>(); info("support" + String.valueOf(support)); + if (support) { + setSupportError(support); + } // create the mapper - final Mapper mpr = manager.findMapper(family, mapper, 1, configuration) + final PartitionMapperImpl mpr = (PartitionMapperImpl)manager.findMapper(family, mapper, 1, configuration) .orElseThrow(() -> new IllegalStateException( String.format("No mapper found for: %s/%s.", family, manager))); - final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) - .orElseThrow(() -> new IllegalStateException( - String.format("No Processor found for: %s/%s.", family, manager))); +// final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) +// .orElseThrow(() -> new IllegalStateException( +// String.format("No Processor found for: %s/%s.", family, manager))); info("create input now."); - List records = ((Supplier>) processor.getDelegate()).get(); + SupportErrorInput seInput = new SupportErrorInput(null); + seInput.init(); - //final SupportErrorInput input = mpr.create(); - //set the property for support or not -// input.setSupportError(String.valueOf(support)); info("getting the record."); -// Record data = input.data(); + Record data = seInput.data(); - for (Record data : records) { - info("Record : " + data.isValid()); - entryout(data, "date"); - entryout(data, "age"); - } - // + info("Record isValid = " + data.isValid()); + entryout(data, "date"); + entryout(data, "age"); + // info("finished."); } catch (Exception e) { error(e); @@ -124,6 +117,13 @@ private static void entryout(final Record data, final String column) { } } + //set support or not. + public void setSupportError(final boolean supportError) { + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + } + public static void main(final String... args) { int exitCode = new CommandLine(new Cli()).execute(args); System.exit(exitCode); @@ -137,7 +137,6 @@ public static void main(final String... args) { static MvnCoordinateToFileConverter mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter(); - public static ComponentManager manager(final File jar, final String artifact) { return new ComponentManager(findM2()) { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index fafd69f1f96bc..99758408a25d8 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -69,13 +69,6 @@ public void init() { .build(); } - //set support or not. - public void setSupportError(final String supportError) { - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); - } - @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); From 441faae02b527966f87b9f34f7c1e02ba80ebd5f Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 14:44:42 +0800 Subject: [PATCH 06/62] fix bug --- .../beam/spi/record/AvroRecordBuilder.java | 9 ------- .../spi/record/AvroRecordBuilderTest.java | 9 ++++++- .../component/runtime/record/RecordImpl.java | 27 ++++--------------- .../runtime/record/RecordBuilderImplTest.java | 12 +++++++-- .../sample/feature/supporterror/Cli.java | 7 ++--- .../supporterror/SupportErrorInput.java | 8 +++++- .../feature/supporterror/Messages.properties | 3 ++- 7 files changed, 34 insertions(+), 41 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java index 35fd6082bd89c..0ce9e2c4c21b6 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java @@ -35,13 +35,4 @@ public AvroRecordBuilder(final Schema providedSchema) { public Record build() { return new AvroRecord(super.build()); } - - protected Record.Builder getErrorBuilder(final Schema.Builder builder) { - - return new AvroRecordBuilder(builder.build()); - } - - protected Schema.Builder getErrorBuilder(final Schema.Type type) { - return new AvroSchemaBuilder().withType(type); - } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index bbfc3eda1e6b7..c3f5c9e8b5c7e 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -221,16 +221,23 @@ void testWithError() { .withNullable(false) .withType(Schema.Type.STRING) .build(); + Schema.Entry nmEntry = factory + .newEntryBuilder() + .withName("normal") + .withNullable(true) + .withType(Schema.Type.STRING) + .build(); Schema.Entry ageEntry = factory .newEntryBuilder() .withName("age") .withNullable(false) .withType(Schema.Type.INT) .build(); - Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(ageEntry).build(); + Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); // record 1 Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); Record record1 = recordBuilder.withError("name", null, "Stirng is null", null) + .withString("normal", "normal") .withError("age", "string", "is not an int", null) .build(); assertFalse(record1.isValid()); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 82b05b3ed618f..f91d8e0b17e7e 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -61,10 +61,10 @@ import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.EntriesOrder; import org.talend.sdk.component.api.record.Schema.Entry; +import org.talend.sdk.component.api.record.SchemaProperty; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.talend.sdk.component.api.record.SchemaProperty; @EqualsAndHashCode public final class RecordImpl implements Record { @@ -515,9 +515,10 @@ public Builder withArray(final Schema.Entry entry, final Collection value } @Override - public Builder withError(String columnName, Object value, String errorMessage, Exception exception) { + public Builder withError(final String columnName, final Object value, final String errorMessage, + final Exception exception) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); - if (!supportError) { + if (!supportError) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry @@ -531,28 +532,10 @@ public Builder withError(String columnName, Object value, String errorMessage, E .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) .withProp(SchemaProperty.ERROR_EXCEPTION, exception == null ? "" : exception.toString()) .build(); - final Schema.Builder builder = getErrorBuilder(providedSchema.getType()); - providedSchema.getAllEntries() -// .filter(e -> Objects.equals(providedSchema.getEntry(e.getName()), e)) - .forEach(e -> { - if (columnName.equals(e.getName())) { - builder.withEntry(updateEntry); - } else { - builder.withEntry(e); - } - }); - return getErrorBuilder(builder); + return updateEntryByName(columnName, updateEntry); } } - protected Builder getErrorBuilder(Schema.Builder builder) { - return new BuilderImpl(builder.build()); - } - - protected Schema.Builder getErrorBuilder(Schema.Type type) { - return new SchemaImpl.BuilderImpl().withType(type); - } - private void assertType(final Schema.Type actual, final Schema.Type expected) { if (actual != expected) { throw new IllegalArgumentException("Expected entry type: " + expected + ", got: " + actual); diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 9cddee216ca8b..6c8fea7482602 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -258,6 +258,11 @@ void withError() { .withNullable(false) .withType(Schema.Type.DATETIME) .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("normal") + .withNullable(true) + .withType(Type.STRING) + .build()) .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() .withName("intValue") .withNullable(false) @@ -269,14 +274,17 @@ void withError() { final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); final Record record3 = builder3.withError("date", null, "date is null", null) + .withString("normal", "normal") .withError("intValue", "string", "wrong int value", null) .build(); assertFalse(record3.isValid()); - final Entry entry = record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); + final Entry entry = + record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); assertNotNull(entry); Assertions.assertFalse(entry.isValid()); - final Entry entry2 = record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); + final Entry entry2 = + record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 608422e334bee..780d19a007dc2 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -50,7 +50,7 @@ public final class Cli implements Callable { //support errors or not. default=false - @Option(names = "-s", defaultValue = "false") + @Option(names = "-s", defaultValue = "true") boolean support; @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") @@ -81,10 +81,6 @@ public Integer call() throws Exception { .orElseThrow(() -> new IllegalStateException( String.format("No mapper found for: %s/%s.", family, manager))); -// final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) -// .orElseThrow(() -> new IllegalStateException( -// String.format("No Processor found for: %s/%s.", family, manager))); - info("create input now."); SupportErrorInput seInput = new SupportErrorInput(null); @@ -94,6 +90,7 @@ public Integer call() throws Exception { Record data = seInput.data(); info("Record isValid = " + data.isValid()); + entryout(data, "name"); entryout(data, "date"); entryout(data, "age"); // diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index 99758408a25d8..9982aacf4641d 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -56,6 +56,11 @@ public SupportErrorInput(@Option("configuration") final InputConfig config) { public void init() { recordSchema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("name") + .withNullable(false) + .withType(Schema.Type.STRING) + .build()) .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() .withName("date") .withNullable(false) @@ -72,7 +77,8 @@ public void init() { @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - final Record record = builder.withError("date", null, "date is null", null) + final Record record = builder.withString("name", "example connector") + .withError("date", null, "date is null", null) .withError("age", "string", "wrong int value", null) .build(); diff --git a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties index aa81e7a64c5fa..0e589d31e4429 100644 --- a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties +++ b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties @@ -14,6 +14,7 @@ # Here you can change all your configuration display names to use more explicit labels # You can also translate your configuration by adding one file by local Messages_fr.properties for french for example supporterror.supportErrorInput._displayName=Support Error Input +supporterror.supportErrorAvroInput._displayName=Support Error Avro Input supporterror.SupportErrorMapper._displayName=Support Error Mapper supporterror.datastore.default._displayName=default supporterror.dataset.default._displayName=dataset @@ -24,4 +25,4 @@ Datastore.date._displayName=date Datastore.name._displayName=name Datastore.age._placeholder= Datastore.date._placeholder= -Datastore.name._placeholder= \ No newline at end of file +Datastore.name._placeholder= From de7129772d52d96325bae9573a3ef5d6407ead6d Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 16:33:26 +0800 Subject: [PATCH 07/62] Add new method to white list. --- .../talend/sdk/component/tools/validator/RecordValidator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java index 8254a0eec55c5..a7fed49240bdc 100644 --- a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java +++ b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java @@ -132,6 +132,7 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.Record$Builder.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", + "org.talend.sdk.component.api.record.Record$Builder.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)", "org.talend.sdk.component.api.record.RecordImpl.withNewSchema(org.talend.sdk.component.api.record.Schema)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.with(org.talend.sdk.component.api.record.Schema$Entry, java.lang.Object)", @@ -161,6 +162,7 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withBoolean(org.talend.sdk.component.api.record.Schema$Entry, boolean)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", - "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)")); + "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", + "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)")); } From d5ab53974bfa548afb1c8a26dbc1b4299648ba7b Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 16:47:32 +0800 Subject: [PATCH 08/62] Add "valid":true to serialization --- .../runtime/beam/spi/record/JsonSchemaSerializationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java index 607083f593ef2..1993f10373c98 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java @@ -49,7 +49,7 @@ void toJson() throws Exception { .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { final String json = jsonb.toJson(schema); assertEquals( - "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", json); } } @@ -71,7 +71,7 @@ void toJsonWithMeta() throws Exception { try (final Jsonb jsonb = JsonbBuilder .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { assertEquals( - "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", jsonb.toJson(schema)); } From 86ad7ee532fcbcc69637eb3d81b0dc37cde18e1c Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 19:59:58 +0800 Subject: [PATCH 09/62] Add "valid":true to serialization --- .../sdk/component/server/front/ActionResourceImplTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java index e357820eecc57..85bb91a1e62c7 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java @@ -199,7 +199,8 @@ void checkSchemaSerialization() { " \"metadata\":[\n ],\n \"props\":{\n\n },\n \"type\":\"STRING\"\n" + " },\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + - " \"props\":{\n\n },\n \"type\":\"ARRAY\"\n }\n ],\n \"metadata\":[\n" + + " \"props\":{\n\n },\n \"type\":\"ARRAY\",\n" + + " \"valid\":true\n }\n ],\n \"metadata\":[\n" + " ],\n \"props\":{\n \"talend.fields.order\":\"array\"\n },\n \"type\":\"RECORD\"\n}"; assertEquals(expected, schema); } @@ -238,7 +239,7 @@ void checkDiscoverProcessorSchema() { }, APPLICATION_JSON_TYPE), JsonObject.class); assertNotNull(guessed); final String expected = - "{\"entries\":[{\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\"},{\"comment\":\"field2 comment\",\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\"},{\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\"},{\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; + "{\"entries\":[{\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"comment\":\"field2 comment\",\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\",\"valid\":true},{\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; assertEquals(expected, guessed.toString()); } From 884420fac0728c4d0f5335f9179d6bd9cbf97d6a Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 09:49:27 +0800 Subject: [PATCH 10/62] Add "valid":true to serialization --- .../server/front/beam/BeamActionSerializationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java index 553bfe64c4870..e30e2286e2d3c 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java @@ -46,7 +46,8 @@ void checkSchemaSerialization() { + " \"entries\":[\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"STRING\"\n" + " },\n" + " \"metadata\":false,\n" + " \"name\":\"array\",\n" + " \"nullable\":false,\n" - + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\"\n" + " }\n" + " ],\n" + + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\",\n" + + " \"valid\":true\n }\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + " \"talend.fields.order\":\"array\"\n" + " },\n" + " \"type\":\"RECORD\"\n" + "}"; assertEquals(attended, schema); From a9450e319a10c7e94203c9f5865c2a342b1d403f Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 10:05:38 +0800 Subject: [PATCH 11/62] rename --- .../feature/supporterror/{SVersions.java => Versions.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/{SVersions.java => Versions.java} (87%) diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java similarity index 87% rename from sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java rename to sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java index 0ad88da683be1..8f19afc5c0e3e 100644 --- a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java +++ b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java @@ -16,8 +16,8 @@ package org.talend.sdk.component.sample.feature.supporterror; -public interface SVersions { +public interface Versions { - String KIT_VERSION = "${project.version}"; + String KIT_VERSION = "1.81.0-SNAPSHOT";//"${project.version}"; } From 727c51259f0682770bc62c5c85f7c85e12d605e7 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 10:21:33 +0800 Subject: [PATCH 12/62] revert junit which no need to change --- .../spi/record/AvroRecordBuilderTest.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index c3f5c9e8b5c7e..797b8b23f06e9 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -54,6 +54,7 @@ import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; +import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.record.SchemaImpl; import org.talend.sdk.component.runtime.record.SchemaImpl.EntryImpl; @@ -191,21 +192,21 @@ void avroTest() { @Test void mixedRecordTest() { - final Schema schema0 = new AvroSchemaBuilder()// - .withType(RECORD) // - .withEntry(dataEntry1) // - .withEntryBefore("data1", meta1) // - .withEntry(dataEntry2) // - .withEntryAfter("meta1", meta2) // + final AvroRecordBuilderFactoryProvider recordBuilderFactoryProvider = new AvroRecordBuilderFactoryProvider(); + System.setProperty("talend.component.beam.record.factory.impl", "avro"); + final RecordBuilderFactory recordBuilderFactory = recordBuilderFactoryProvider.apply("test"); + + final RecordBuilderFactory otherFactory = new RecordBuilderFactoryImpl("test"); + final Schema schema = otherFactory + .newSchemaBuilder(RECORD) + .withEntry(otherFactory.newEntryBuilder().withName("e1").withType(INT).build()) .build(); - final Record.Builder builder0 = factory.newRecordBuilder(schema0); - builder0.withInt("data1", 101) - .withString("data2", "102") - .withInt("meta1", 103) - .withString("meta2", "104"); - final Record record0 = builder0.build(); - assertEquals(101, record0.getInt("data1")); + final Schema arrayType = recordBuilderFactory // + .newSchemaBuilder(Schema.Type.ARRAY) // + .withElementSchema(schema) + .build(); + Assertions.assertNotNull(arrayType); } @Test From e2dbb27f4c173a4427f6c678b0ec6d7802428c79 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 10:24:48 +0800 Subject: [PATCH 13/62] fix one problem --- .../sample/feature/supporterror/Cli.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 780d19a007dc2..bb36e2684db20 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -15,7 +15,6 @@ */ package org.talend.sdk.component.sample.feature.supporterror; - import static java.util.Optional.ofNullable; import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; @@ -25,6 +24,7 @@ import picocli.CommandLine.Option; import java.io.File; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; @@ -42,7 +42,6 @@ import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; - import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) @@ -50,7 +49,7 @@ public final class Cli implements Callable { //support errors or not. default=false - @Option(names = "-s", defaultValue = "true") + @Option(names = "-s", defaultValue = "false") boolean support; @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") @@ -90,9 +89,9 @@ public Integer call() throws Exception { Record data = seInput.data(); info("Record isValid = " + data.isValid()); - entryout(data, "name"); - entryout(data, "date"); - entryout(data, "age"); + entryout(data, "name", String.class); + entryout(data, "date", Date.class); + entryout(data, "age", Integer.class); // info("finished."); } catch (Exception e) { @@ -102,12 +101,11 @@ public Integer call() throws Exception { return 0; } - private static void entryout(final Record data, final String column) { + private static void entryout(final Record data, final String column, final Class type) { Schema.Entry ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny().get(); if(ageEntry.isValid()) { - Integer age = data.get(Integer.class, column); - // process the age... - info("Record '"+ column +"': " + age); + Object value = data.get(type, column); + info("Record '"+ column +"': " + value); } else{ String errorMessage = ageEntry.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); info("ERROR: " + errorMessage); From d2cf309bc59b0b3e96ec264b7d15fd477c24fd9d Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 15:42:53 +0800 Subject: [PATCH 14/62] fix import order --- .../sdk/component/sample/feature/supporterror/Cli.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index bb36e2684db20..5e95c5fb2008f 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,10 +19,6 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - import java.io.File; import java.util.Date; import java.util.HashMap; @@ -42,6 +38,10 @@ import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) From 02a5ebddbe3f3ac72bfd8315b0e6810b013ebff6 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 16:10:05 +0800 Subject: [PATCH 15/62] fix import order --- .../sdk/component/sample/feature/supporterror/Cli.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 5e95c5fb2008f..bb36e2684db20 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,6 +19,10 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + import java.io.File; import java.util.Date; import java.util.HashMap; @@ -38,10 +42,6 @@ import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) From 92df0c0abc931575b6f57d077b425be0e7907e57 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 8 May 2025 14:19:52 +0800 Subject: [PATCH 16/62] change command jar --- .../sample-features/supporterror/pom.xml | 9 +-- .../sample/feature/supporterror/Cli.java | 59 ++++++++++--------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/sample-parent/sample-features/supporterror/pom.xml b/sample-parent/sample-features/supporterror/pom.xml index 4f1279e001ebb..b5a57cfc4f32e 100644 --- a/sample-parent/sample-features/supporterror/pom.xml +++ b/sample-parent/sample-features/supporterror/pom.xml @@ -33,19 +33,16 @@ org.talend.sdk.component component-api ${project.version} - compile org.talend.sdk.component component-runtime-manager ${project.version} - compile - info.picocli - picocli - 4.7.6 - compile + org.tomitribe + tomitribe-crest + 0.32 diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index bb36e2684db20..7de7599cc09cd 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,15 +19,11 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; import java.io.File; import java.util.Date; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import org.talend.sdk.component.api.record.Record; @@ -41,31 +37,39 @@ import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; +import org.tomitribe.crest.Main; +import org.tomitribe.crest.api.Command; +import org.tomitribe.crest.api.Default; +import org.tomitribe.crest.api.Option; import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) -@Command(name="supportError") -public final class Cli implements Callable { - - //support errors or not. default=false - @Option(names = "-s", defaultValue = "false") - boolean support; - - @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") - File jar; - - @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") - String mapper; - - @Option(names = { "-fl", "--family" },defaultValue = "supporterror") - String family; +public final class Cli { + +// //support errors or not. default=false +// @Option(names = "-s", defaultValue = "false") +// boolean support; +// +// @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") +// File jar; +// +// @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") +// String mapper; +// +// @Option(names = { "-fl", "--family" },defaultValue = "supporterror") +// String family; static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + Versions.KIT_VERSION; - @Override - public Integer call() throws Exception { + @Command("supporterror") + public static void runInput( + @Option("gav") @Default(GAV) final String gav, + @Option("s") @Default("false") final boolean support, + @Option("jar") final File jar, + @Option("family") @Default("supporterror") final String family, + @Option("mapper") @Default("SupportErrorMapper") final String mapper) { try (final ComponentManager manager = manager(jar, GAV)) { // final JsonObject jsonConfig = readJsonFromFile(configurationFile); @@ -97,8 +101,6 @@ public Integer call() throws Exception { } catch (Exception e) { error(e); } - - return 0; } private static void entryout(final Record data, final String column, final Class type) { @@ -113,15 +115,18 @@ private static void entryout(final Record data, final String column, final Class } //set support or not. - public void setSupportError(final boolean supportError) { + public static void setSupportError(final boolean supportError) { final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); } - public static void main(final String... args) { - int exitCode = new CommandLine(new Cli()).execute(args); - System.exit(exitCode); + public static void main(final String[] args) throws Exception { + ofNullable(run(args)).ifPresent(System.out::println); + } + + public static Object run(final String[] args) throws Exception { + return new Main(Cli.class).exec(args); } static final String ERROR = "[ERROR] "; From fb6a8911fabb82b34f22da9fb9981dcda0cd0011 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 8 May 2025 15:38:33 +0800 Subject: [PATCH 17/62] fix sonar --- .../sdk/component/api/record/Record.java | 4 +- .../sample/feature/supporterror/Cli.java | 40 ++++++++----------- .../supporterror/SupportErrorInput.java | 5 +-- .../supporterror/SupportErrorMapper.java | 2 +- 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 8fceaf047897a..06a3e4c5956b7 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -315,9 +315,7 @@ default Optional getOptionalRecord(final String name) { default boolean isValid() { return !getSchema().getAllEntries() - .filter(entry -> !entry.isValid()) - .findAny() - .isPresent(); + .anyMatch(entry -> !entry.isValid()); } /** diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 7de7599cc09cd..26019ec792190 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -24,6 +24,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.talend.sdk.component.api.record.Record; @@ -32,7 +33,6 @@ import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; -import org.talend.sdk.component.runtime.input.PartitionMapperImpl; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; @@ -72,18 +72,12 @@ public static void runInput( @Option("mapper") @Default("SupportErrorMapper") final String mapper) { try (final ComponentManager manager = manager(jar, GAV)) { - // final JsonObject jsonConfig = readJsonFromFile(configurationFile); final Map configuration = new HashMap<>(); - info("support" + String.valueOf(support)); + info("support " + support); if (support) { setSupportError(support); } - // create the mapper - final PartitionMapperImpl mpr = (PartitionMapperImpl)manager.findMapper(family, mapper, 1, configuration) - .orElseThrow(() -> new IllegalStateException( - String.format("No mapper found for: %s/%s.", family, manager))); - info("create input now."); SupportErrorInput seInput = new SupportErrorInput(null); @@ -93,9 +87,9 @@ public static void runInput( Record data = seInput.data(); info("Record isValid = " + data.isValid()); - entryout(data, "name", String.class); - entryout(data, "date", Date.class); - entryout(data, "age", Integer.class); + entryOut(data, "name", String.class); + entryOut(data, "date", Date.class); + entryOut(data, "age", Integer.class); // info("finished."); } catch (Exception e) { @@ -103,22 +97,22 @@ public static void runInput( } } - private static void entryout(final Record data, final String column, final Class type) { - Schema.Entry ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny().get(); - if(ageEntry.isValid()) { - Object value = data.get(type, column); - info("Record '"+ column +"': " + value); - } else{ - String errorMessage = ageEntry.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); - info("ERROR: " + errorMessage); + private static void entryOut(final Record data, final String column, final Class type) { + Optional ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny(); + if(ageEntry.isPresent()) { + if (ageEntry.get().isValid()) { + Object value = data.get(type, column); + info("Record '" + column + "': " + value); + } else { + String errorMessage = ageEntry.get().getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); + info("ERROR: " + errorMessage); + } } } //set support or not. public static void setSupportError(final boolean supportError) { - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); } public static void main(final String[] args) throws Exception { @@ -174,9 +168,9 @@ public void close() { public static class DynamicContainerFinder implements ContainerFinder { - public static final Map LOADERS = new ConcurrentHashMap<>(); + static final Map LOADERS = new ConcurrentHashMap<>(); - public static final Map, Object> SERVICES = new ConcurrentHashMap<>(); + static final Map, Object> SERVICES = new ConcurrentHashMap<>(); @Override public LightContainer find(final String plugin) { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index 9982aacf4641d..13fe1001201cc 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -42,7 +42,6 @@ @Emitter(family = "supporterror", name = "supportErrorInput") public class SupportErrorInput implements Serializable { -// private final transient JsonBuilderFactory factory; private transient Schema recordSchema; @@ -77,12 +76,10 @@ public void init() { @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - final Record record = builder.withString("name", "example connector") + return builder.withString("name", "example connector") .withError("date", null, "date is null", null) .withError("age", "string", "wrong int value", null) .build(); - - return record; } @Data diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java index 3aa4210aae676..50f078aec5147 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java @@ -35,7 +35,7 @@ @Documentation("Doc: default SupportErrorMapper documentation without Internationalization.") public class SupportErrorMapper implements Serializable { - private SupportErrorInput.InputConfig config; + private transient SupportErrorInput.InputConfig config; public SupportErrorMapper(final @Option("configin") SupportErrorInput.InputConfig config) { this.config = config; From 7314bfba170bf802a913e093ba79f8f2dded6963 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 8 May 2025 15:44:14 +0800 Subject: [PATCH 18/62] remove comment --- .../component/sample/feature/supporterror/Cli.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 26019ec792190..c70fce4534add 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,7 +19,6 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; - import java.io.File; import java.util.Date; import java.util.HashMap; @@ -47,19 +46,6 @@ @NoArgsConstructor(access = PRIVATE) public final class Cli { -// //support errors or not. default=false -// @Option(names = "-s", defaultValue = "false") -// boolean support; -// -// @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") -// File jar; -// -// @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") -// String mapper; -// -// @Option(names = { "-fl", "--family" },defaultValue = "supporterror") -// String family; - static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + Versions.KIT_VERSION; From 01389f5db64e18dd706d4969500b27472bd03d12 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Fri, 9 May 2025 13:25:00 +0800 Subject: [PATCH 19/62] add readme --- .../sample-features/supporterror/README.md | 89 +++++++++++++++++++ .../sample/feature/supporterror/Versions.java | 2 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 sample-parent/sample-features/supporterror/README.md diff --git a/sample-parent/sample-features/supporterror/README.md b/sample-parent/sample-features/supporterror/README.md new file mode 100644 index 0000000000000..6b5756f4d20ff --- /dev/null +++ b/sample-parent/sample-features/supporterror/README.md @@ -0,0 +1,89 @@ +# Component Runtime :: Sample Feature :: SupportError + +## Table of Contents +- [Overview](#overview) +- [Usage](#usage) + - [How to build the sample connector plugin](#how-to-build-the-sample-connector-plugin) + - [How to run](#how-to-run) + - [How to use](#how-to-use) +- [Plugin artifact](#plugin-artifact) +- [Execution examples](#execution-examples) + - [Run supporterror connector with default behavior](#run-checkpoint-connector-with-default-behavior) + - [Run with framework feature disabled](#run-with-framework-feature-disabled) + + +## Overview +This is a simple TCK connector plugin to test and validate the feature about Improve Record API to add error info for invalid entries. + +This project contains two things: + 1. A sample connector plugin that implements the supporterror feature. + 2. A simple Cli runner that can be used to run the connector plugin and test the supporterror feature. + + +## Usage +### How to build the sample connector plugin +Checkout the code from the repository and build the project using `mvn clean install` +Alternatively build the feature module using `mvn install -am -pl :supporterror` + +### How to run +To run the connector, you need exec the generated artifact `org.talend.sdk.component.sample.feature:supporterror`. +* You can run it directly from `target` folder or repository folder + * `java -jar target/supporterror-1.81.0-SNAPSHOT.jar` +* or you can run it from the maven repository + * `java -jar ~/.m2/repository/org/talend/sdk/component/sample/feature/supporterror/1.81.0-SNAPSHOT/supporterror-1.81.0-SNAPSHOT.jar` + +For later usage, will use the variable `$RUNCMD` as the way you may choose. +⚠️ If you're using jdk17, don't forget to add the `--add-opens` option to the command line +(see profile _jdk9_ in master pom at the repository's root) or use instead jdk11. + +### How to use +Run supporterror with the option "-s" for execution: + +```bash +$ $RUNCMD supporterror + +Usage: supporterror [-s] + +Options: + --s use supporterror or not. + +``` + +## Plugin artifact +There are two ways to run the supporterror runner with a specific plugin artifact: +- Using the `--gav` option to specify the GAV of the plugin artifact. + Syntax: `groupId:artifactId:version[:packaging[:classifier]]` + +- Using the `--jar` option to specify the path to the plugin artifact. + Syntax: `/path/to/plugin.jar` + +⚠️ You cannot use both options at the same time. + +## Execution examples +### Run supporterror connector with default behavior +`java -jar target/supporterror-1.81.0-SNAPSHOT.jar supporterror -s` + +```bash +[INFO] Manager is using plugin: supporterror from GAV org.talend.sdk.component.sample.feature:supporterror:jar:1.81.0-SNAPSHOT. +[INFO] support true +[INFO] create input now. +[INFO] getting the record. +[INFO] Record isValid = false +[INFO] Record 'name': example connector +[INFO] ERROR: date is null +[INFO] ERROR: wrong int value +[INFO] finished. +``` + +### Run with framework feature disabled +In this example we turn off the supporterror capability. +`% java -jar target/supporterror-1.81.0-SNAPSHOT.jar supporterror` + +```bash +[INFO] Manager is using plugin: supporterror from GAV org.talend.sdk.component.sample.feature:supporterror:jar:1.81.0-SNAPSHOT. +[INFO] support false +[INFO] create input now. +[INFO] getting the record. +[ERROR] date is null +``` +We can see that no record is returned because throw errors. diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java index 8f19afc5c0e3e..242683a0ce5e8 100644 --- a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java +++ b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java @@ -18,6 +18,6 @@ public interface Versions { - String KIT_VERSION = "1.81.0-SNAPSHOT";//"${project.version}"; + String KIT_VERSION = "${project.version}"; } From 788408606edde127bd417c698b2d0a384f008a3f Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 12 May 2025 15:11:07 +0800 Subject: [PATCH 20/62] remove withError in record API --- .../sdk/component/api/record/Record.java | 4 +- .../spi/record/AvroRecordBuilderTest.java | 6 +- .../component/runtime/record/RecordImpl.java | 24 +++---- .../runtime/record/RecordBuilderImplTest.java | 70 ++++++++++++++----- .../tools/validator/RecordValidator.java | 4 +- .../sample/feature/supporterror/Cli.java | 3 - .../supporterror/SupportErrorInput.java | 58 ++++++++------- .../supporterror/SupportErrorMapper.java | 1 - 8 files changed, 102 insertions(+), 68 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 06a3e4c5956b7..341eb178d0783 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -34,7 +34,7 @@ public interface Record { - String RECORD_ERROR_SUPPORT = "talend.sdk.runtime.record.error.support"; + String RECORD_ERROR_SUPPORT = "talend.component.record.error.support"; /** * @return the schema of this record. @@ -454,7 +454,5 @@ default Builder withInstant(Schema.Entry entry, Instant value) { Builder withRecord(String name, Record value); Builder withArray(Schema.Entry entry, Collection values); - - Builder withError(String columnName, Object value, String errorMessage, Exception exception); } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 797b8b23f06e9..4a7078b7b0d91 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -237,9 +237,9 @@ void testWithError() { Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); // record 1 Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record1 = recordBuilder.withError("name", null, "Stirng is null", null) - .withString("normal", "normal") - .withError("age", "string", "is not an int", null) + Record record1 = recordBuilder.with(nameEntry, null) + .with(nmEntry, "normal") + .with(ageEntry, "is not an int") .build(); assertFalse(record1.isValid()); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index f91d8e0b17e7e..841c43abfb187 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -172,9 +172,13 @@ public Object getValue(final String name) { @Override public Builder with(final Entry entry, final Object value) { - validateTypeAgainstProvidedSchema(entry.getName(), entry.getType(), value); + try { + validateTypeAgainstProvidedSchema(entry.getName(), entry.getType(), value); + } catch (Exception e) { + return withError(entry, value, e.getMessage()); + } if (!entry.getType().isCompatible(value)) { - throw new IllegalArgumentException(String + return withError(entry, value, String .format("Entry '%s' of type %s is not compatible with value of type '%s'", entry.getName(), entry.getType(), value.getClass().getName())); } @@ -514,25 +518,21 @@ public Builder withArray(final Schema.Entry entry, final Collection value return append(entry, values); } - @Override - public Builder withError(final String columnName, final Object value, final String errorMessage, - final Exception exception) { + private Builder withError(final Entry entry, final Object value, final String errorMessage) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); if (!supportError) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry - final Entry oldEntry = this.findExistingEntry(columnName); - final Entry updateEntry = oldEntry.toBuilder() - .withName(columnName) + final Entry updateEntry = entry.toBuilder() + .withName(entry.getName()) .withNullable(true) - .withType(oldEntry.getType()) + .withType(entry.getType()) .withProp(SchemaProperty.ENTRY_IS_ON_ERROR, "true") .withProp(SchemaProperty.ENTRY_ERROR_MESSAGE, errorMessage) .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) - .withProp(SchemaProperty.ERROR_EXCEPTION, exception == null ? "" : exception.toString()) - .build(); - return updateEntryByName(columnName, updateEntry); + .build(); + return updateEntryByName(entry.getName(), updateEntry); } } diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 6c8fea7482602..42a3fb0538aa3 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -251,31 +251,43 @@ void dateTime() { @Test void withError() { + Entry dateNull = new EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Type.DATETIME) + .build(); + Entry date = new EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Type.DATETIME) + .build(); + Entry normal = new EntryImpl.BuilderImpl() + .withName("normal") + .withNullable(true) + .withType(Type.STRING) + .build(); + Entry intEntry = new EntryImpl.BuilderImpl() + .withName("intValue") + .withNullable(false) + .withType(Type.INT) + .build(); final Schema schema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Schema.Type.DATETIME) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("normal") - .withNullable(true) - .withType(Type.STRING) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("intValue") - .withNullable(false) - .withType(Type.INT) - .build()) + .withEntry(dateNull) + .withEntry(date) + .withEntry(normal) + .withEntry(intEntry) .build(); final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); - final Record record3 = builder3.withError("date", null, "date is null", null) - .withString("normal", "normal") - .withError("intValue", "string", "wrong int value", null) + + final Record record3 = builder3 + .with(dateNull, null) + .with(date, "not a date") + .with(normal, "normal") + .with(intEntry, "wrong int value") .build(); assertFalse(record3.isValid()); final Entry entry = @@ -288,9 +300,31 @@ void withError() { assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); + final Entry entry3 = + record3.getSchema().getEntries().stream().filter(e -> "normal".equals(e.getName())).findAny().get(); + assertNotNull(entry3); + Assertions.assertTrue(entry3.isValid()); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); } + @Test + void testWithWrongEntryType() { + Entry entry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build(); + final Schema schema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(entry) + .build(); + final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(schema); + assertNotNull(builder.getEntry("date")); + + assertThrows(IllegalArgumentException.class, () -> builder.with(entry, "String")); + } + @Test void zonedDateTimeVSInstant() { final Schema schema = new SchemaImpl.BuilderImpl() diff --git a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java index a7fed49240bdc..8254a0eec55c5 100644 --- a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java +++ b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java @@ -132,7 +132,6 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.Record$Builder.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", - "org.talend.sdk.component.api.record.Record$Builder.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)", "org.talend.sdk.component.api.record.RecordImpl.withNewSchema(org.talend.sdk.component.api.record.Schema)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.with(org.talend.sdk.component.api.record.Schema$Entry, java.lang.Object)", @@ -162,7 +161,6 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withBoolean(org.talend.sdk.component.api.record.Schema$Entry, boolean)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", - "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", - "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)")); + "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)")); } diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index c70fce4534add..49aa2f1f4a5ab 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -21,7 +21,6 @@ import java.io.File; import java.util.Date; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -58,8 +57,6 @@ public static void runInput( @Option("mapper") @Default("SupportErrorMapper") final String mapper) { try (final ComponentManager manager = manager(jar, GAV)) { - final Map configuration = new HashMap<>(); - info("support " + support); if (support) { setSupportError(support); diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index 13fe1001201cc..c9c3d61577ff2 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,9 +42,14 @@ @Emitter(family = "supporterror", name = "supportErrorInput") public class SupportErrorInput implements Serializable { - private transient Schema recordSchema; + private transient Schema.Entry nameEntry; + + private transient Schema.Entry dateEntry; + + private transient Schema.Entry ageEntry; + private final transient InputConfig configuration; public SupportErrorInput(@Option("configuration") final InputConfig config) { @@ -53,32 +58,35 @@ public SupportErrorInput(@Option("configuration") final InputConfig config) { @PostConstruct public void init() { + nameEntry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("name") + .withNullable(false) + .withType(Schema.Type.STRING) + .build(); + dateEntry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build(); + ageEntry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("age") + .withNullable(false) + .withType(Schema.Type.INT) + .build(); recordSchema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("name") - .withNullable(false) - .withType(Schema.Type.STRING) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Schema.Type.DATETIME) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("age") - .withNullable(false) - .withType(Schema.Type.INT) - .build()) + .withEntry(nameEntry) + .withEntry(dateEntry) + .withEntry(ageEntry) .build(); } @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - return builder.withString("name", "example connector") - .withError("date", null, "date is null", null) - .withError("age", "string", "wrong int value", null) + return builder.with(nameEntry, "example connector") + .with(dateEntry, "not a date value") + .with(ageEntry, "wrong int value") .build(); } @@ -86,7 +94,7 @@ public Record data() { @GridLayout(value = { @GridLayout.Row("dataset"), }) - @Version + @Version public static class InputConfig { @Option @@ -96,7 +104,7 @@ public static class InputConfig { @DataSet @Data - @GridLayout(value = { @GridLayout.Row("datastore") }) + @GridLayout(value = {@GridLayout.Row("datastore")}) public static class Dataset implements Serializable { @Option @@ -107,7 +115,7 @@ public static class Dataset implements Serializable { @DataStore @Data - @GridLayout(value = { @GridLayout.Row("name"), @GridLayout.Row("age"), + @GridLayout(value = {@GridLayout.Row("name"), @GridLayout.Row("age"), @GridLayout.Row("date")}) public static class Datastore implements Serializable { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java index 50f078aec5147..db3d0506c9c98 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java @@ -53,7 +53,6 @@ public List split(@PartitionSize final int desiredNbSplits) @Emitter public SupportErrorInput createSource() { - return new SupportErrorInput(this.config); } From e9ea73667f24d2a9093eec48d72369b5b45c8532 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 12 May 2025 16:13:36 +0800 Subject: [PATCH 21/62] add getErrorMessage,getErrorFallbackValue in Entry API --- .../java/org/talend/sdk/component/api/record/Schema.java | 6 +++++- .../sdk/component/runtime/record/RecordBuilderImplTest.java | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java index da4b0806638be..7d4c814715ef4 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java @@ -233,7 +233,7 @@ interface Entry { boolean isMetadata(); /** - * @return true if the value of this entry is valid; false for invalid value. + * @return true if the value of this entry is valid; false for invalid value. */ boolean isValid(); @@ -301,6 +301,10 @@ default Entry.Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder is not implemented"); } + default String getErrorMessage() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)).orElse(null); } + + default String getErrorFallbackValue() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)).orElse(null); } + /** * Plain builder matching {@link Entry} structure. */ diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 42a3fb0538aa3..94d2d730cf2f5 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -294,11 +294,15 @@ void withError() { record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); assertNotNull(entry); Assertions.assertFalse(entry.isValid()); + assertEquals("Entry 'date' of type DATETIME is not compatible with value of type 'java.lang.String'", + entry.getErrorMessage()); final Entry entry2 = record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); + assertEquals("Entry 'intValue' of type INT is not compatible with value of type 'java.lang.String'", + entry2.getErrorMessage()); final Entry entry3 = record3.getSchema().getEntries().stream().filter(e -> "normal".equals(e.getName())).findAny().get(); From 8633ea1d4a60da975252461661bfc7ee49c46890 Mon Sep 17 00:00:00 2001 From: ypiel Date: Tue, 13 May 2025 05:21:44 +0200 Subject: [PATCH 22/62] feat(QTDI-1305): Fix RecordBuilder builder pattern +some improvements. (#1042) * feat(QTDI-1305): Fix RecordBuilder builder pattern +some improvements. * feat(QTDI-1305): rename sample supporterror to entry-with-error. --- .../sdk/component/api/record/Schema.java | 15 +- .../component/api/service/schema/Schema.java | 9 +- .../sdk/component/api/record/RecordTest.java | 9 +- .../sdk/component/api/record/SchemaTest.java | 9 +- .../record/RecordBuilderFactoryTest.java | 12 +- .../component/runtime/record/RecordImpl.java | 52 ++++-- .../component/runtime/record/SchemaImpl.java | 20 +++ .../runtime/record/RecordBuilderImplTest.java | 94 ++++++----- .../README.md | 2 +- .../pom.xml | 49 +++++- .../feature/entrywitherror}/Versions.java | 2 +- .../sample/feature/entrywitherror}/Cli.java | 92 +++++++---- .../RecordWithEntriesInErrorEmitter.java | 150 ++++++++++++++++++ .../feature/entrywitherror}/package-info.java | 6 +- ...nent.runtime.serialization.ContainerFinder | 0 .../src/main/resources/icons/dark/icon.svg | 0 .../src/main/resources/icons/dark/mapper.svg | 0 .../src/main/resources/icons/light/icon.svg | 0 .../src/main/resources/icons/light/mapper.svg | 0 .../entrywitherror}/Messages.properties | 17 +- sample-parent/sample-features/pom.xml | 2 +- .../supporterror/SupportErrorInput.java | 135 ---------------- .../supporterror/SupportErrorMapper.java | 59 ------- 23 files changed, 427 insertions(+), 307 deletions(-) rename sample-parent/sample-features/{supporterror => entry-with-error}/README.md (98%) rename sample-parent/sample-features/{supporterror => entry-with-error}/pom.xml (59%) rename sample-parent/sample-features/{supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror}/Versions.java (91%) rename sample-parent/sample-features/{supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror}/Cli.java (65%) create mode 100644 sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java rename sample-parent/sample-features/{supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror}/package-info.java (79%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/dark/icon.svg (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/dark/mapper.svg (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/light/icon.svg (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/light/mapper.svg (100%) rename sample-parent/sample-features/{supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror}/Messages.properties (58%) delete mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java delete mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java index 7d4c814715ef4..210d903cca1e4 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java @@ -232,6 +232,11 @@ interface Entry { */ boolean isMetadata(); + /** + * @return Is this entry can be in error. + */ + boolean isErrorCapable(); + /** * @return true if the value of this entry is valid; false for invalid value. */ @@ -301,9 +306,13 @@ default Entry.Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder is not implemented"); } - default String getErrorMessage() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)).orElse(null); } + default String getErrorMessage() { + return getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); + } - default String getErrorFallbackValue() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)).orElse(null); } + default String getErrorFallbackValue() { + return getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE); + } /** * Plain builder matching {@link Entry} structure. @@ -326,6 +335,8 @@ default Builder withLogicalType(String logicalType) { Builder withNullable(boolean nullable); + Builder withErrorCapable(boolean errorCapable); + Builder withMetadata(boolean metadata); Builder withDefaultValue(T value); diff --git a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java index 433673db4b170..5192dc9f5c4c5 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java @@ -129,6 +129,11 @@ public boolean isNullable() { return true; } + @Override + public boolean isErrorCapable() { + return false; + } + @Override public boolean isMetadata() { return false; @@ -165,7 +170,9 @@ public Builder toBuilder() { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java index c07eda9008351..63a9852487f4b 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java @@ -106,6 +106,11 @@ public boolean isNullable() { return false; } + @Override + public boolean isErrorCapable() { + return false; + } + @Override public boolean isMetadata() { return false; @@ -137,7 +142,9 @@ public String getProp(String property) { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } }; Assertions.assertEquals("value", record.get(String.class, e1)); } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java index 11c5e325f3c9d..d1b8e6bb8094b 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java @@ -291,6 +291,11 @@ public boolean isNullable() { throw new UnsupportedOperationException("#isNullable()"); } + @Override + public boolean isErrorCapable() { + throw new UnsupportedOperationException("#isErrorCapable()"); + } + @Override public boolean isMetadata() { throw new UnsupportedOperationException("#isMetadata()"); @@ -327,7 +332,9 @@ public JsonValue getJsonProp(final String name) { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java index a34a6cc58d82f..ff2c4fba06b14 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java @@ -78,6 +78,8 @@ static class MockEntry implements Schema.Entry { private final boolean nullable; + private final boolean errorCapable; + private final boolean metadata; private final Object defaultVal; @@ -104,7 +106,9 @@ public Builder toBuilder() { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } } @RequiredArgsConstructor @@ -138,6 +142,12 @@ public Entry.Builder withNullable(boolean nullable) { return this; } + @Override + public Entry.Builder withErrorCapable(boolean errorCapable) { + this.builder.withErrorCapable(errorCapable); + return this; + } + @Override public Entry.Builder withMetadata(boolean metadata) { this.builder.withMetadata(metadata); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 841c43abfb187..baa0ba2571401 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -130,6 +130,8 @@ public static class BuilderImpl implements Builder { private OrderState orderState; + private Map entriesInError = new HashMap<>(); + public BuilderImpl() { this(null); } @@ -151,7 +153,7 @@ private void initOrderState() { } else { final List fields = this.providedSchema.naturalOrder() .getFieldsOrder() - .map(this.providedSchema::getEntry) + .map(n -> this.getEntryWithErrorIfAny(this.providedSchema.getEntry(n))) .collect(Collectors.toList()); this.orderState = new OrderState(fields); } @@ -179,8 +181,9 @@ public Builder with(final Entry entry, final Object value) { } if (!entry.getType().isCompatible(value)) { return withError(entry, value, String - .format("Entry '%s' of type %s is not compatible with value of type '%s'", entry.getName(), - entry.getType(), value.getClass().getName())); + .format("Entry '%s' of type %s is not compatible with given value of type '%s': '%s'.", + entry.getName(), + entry.getType(), value.getClass().getName(), value)); } if (entry.getType() == Schema.Type.DATETIME) { @@ -328,25 +331,53 @@ private Schema.Entry validateTypeAgainstProvidedSchema(final String name, final return entry; } + /** + * This method return the updated entry with error information if any. + * + * @param e The entry to check. + * @return The entry updated with error information or the given one. + */ + private Entry getEntryWithErrorIfAny(final Entry e) { + if (!e.isErrorCapable()) { + // The entry doesn't support error management + return e; + } + + return entriesInError.getOrDefault(e.getOriginalFieldName(), e); + } + public Record build() { final Schema currentSchema; if (this.providedSchema != null) { final String missing = this.providedSchema .getAllEntries() + .map(this::getEntryWithErrorIfAny) .filter(it -> !it.isNullable() && !values.containsKey(it.getName())) .map(Schema.Entry::getName) .collect(joining(", ")); if (!missing.isEmpty()) { throw new IllegalArgumentException("Missing entries: " + missing); } + + Schema schemaWithErrors = this.providedSchema; + if (!this.entriesInError.isEmpty()) { + Schema.Builder schemaBuilder = new SchemaImpl.BuilderImpl() + .withType(this.providedSchema.getType()); + this.providedSchema.getEntries() + .stream() + .map(this::getEntryWithErrorIfAny) + .forEach(schemaBuilder::withEntry); + schemaWithErrors = schemaBuilder.build(); + } + if (orderState != null && orderState.isOverride()) { - currentSchema = this.providedSchema.toBuilder().build(this.orderState.buildComparator()); + currentSchema = schemaWithErrors.toBuilder().build(this.orderState.buildComparator()); } else { - currentSchema = this.providedSchema; + currentSchema = schemaWithErrors; } } else { final Schema.Builder builder = new SchemaImpl.BuilderImpl().withType(RECORD); - this.entries.forEachValue(builder::withEntry); + this.entries.streams().map(this::getEntryWithErrorIfAny).forEach(builder::withEntry); initOrderState(); currentSchema = builder.build(orderState.buildComparator()); } @@ -520,19 +551,20 @@ public Builder withArray(final Schema.Entry entry, final Collection value private Builder withError(final Entry entry, final Object value, final String errorMessage) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); - if (!supportError) { + if (!supportError || !entry.isErrorCapable()) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry - final Entry updateEntry = entry.toBuilder() + final Entry updatedEntry = entry.toBuilder() .withName(entry.getName()) .withNullable(true) .withType(entry.getType()) .withProp(SchemaProperty.ENTRY_IS_ON_ERROR, "true") .withProp(SchemaProperty.ENTRY_ERROR_MESSAGE, errorMessage) .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) - .build(); - return updateEntryByName(entry.getName(), updateEntry); + .build(); + this.entriesInError.put(updatedEntry.getOriginalFieldName(), updatedEntry); + return this; } } diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java index 6348c4d0f69f0..41886e2ee38b2 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java @@ -352,6 +352,7 @@ private EntryImpl(final EntryImpl.BuilderImpl builder) { this.type = builder.type; } this.nullable = builder.nullable; + this.errorCapable = builder.errorCapable; this.metadata = builder.metadata; this.defaultValue = builder.defaultValue; this.elementSchema = builder.elementSchema; @@ -379,6 +380,11 @@ private EntryImpl(final EntryImpl.BuilderImpl builder) { */ private final boolean nullable; + /** + * Is this entry can be in error. + */ + private final boolean errorCapable; + /** * Is this entry a metadata entry. */ @@ -440,6 +446,11 @@ public boolean isNullable() { return this.nullable; } + @Override + public boolean isErrorCapable() { + return this.errorCapable; + } + @Override public boolean isMetadata() { return this.metadata; @@ -487,6 +498,8 @@ public static class BuilderImpl implements Entry.Builder { private boolean nullable; + private boolean errorCapable; + private boolean metadata = false; private Object defaultValue; @@ -506,6 +519,7 @@ private BuilderImpl(final Entry entry) { this.name = entry.getName(); this.rawName = entry.getRawName(); this.nullable = entry.isNullable(); + this.errorCapable = entry.isErrorCapable(); this.type = entry.getType(); this.comment = entry.getComment(); this.elementSchema = entry.getElementSchema(); @@ -549,6 +563,12 @@ public Builder withNullable(final boolean nullable) { return this; } + @Override + public Builder withErrorCapable(final boolean errorCapable) { + this.errorCapable = errorCapable; + return this; + } + @Override public Builder withMetadata(final boolean metadata) { this.metadata = metadata; diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 94d2d730cf2f5..85e233722a835 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -249,67 +249,79 @@ void dateTime() { assertThrows(IllegalArgumentException.class, () -> builder2.withDateTime("date", (ZonedDateTime) null)); } + @Test + void withErrorWhenNotSupported() { + Assertions.assertThrows(IllegalArgumentException.class, () -> withError("false")); + } + @Test void withError() { - Entry dateNull = new EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Type.DATETIME) - .build(); - Entry date = new EntryImpl.BuilderImpl() + Record record = withError("true"); + + assertFalse(record.isValid()); + + final Entry retrievedDateEntry = record.getSchema().getEntry("date"); + assertNotNull(retrievedDateEntry); + Assertions.assertFalse(retrievedDateEntry.isValid()); + assertEquals( + "Entry 'date' of type DATETIME is not compatible with given value of type 'java.lang.String': 'not a date'.", + retrievedDateEntry.getErrorMessage()); + Assertions.assertNull(record.getDateTime("date")); + + final Entry retrievedIntEntry = record.getSchema().getEntry("intValue"); + assertNotNull(retrievedIntEntry); + Assertions.assertFalse(retrievedIntEntry.isValid()); + assertEquals( + "Entry 'intValue' of type INT is not compatible with given value of type 'java.lang.String': 'wrong int value'.", + retrievedIntEntry.getErrorMessage()); + Assertions.assertNull(record.getDateTime("intValue")); + + final Entry retrievedStringEntry = record.getSchema().getEntry("normal"); + assertNotNull(retrievedStringEntry); + Assertions.assertTrue(retrievedStringEntry.isValid()); + Assertions.assertEquals("No error", record.getString("normal")); + + } + + private Record withError(final String supported) { + final String errorSupportBackup = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, supported); + + Entry dateEntry = new EntryImpl.BuilderImpl() .withName("date") .withNullable(false) + .withErrorCapable(true) .withType(Type.DATETIME) .build(); - Entry normal = new EntryImpl.BuilderImpl() + Entry stringEntry = new EntryImpl.BuilderImpl() .withName("normal") .withNullable(true) + .withErrorCapable(true) .withType(Type.STRING) .build(); Entry intEntry = new EntryImpl.BuilderImpl() .withName("intValue") .withNullable(false) + .withErrorCapable(true) .withType(Type.INT) .build(); final Schema schema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) - .withEntry(dateNull) - .withEntry(date) - .withEntry(normal) + .withEntry(dateEntry) + .withEntry(stringEntry) .withEntry(intEntry) .build(); - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); - final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); - final Record record3 = builder3 - .with(dateNull, null) - .with(date, "not a date") - .with(normal, "normal") - .with(intEntry, "wrong int value") - .build(); - assertFalse(record3.isValid()); - final Entry entry = - record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); - assertNotNull(entry); - Assertions.assertFalse(entry.isValid()); - assertEquals("Entry 'date' of type DATETIME is not compatible with value of type 'java.lang.String'", - entry.getErrorMessage()); - - final Entry entry2 = - record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); - assertNotNull(entry2); - Assertions.assertFalse(entry2.isValid()); - assertEquals("Entry 'intValue' of type INT is not compatible with value of type 'java.lang.String'", - entry2.getErrorMessage()); - - final Entry entry3 = - record3.getSchema().getEntries().stream().filter(e -> "normal".equals(e.getName())).findAny().get(); - assertNotNull(entry3); - Assertions.assertTrue(entry3.isValid()); - - System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); + final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(schema); + + builder.with(stringEntry, "No error"); + builder.with(dateEntry, "not a date"); + builder.with(intEntry, "wrong int value"); + final Record record = builder.build(); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, errorSupportBackup == null ? "false" : errorSupportBackup); + + return record; } @Test diff --git a/sample-parent/sample-features/supporterror/README.md b/sample-parent/sample-features/entry-with-error/README.md similarity index 98% rename from sample-parent/sample-features/supporterror/README.md rename to sample-parent/sample-features/entry-with-error/README.md index 6b5756f4d20ff..a37bef27b3eeb 100644 --- a/sample-parent/sample-features/supporterror/README.md +++ b/sample-parent/sample-features/entry-with-error/README.md @@ -1,4 +1,4 @@ -# Component Runtime :: Sample Feature :: SupportError +# Component Runtime :: Sample Feature :: Entry with error ## Table of Contents - [Overview](#overview) diff --git a/sample-parent/sample-features/supporterror/pom.xml b/sample-parent/sample-features/entry-with-error/pom.xml similarity index 59% rename from sample-parent/sample-features/supporterror/pom.xml rename to sample-parent/sample-features/entry-with-error/pom.xml index b5a57cfc4f32e..b136e3b977d73 100644 --- a/sample-parent/sample-features/supporterror/pom.xml +++ b/sample-parent/sample-features/entry-with-error/pom.xml @@ -26,7 +26,7 @@ supporterror jar - Component Runtime :: Sample Feature :: Supporterror + Component Runtime :: Sample Feature :: Entry with error support @@ -56,6 +56,47 @@ 1.8 + + org.talend.sdk.component + talend-component-maven-plugin + ${project.version} + + + talend-component-validate + + validate + + process-classes + + true + true + false + true + true + true + true + true + true + false + + false + + true + true + true + true + true + true + true + true + true + true + true + true + + + + org.apache.maven.plugins maven-jar-plugin @@ -63,7 +104,7 @@ true - org.talend.sdk.component.sample.feature.supporterror.Cli + org.talend.sdk.component.sample.feature.entrywitherror.Cli @@ -80,7 +121,7 @@ - org.talend.sdk.component.sample.feature.supporterror.Cli + org.talend.sdk.component.sample.feature.entrywitherror.Cli @@ -93,7 +134,7 @@ exec-maven-plugin 3.0.0 - org.talend.sdk.component.sample.feature.supporterror.Cli + org.talend.sdk.component.sample.feature.entrywitherror.Cli diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java b/sample-parent/sample-features/entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror/Versions.java similarity index 91% rename from sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java rename to sample-parent/sample-features/entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror/Versions.java index 242683a0ce5e8..fef316911253f 100644 --- a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror/Versions.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.talend.sdk.component.sample.feature.supporterror; +package org.talend.sdk.component.sample.feature.entrywitherror; public interface Versions { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java similarity index 65% rename from sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java rename to sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 49aa2f1f4a5ab..9570ec019fc37 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -13,24 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.talend.sdk.component.sample.feature.supporterror; +package org.talend.sdk.component.sample.feature.entrywitherror; import static java.util.Optional.ofNullable; import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; import java.io.File; -import java.util.Date; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.talend.sdk.component.api.record.Record; -import org.talend.sdk.component.api.record.Schema; -import org.talend.sdk.component.api.record.SchemaProperty; +import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; +import org.talend.sdk.component.runtime.input.InputImpl; +import org.talend.sdk.component.runtime.input.Mapper; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; @@ -51,49 +52,74 @@ public final class Cli { @Command("supporterror") public static void runInput( @Option("gav") @Default(GAV) final String gav, - @Option("s") @Default("false") final boolean support, + @Option("support") @Default("false") final boolean support, + @Option("gen-some-errors") @Default("true") final boolean genErrors, + @Option("gen-nbrecords") @Default("10") final int nbRecords, @Option("jar") final File jar, @Option("family") @Default("supporterror") final String family, @Option("mapper") @Default("SupportErrorMapper") final String mapper) { - try (final ComponentManager manager = manager(jar, GAV)) { - info("support " + support); - if (support) { - setSupportError(support); - } - info("create input now."); - - SupportErrorInput seInput = new SupportErrorInput(null); - seInput.init(); + info("support " + support); + if (support) { + setSupportError(support); + } - info("getting the record."); - Record data = seInput.data(); + Map config = new HashMap<>(); + config.put("configuration.generateErrors", String.valueOf(genErrors)); + config.put("configuration.nbRecords", String.valueOf(nbRecords)); + run(jar, gav, config, "sampleRecordWithEntriesInError", "RecordWithEntriesInErrorEmitter"); + } - info("Record isValid = " + data.isValid()); - entryOut(data, "name", String.class); - entryOut(data, "date", Date.class); - entryOut(data, "age", Integer.class); - // + private static void run(final File jar, final String gav, final Map configuration, + final String family, final String mapper) { + try (final ComponentManager manager = manager(jar, gav)) { + info("configuration: " + configuration); + + // create the mapper + final Mapper mpr = manager.findMapper(family, mapper, 1, configuration) + .orElseThrow(() -> new IllegalStateException( + String.format("No mapper found for: %s/%s.", family, manager))); + + List mappers = mpr.split(1); + Record data; + + int count = 0; + for (Mapper currentMapper : mappers) { + final InputImpl input = InputImpl.class.cast(currentMapper.create()); + input.start(); + while ((data = (Record) input.next()) != null) { + count++; + recordOut(count, data); + } + input.stop(); + } info("finished."); } catch (Exception e) { error(e); } } - private static void entryOut(final Record data, final String column, final Class type) { - Optional ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny(); - if(ageEntry.isPresent()) { - if (ageEntry.get().isValid()) { - Object value = data.get(type, column); - info("Record '" + column + "': " + value); - } else { - String errorMessage = ageEntry.get().getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); - info("ERROR: " + errorMessage); - } + private static void recordOut(final int count, final Record record) { + System.out.printf("Record no %s is valid ? %s\n", count, record.isValid() ? "yes" : "no"); + System.out.printf("\tName: %s\n", record.getString("name")); + Entry date = record.getSchema().getEntry("date"); + if (date.isValid()) { + System.out.printf("\tDate: %s\n", record.getDateTime("date")); + } else { + System.out.printf("\tDate is on error: \n\t\tMessage:%s\n\t\tFallback value: %s\n", + date.getErrorMessage(), date.getErrorFallbackValue()); + } + + Entry age = record.getSchema().getEntry("age"); + if (age.isValid()) { + System.out.printf("\tAge: %s\n", record.getInt("age")); + } else { + System.out.printf("\tAge is on error: \n\t\tMessage:%s\n\t\tFallback value: %s\n", + age.getErrorMessage(), age.getErrorFallbackValue()); } } - //set support or not. + // set support or not. public static void setSupportError(final boolean supportError) { System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); } diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java new file mode 100644 index 0000000000000..576732e75298f --- /dev/null +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.entrywitherror; + +import java.io.Serializable; +import java.time.Month; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.function.Function; + +import javax.annotation.PostConstruct; + +import org.talend.sdk.component.api.component.Icon; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.Producer; +import org.talend.sdk.component.api.meta.Documentation; +import org.talend.sdk.component.api.record.Record; +import org.talend.sdk.component.api.record.Record.Builder; +import org.talend.sdk.component.api.record.Schema; +import org.talend.sdk.component.api.record.Schema.Entry; +import org.talend.sdk.component.api.service.record.RecordBuilderFactory; +import org.talend.sdk.component.runtime.record.SchemaImpl; + +import lombok.Data; + +@Version +@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") +@Emitter(name = "RecordWithEntriesInErrorEmitter") +@Documentation("Generated record with entries in error.") +public class RecordWithEntriesInErrorEmitter implements Serializable { + + private final RecordBuilderFactory recordBuilderFactory; + + private final Config config; + + private transient Schema recordSchema; + + private transient Function createRecordFunction; + + private transient int index; + + public RecordWithEntriesInErrorEmitter( + final RecordBuilderFactory recordBuilderFactory, + final @Option("configuration") Config config) { + this.recordBuilderFactory = recordBuilderFactory; + this.config = config; + } + + @PostConstruct + public void init() { + recordSchema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("name") + .withNullable(false) + .withErrorCapable(true) + .withType(Schema.Type.STRING) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withErrorCapable(true) + .withType(Schema.Type.DATETIME) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("age") + .withNullable(false) + .withErrorCapable(true) + .withType(Schema.Type.INT) + .build()) + .build(); + + createRecordFunction = i -> { + Builder builder = recordBuilderFactory.newRecordBuilder(recordSchema).withString("name", "name " + i); + + // Generate error only on odd generated records + boolean generateErrors = config.isGenerateErrors() && i % 2 == 0; + + if (generateErrors) { + Entry dateEntry = recordSchema.getEntry("date"); + builder.with(dateEntry, "789-555"); + } else { + ZonedDateTime dateTime = ZonedDateTime.of( + 2025, // Year + Month.APRIL.getValue(), // Month + 1 + i, // Day + 15, // Hours + 30, // Minutes + 0, // seconds + 0, // nanoseconds + ZoneId.of("UTC") // Timezone + ); + builder.withDateTime("date", dateTime); + } + + if (generateErrors) { + Entry ageEntry = recordSchema.getEntry("age"); + builder.with(ageEntry, "-78"); + } else { + builder.withInt("age", 50 + i); + } + + return builder.build(); + }; + } + + @Producer + public Record data() { + index++; + if (index <= config.getNbRecords()) { + return createRecordFunction.apply(index); + } + + return null; + } + + @Data + @GridLayout(value = { + @GridLayout.Row("generateErrors"), + @GridLayout.Row("nbRecords"), + }) + public static class Config implements Serializable { + + @Option + @Documentation("If true, generate some errors.") + private boolean generateErrors = true; + + @Option + @Documentation("Number of generated records.") + private int nbRecords = 5; + + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/package-info.java similarity index 79% rename from sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java rename to sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/package-info.java index ff1441932bd2a..0668f6bcaa05c 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/package-info.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Components(family = "supporterror", categories = "sample") +@Components(family = "sampleRecordWithEntriesInError", categories = "sample") @Icon(value = Icon.IconType.CUSTOM, custom = "icon") -package org.talend.sdk.component.sample.feature.supporterror; +package org.talend.sdk.component.sample.feature.entrywitherror; import org.talend.sdk.component.api.component.Components; -import org.talend.sdk.component.api.component.Icon; \ No newline at end of file +import org.talend.sdk.component.api.component.Icon; diff --git a/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder b/sample-parent/sample-features/entry-with-error/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder rename to sample-parent/sample-features/entry-with-error/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties similarity index 58% rename from sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties rename to sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties index 0e589d31e4429..87398b474113c 100644 --- a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties +++ b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties @@ -13,16 +13,7 @@ # limitations under the License. # Here you can change all your configuration display names to use more explicit labels # You can also translate your configuration by adding one file by local Messages_fr.properties for french for example -supporterror.supportErrorInput._displayName=Support Error Input -supporterror.supportErrorAvroInput._displayName=Support Error Avro Input -supporterror.SupportErrorMapper._displayName=Support Error Mapper -supporterror.datastore.default._displayName=default -supporterror.dataset.default._displayName=dataset -Dataset.datastore._displayName=Datastore -InputConfig.dataset._displayName=Dataset -Datastore.age._displayName=age -Datastore.date._displayName=date -Datastore.name._displayName=name -Datastore.age._placeholder= -Datastore.date._placeholder= -Datastore.name._placeholder= +sampleRecordWithEntriesInError.RecordWithEntriesInErrorEmitter._displayName = Record with entries in error emitter +Config.generateErrors._displayName = Generate some errors +Config.nbRecords._displayName = Nb generated records +Config.nbRecords._placeholder = diff --git a/sample-parent/sample-features/pom.xml b/sample-parent/sample-features/pom.xml index 1b1046b48b1b5..4b0d77c93bc41 100644 --- a/sample-parent/sample-features/pom.xml +++ b/sample-parent/sample-features/pom.xml @@ -32,7 +32,7 @@ aftergroup-lastgroup conditional-outputs checkpoint-runner - supporterror + entry-with-error diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java deleted file mode 100644 index c9c3d61577ff2..0000000000000 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.talend.sdk.component.sample.feature.supporterror; - -import java.io.Serializable; -import java.util.Date; - -import javax.annotation.PostConstruct; - -import org.talend.sdk.component.api.component.Icon; -import org.talend.sdk.component.api.component.Version; -import org.talend.sdk.component.api.configuration.Option; -import org.talend.sdk.component.api.configuration.type.DataSet; -import org.talend.sdk.component.api.configuration.type.DataStore; -import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; -import org.talend.sdk.component.api.input.Emitter; -import org.talend.sdk.component.api.input.Producer; -import org.talend.sdk.component.api.meta.Documentation; -import org.talend.sdk.component.api.record.Record; -import org.talend.sdk.component.api.record.Schema; -import org.talend.sdk.component.runtime.record.RecordImpl; -import org.talend.sdk.component.runtime.record.SchemaImpl; - -import lombok.Data; - -@Version -@Icon(value = Icon.IconType.CUSTOM, custom = "icon") -@Documentation("Support Error Input sample processor connector.") -@Emitter(family = "supporterror", name = "supportErrorInput") -public class SupportErrorInput implements Serializable { - - private transient Schema recordSchema; - - private transient Schema.Entry nameEntry; - - private transient Schema.Entry dateEntry; - - private transient Schema.Entry ageEntry; - - private final transient InputConfig configuration; - - public SupportErrorInput(@Option("configuration") final InputConfig config) { - this.configuration = config; - } - - @PostConstruct - public void init() { - nameEntry = new SchemaImpl.EntryImpl.BuilderImpl() - .withName("name") - .withNullable(false) - .withType(Schema.Type.STRING) - .build(); - dateEntry = new SchemaImpl.EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Schema.Type.DATETIME) - .build(); - ageEntry = new SchemaImpl.EntryImpl.BuilderImpl() - .withName("age") - .withNullable(false) - .withType(Schema.Type.INT) - .build(); - recordSchema = new SchemaImpl.BuilderImpl() - .withType(Schema.Type.RECORD) - .withEntry(nameEntry) - .withEntry(dateEntry) - .withEntry(ageEntry) - .build(); - } - - @Producer - public Record data() { - final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - return builder.with(nameEntry, "example connector") - .with(dateEntry, "not a date value") - .with(ageEntry, "wrong int value") - .build(); - } - - @Data - @GridLayout(value = { - @GridLayout.Row("dataset"), - }) - @Version - public static class InputConfig { - - @Option - @Documentation("Dataset.") - private Dataset dataset = new Dataset(); - } - - @DataSet - @Data - @GridLayout(value = {@GridLayout.Row("datastore")}) - public static class Dataset implements Serializable { - - @Option - @Documentation("Datastore.") - private Datastore datastore = new Datastore(); - - } - - @DataStore - @Data - @GridLayout(value = {@GridLayout.Row("name"), @GridLayout.Row("age"), - @GridLayout.Row("date")}) - public static class Datastore implements Serializable { - - @Option - @Documentation("Name prop.") - private String name = "test"; - - @Option - @Documentation("Age prop.") - private int age = 0; - - @Option - @Documentation("Date prop.") - private Date date; - } - -} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java deleted file mode 100644 index db3d0506c9c98..0000000000000 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.talend.sdk.component.sample.feature.supporterror; - -import java.io.Serializable; -import java.util.Collections; -import java.util.List; - -import org.talend.sdk.component.api.component.Icon; -import org.talend.sdk.component.api.component.Version; -import org.talend.sdk.component.api.configuration.Option; -import org.talend.sdk.component.api.input.Assessor; -import org.talend.sdk.component.api.input.Emitter; -import org.talend.sdk.component.api.input.PartitionMapper; -import org.talend.sdk.component.api.input.PartitionSize; -import org.talend.sdk.component.api.input.Split; -import org.talend.sdk.component.api.meta.Documentation; - -@Version(1) -@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") -@PartitionMapper(name = "SupportErrorMapper", infinite = false) -@Documentation("Doc: default SupportErrorMapper documentation without Internationalization.") -public class SupportErrorMapper implements Serializable { - - private transient SupportErrorInput.InputConfig config; - - public SupportErrorMapper(final @Option("configin") SupportErrorInput.InputConfig config) { - this.config = config; - } - - @Assessor - public long estimateSize() { - return 1500L; - } - - @Split - public List split(@PartitionSize final int desiredNbSplits) { - return Collections.singletonList(this); - } - - @Emitter - public SupportErrorInput createSource() { - return new SupportErrorInput(this.config); - } - -} From 6e3ad2b226c8dfe94016266c1f5e510e7c5b54d8 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Tue, 13 May 2025 15:19:23 +0200 Subject: [PATCH 23/62] feat(QTDI-1305): set RecordImpl#BuilderImpl#entriesInError final. --- .../org/talend/sdk/component/runtime/record/RecordImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index baa0ba2571401..1bc5d45484878 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -130,7 +130,7 @@ public static class BuilderImpl implements Builder { private OrderState orderState; - private Map entriesInError = new HashMap<>(); + private final Map entriesInError = new HashMap<>(); public BuilderImpl() { this(null); From 9df828e6f6a6adbf1da20247006b1a83aab9ff92 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Tue, 13 May 2025 18:05:05 +0200 Subject: [PATCH 24/62] feat(QTDI-1305): Fix entry with error in AvroRecord + unit test. --- .../org/talend/sdk/component/api/record/SchemaProperty.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java index 95bf16be2d181..25954bba2adb9 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java @@ -43,8 +43,6 @@ public interface SchemaProperty { String ENTRY_ERROR_FALLBACK_VALUE = "entry.error.fallback.value"; - String ERROR_EXCEPTION = "entry.error.exception"; - enum LogicalType { DATE("date"), From fc71e1ab9774b03ebfc39b525269eb49cb7b69cb Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 14 May 2025 16:08:15 +0800 Subject: [PATCH 25/62] add transfer "errorcapable" between Avro.Field and Schema.Entry. --- .../sdk/component/runtime/beam/spi/record/AvroSchema.java | 1 + .../runtime/beam/spi/record/AvroSchemaBuilder.java | 4 +++- .../runtime/beam/spi/record/KeysForAvroProperty.java | 2 ++ .../runtime/beam/spi/record/AvroRecordBuilderTest.java | 6 ++++-- .../beam/spi/record/JsonSchemaSerializationTest.java | 4 ++-- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java index 0a976b980ab81..e6f08b6b55eb2 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java @@ -222,6 +222,7 @@ private static Entry buildFromAvro(final Field field, final Type type, .withRawName(field.getProp(KeysForAvroProperty.LABEL)) // .withType(type) // .withNullable(field.schema().getType() == UNION) // + .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.ERROR_CAPABLE))) .withMetadata(AvroSchema.isMetadata(field)) // .withDefaultValue(field.defaultVal()) // .withElementSchema(elementSchema) // diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java index 948501a242d33..00a72e93686d3 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java @@ -463,7 +463,9 @@ public static Field toField(final org.apache.avro.Schema schema, final Schema.En for (Map.Entry e : entry.getProps().entrySet()) { field.addProp(e.getKey(), e.getValue()); } - + if (entry.isErrorCapable()) { + field.addProp(KeysForAvroProperty.ERROR_CAPABLE, String.valueOf(entry.isErrorCapable())); + } return field; } } diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java index a5d5b541cbaae..69860b99b9406 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java @@ -21,4 +21,6 @@ public interface KeysForAvroProperty { // alias that indicate field is metadata if present. String METADATA_ALIAS_NAME = "talend.field.__METADATA__"; + + String ERROR_CAPABLE = "entry.errorCapable"; } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 4a7078b7b0d91..6cb91a48e12f2 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -213,13 +213,13 @@ void mixedRecordTest() { void testWithError() { final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD); Schema.Entry nameEntry = factory .newEntryBuilder() .withName("name") .withNullable(false) + .withErrorCapable(true) .withType(Schema.Type.STRING) .build(); Schema.Entry nmEntry = factory @@ -232,13 +232,15 @@ void testWithError() { .newEntryBuilder() .withName("age") .withNullable(false) + .withErrorCapable(true) .withType(Schema.Type.INT) .build(); Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); // record 1 Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record1 = recordBuilder.with(nameEntry, null) + Record record1 = recordBuilder .with(nmEntry, "normal") + .with(nameEntry, null) .with(ageEntry, "is not an int") .build(); assertFalse(record1.isValid()); diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java index 1993f10373c98..607083f593ef2 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java @@ -49,7 +49,7 @@ void toJson() throws Exception { .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { final String json = jsonb.toJson(schema); assertEquals( - "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", json); } } @@ -71,7 +71,7 @@ void toJsonWithMeta() throws Exception { try (final Jsonb jsonb = JsonbBuilder .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { assertEquals( - "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", jsonb.toJson(schema)); } From b4e2ad3380e1b47a7e445c8ad01bade8973b1fe3 Mon Sep 17 00:00:00 2001 From: ypiel Date: Thu, 15 May 2025 10:09:08 +0200 Subject: [PATCH 26/62] feat(QTDI-1305): Fix some unit test + improve sample CLI * feat(QTDI-1305): Complete unit test for entry with error in beam impl.. * feat(QTDI-1305): Fix beam impl unit test. * feat(QTDI-1305): Fix beam impl unit test bis. * feat(QTDI-1305): Fix unit test after adding Entry#isErrorCapable attribute. * feat(QTDI-1305): Imprive sample feature CLI to let user define the number of generated errors.. * feat(QTDI-1305): the CLI support --use-avro-impl option. * feat(QTDI-1305): the CLI support --fields-in-error to be able to decide which fields are in error. --- .../runtime/beam/spi/record/AvroSchema.java | 7 +- .../beam/spi/record/AvroSchemaBuilder.java | 2 +- .../beam/spi/record/KeysForAvroProperty.java | 2 +- .../spi/record/AvroRecordBuilderTest.java | 82 +++++++++++----- .../record/JsonSchemaSerializationTest.java | 4 +- .../server/front/ActionResourceImplTest.java | 5 +- .../beam/BeamActionSerializationTest.java | 5 +- .../sample-features/entry-with-error/pom.xml | 42 ++++----- .../sample/feature/entrywitherror/Cli.java | 93 ++++++++++++------- .../RecordWithEntriesInErrorEmitter.java | 39 +++++--- .../entrywitherror/Messages.properties | 6 +- 11 files changed, 182 insertions(+), 105 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java index e6f08b6b55eb2..a6dc888ba0128 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java @@ -216,13 +216,14 @@ private Entry fromAvro(final Field field) { } private static Entry buildFromAvro(final Field field, final Type type, - final Optional logicalType, final AvroSchema elementSchema) { + final Optional logicalType, + final AvroSchema elementSchema) { Entry.Builder builder = new EntryImpl.BuilderImpl() // .withName(field.name()) // .withRawName(field.getProp(KeysForAvroProperty.LABEL)) // .withType(type) // .withNullable(field.schema().getType() == UNION) // - .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.ERROR_CAPABLE))) + .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.IS_ERROR_CAPABLE))) .withMetadata(AvroSchema.isMetadata(field)) // .withDefaultValue(field.defaultVal()) // .withElementSchema(elementSchema) // @@ -230,6 +231,8 @@ private static Entry buildFromAvro(final Field field, final Type type, .withProps(field.getObjectProps() .entrySet() .stream() + // KeysForAvroProperty.IS_ERROR_CAPABLE is already managed above + .filter(p -> !p.getKey().equals(KeysForAvroProperty.IS_ERROR_CAPABLE)) .collect(toMap(Map.Entry::getKey, e -> String.valueOf(e.getValue())))); logicalType.ifPresent(builder::withLogicalType); diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java index 00a72e93686d3..e234a3a031ba7 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java @@ -464,7 +464,7 @@ public static Field toField(final org.apache.avro.Schema schema, final Schema.En field.addProp(e.getKey(), e.getValue()); } if (entry.isErrorCapable()) { - field.addProp(KeysForAvroProperty.ERROR_CAPABLE, String.valueOf(entry.isErrorCapable())); + field.addProp(KeysForAvroProperty.IS_ERROR_CAPABLE, String.valueOf(entry.isErrorCapable())); } return field; } diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java index 69860b99b9406..937fbeeea3734 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java @@ -22,5 +22,5 @@ public interface KeysForAvroProperty { // alias that indicate field is metadata if present. String METADATA_ALIAS_NAME = "talend.field.__METADATA__"; - String ERROR_CAPABLE = "entry.errorCapable"; + String IS_ERROR_CAPABLE = "entry.errorCapable"; } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 6cb91a48e12f2..265c713ba2d8e 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -41,6 +41,7 @@ import javax.json.JsonArray; import javax.json.JsonObject; +import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.IndexedRecord; import org.apache.avro.io.EncoderFactory; @@ -51,6 +52,7 @@ import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.EntriesOrder; +import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; @@ -209,53 +211,91 @@ void mixedRecordTest() { Assertions.assertNotNull(arrayType); } + @Test + void testWithoutErrorSupport() { + Assertions.assertThrows(IllegalArgumentException.class, () -> testWithError("false")); + } + @Test void testWithError() { - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + Record record = testWithError("true"); + assertFalse(record.isValid()); + + final Schema.Entry entry = record.getSchema().getEntry("name"); + assertNotNull(entry); + Assertions.assertFalse(entry.isValid()); + Assertions.assertNull(record.getString("name")); + + final Schema.Entry entry2 = record.getSchema().getEntry("age"); + assertNotNull(entry2); + Assertions.assertFalse(entry2.isValid()); + Assertions.assertNull(record.get(Integer.class, "age")); + + IndexedRecord unwrap = ((AvroRecord) record).unwrap(IndexedRecord.class); + + Field nameField = unwrap.getSchema().getFields().get(0); + Assertions.assertEquals("true", + nameField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + Assertions.assertEquals("Entry 'name' is not nullable", + nameField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); + Assertions.assertEquals("null", + nameField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); + + Field ageField = unwrap.getSchema().getFields().get(2); + Assertions.assertEquals("true", + ageField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + Assertions.assertEquals("Entry 'age' of type INT is not compatible with given value of type " + + "'java.lang.String': 'is not an int'.", + ageField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); + Assertions.assertEquals("is not an int", + ageField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); + + Field noErrorField = unwrap.getSchema().getFields().get(1); + Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); + Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); + + } + + private Record testWithError(final String supported) { + final String errorSupportBackup = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, supported); org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD); Schema.Entry nameEntry = factory .newEntryBuilder() .withName("name") - .withNullable(false) .withErrorCapable(true) + .withNullable(false) .withType(Schema.Type.STRING) .build(); - Schema.Entry nmEntry = factory + Schema.Entry noErrorEntry = factory .newEntryBuilder() .withName("normal") + .withErrorCapable(true) .withNullable(true) .withType(Schema.Type.STRING) .build(); Schema.Entry ageEntry = factory .newEntryBuilder() .withName("age") - .withNullable(false) .withErrorCapable(true) + .withNullable(false) .withType(Schema.Type.INT) .build(); - Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); - // record 1 + Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(noErrorEntry).withEntry(ageEntry).build(); + + Entry age = customerSchema.getEntry("age"); + Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record1 = recordBuilder - .with(nmEntry, "normal") - .with(nameEntry, null) + Record record = recordBuilder.with(nameEntry, null) + .with(noErrorEntry, "normal") .with(ageEntry, "is not an int") .build(); - assertFalse(record1.isValid()); - - final Schema.Entry entry = - record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); - assertNotNull(entry); - Assertions.assertFalse(entry.isValid()); - final Schema.Entry entry2 = - record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); - assertNotNull(entry2); - Assertions.assertFalse(entry2.isValid()); + System.setProperty(Record.RECORD_ERROR_SUPPORT, errorSupportBackup == null ? "false" : errorSupportBackup); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); + return record; } @Test diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java index 607083f593ef2..791d653b0eb1c 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java @@ -49,7 +49,7 @@ void toJson() throws Exception { .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { final String json = jsonb.toJson(schema); assertEquals( - "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"errorCapable\":false,\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", json); } } @@ -71,7 +71,7 @@ void toJsonWithMeta() throws Exception { try (final Jsonb jsonb = JsonbBuilder .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { assertEquals( - "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"errorCapable\":false,\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", jsonb.toJson(schema)); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java index 85bb91a1e62c7..9233c5bdc82d4 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java @@ -198,7 +198,8 @@ void checkSchemaSerialization() { "{\n \"entries\":[\n {\n \"elementSchema\":{\n \"entries\":[\n ],\n" + " \"metadata\":[\n ],\n \"props\":{\n\n },\n \"type\":\"STRING\"\n" + - " },\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + + " },\n \"errorCapable\":false," + + "\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + " \"props\":{\n\n },\n \"type\":\"ARRAY\",\n" + " \"valid\":true\n }\n ],\n \"metadata\":[\n" + " ],\n \"props\":{\n \"talend.fields.order\":\"array\"\n },\n \"type\":\"RECORD\"\n}"; @@ -239,7 +240,7 @@ void checkDiscoverProcessorSchema() { }, APPLICATION_JSON_TYPE), JsonObject.class); assertNotNull(guessed); final String expected = - "{\"entries\":[{\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"comment\":\"field2 comment\",\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\",\"valid\":true},{\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; + "{\"entries\":[{\"errorCapable\":false,\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"comment\":\"field2 comment\",\"errorCapable\":false,\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\",\"valid\":true},{\"errorCapable\":false,\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"errorCapable\":false,\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; assertEquals(expected, guessed.toString()); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java index e30e2286e2d3c..d4111f8379d3d 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java @@ -45,8 +45,9 @@ void checkSchemaSerialization() { final String attended = "{\n" + " \"entries\":[\n" + " {\n" + " \"elementSchema\":{\n" + " \"entries\":[\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"STRING\"\n" + " },\n" - + " \"metadata\":false,\n" + " \"name\":\"array\",\n" + " \"nullable\":false,\n" - + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\",\n" + + + " \"errorCapable\":false,\n \"metadata\":false,\n" + + " \"name\":\"array\",\n" + " \"nullable\":false,\n" + + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\",\n" + " \"valid\":true\n }\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + " \"talend.fields.order\":\"array\"\n" + " },\n" + " \"type\":\"RECORD\"\n" + "}"; diff --git a/sample-parent/sample-features/entry-with-error/pom.xml b/sample-parent/sample-features/entry-with-error/pom.xml index b136e3b977d73..dcccec81bae86 100644 --- a/sample-parent/sample-features/entry-with-error/pom.xml +++ b/sample-parent/sample-features/entry-with-error/pom.xml @@ -23,7 +23,7 @@ org.talend.sdk.component.sample.feature - supporterror + entrywitherror jar Component Runtime :: Sample Feature :: Entry with error support @@ -39,6 +39,15 @@ component-runtime-manager ${project.version} + + org.talend.sdk.component + component-runtime-beam + ${project.version} + + + org.apache.avro + avro + org.tomitribe tomitribe-crest @@ -97,18 +106,6 @@ - - org.apache.maven.plugins - maven-jar-plugin - - - - true - org.talend.sdk.component.sample.feature.entrywitherror.Cli - - - - maven-shade-plugin 2.1 @@ -124,19 +121,20 @@ org.talend.sdk.component.sample.feature.entrywitherror.Cli + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + - - - org.codehaus.mojo - exec-maven-plugin - 3.0.0 - - org.talend.sdk.component.sample.feature.entrywitherror.Cli - - diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 9570ec019fc37..f9183843a1925 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -24,12 +24,15 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.avro.generic.IndexedRecord; import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; +import org.talend.sdk.component.runtime.beam.spi.record.AvroRecord; import org.talend.sdk.component.runtime.input.InputImpl; import org.talend.sdk.component.runtime.input.Mapper; import org.talend.sdk.component.runtime.manager.ComponentManager; @@ -46,32 +49,43 @@ @NoArgsConstructor(access = PRIVATE) public final class Cli { - static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + static final String GAV = "org.talend.sdk.component.sample.feature:entrywitherror:jar:" + Versions.KIT_VERSION; - @Command("supporterror") + @Command("entry-with-error") public static void runInput( @Option("gav") @Default(GAV) final String gav, - @Option("support") @Default("false") final boolean support, - @Option("gen-some-errors") @Default("true") final boolean genErrors, - @Option("gen-nbrecords") @Default("10") final int nbRecords, + @Option("support-entry-with-error") @Default("false") final boolean supportEntryWithError, + @Option("use-avro-impl") @Default("false") final boolean useAvroImpl, + @Option("how-many-errors") @Default("0") final int howManyErrors, + @Option("gen-nb-records") @Default("10") final int nbRecords, + @Option("fields-in-error") @Default("age,date") final String fieldsInError, @Option("jar") final File jar, - @Option("family") @Default("supporterror") final String family, - @Option("mapper") @Default("SupportErrorMapper") final String mapper) { + @Option("family") @Default("sampleRecordWithEntriesInError") final String family, + @Option("mapper") @Default("RecordWithEntriesInErrorEmitter") final String mapper) { - info("support " + support); - if (support) { - setSupportError(support); - } + System.out.printf( + "Parameters:%n\tgav: %s%n\tsupport-entry-with-error: %s%n\tuse-avro-impl: %s%n\thow-many-errors: %d%n" + + "\tgen-nb-records: %d%n\tgfields-in-error: %s%n\tjar: %s%n\tfamily: %s%n\tmapper: %s%n", + gav, supportEntryWithError, useAvroImpl, howManyErrors, nbRecords, fieldsInError, jar, family, mapper); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportEntryWithError)); + System.setProperty("talend.component.beam.record.factory.impl", useAvroImpl ? "avro" : "default"); Map config = new HashMap<>(); - config.put("configuration.generateErrors", String.valueOf(genErrors)); + config.put("configuration.howManyErrors", String.valueOf(howManyErrors)); config.put("configuration.nbRecords", String.valueOf(nbRecords)); - run(jar, gav, config, "sampleRecordWithEntriesInError", "RecordWithEntriesInErrorEmitter"); + + String[] fields = fieldsInError.split(","); + for (int i = 0; i < fields.length; i++) { + config.put("configuration.fieldsInError[" + i + "]", fields[i]); + } + + run(jar, gav, config, family, mapper, useAvroImpl); } private static void run(final File jar, final String gav, final Map configuration, - final String family, final String mapper) { + final String family, final String mapper, final boolean avro) { try (final ComponentManager manager = manager(jar, gav)) { info("configuration: " + configuration); @@ -89,39 +103,52 @@ private static void run(final File jar, final String gav, final Map { + String props = f.getObjectProps() + .entrySet() + .stream() + .map(es -> "\t\t\t" + es.getKey() + " = " + es.getValue()) + .collect(Collectors.joining("\n")); + System.out.printf("\t\tField '%s', properties: %n%s%n", f.name(), props); + }); + } + } public static void main(final String[] args) throws Exception { @@ -132,9 +159,7 @@ public static Object run(final String[] args) throws Exception { return new Main(Cli.class).exec(args); } - static final String ERROR = "[ERROR] "; - - static final String WARN = "[WARN] "; + static final String EXCEPTION = "[EXCEPTION] "; static final String INFO = "[INFO] "; @@ -202,12 +227,8 @@ public static void info(final String message) { System.out.println(INFO + message); } - public static void warn(final String message) { - System.err.println(WARN + message); - } - - public static void error(final Throwable e) { - System.err.println(ERROR + e.getMessage()); + public static void exception(final Throwable e) { + System.err.println(EXCEPTION + e.getMessage()); System.exit(501); } diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java index 576732e75298f..8a8558dc4625a 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java @@ -19,6 +19,8 @@ import java.time.Month; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; import java.util.function.Function; import javax.annotation.PostConstruct; @@ -35,7 +37,6 @@ import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; -import org.talend.sdk.component.runtime.record.SchemaImpl; import lombok.Data; @@ -64,21 +65,20 @@ public RecordWithEntriesInErrorEmitter( @PostConstruct public void init() { - recordSchema = new SchemaImpl.BuilderImpl() - .withType(Schema.Type.RECORD) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + recordSchema = recordBuilderFactory.newSchemaBuilder(Schema.Type.RECORD) + .withEntry(recordBuilderFactory.newEntryBuilder() .withName("name") .withNullable(false) .withErrorCapable(true) .withType(Schema.Type.STRING) .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withEntry(recordBuilderFactory.newEntryBuilder() .withName("date") .withNullable(false) .withErrorCapable(true) .withType(Schema.Type.DATETIME) .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withEntry(recordBuilderFactory.newEntryBuilder() .withName("age") .withNullable(false) .withErrorCapable(true) @@ -87,12 +87,18 @@ public void init() { .build(); createRecordFunction = i -> { - Builder builder = recordBuilderFactory.newRecordBuilder(recordSchema).withString("name", "name " + i); + Builder builder = recordBuilderFactory.newRecordBuilder(recordSchema); - // Generate error only on odd generated records - boolean generateErrors = config.isGenerateErrors() && i % 2 == 0; + boolean generateErrors = config.getHowManyErrors() >= i; - if (generateErrors) { + if (generateErrors && config.getFieldsInError().contains("name")) { + Entry nameEntry = recordSchema.getEntry("name"); + builder.with(nameEntry, null); + } else { + builder.withString("name", "name " + i); + } + + if (generateErrors && config.getFieldsInError().contains("date")) { Entry dateEntry = recordSchema.getEntry("date"); builder.with(dateEntry, "789-555"); } else { @@ -109,7 +115,7 @@ public void init() { builder.withDateTime("date", dateTime); } - if (generateErrors) { + if (generateErrors && config.getFieldsInError().contains("age")) { Entry ageEntry = recordSchema.getEntry("age"); builder.with(ageEntry, "-78"); } else { @@ -132,19 +138,24 @@ public Record data() { @Data @GridLayout(value = { - @GridLayout.Row("generateErrors"), + @GridLayout.Row("howManyErrors"), @GridLayout.Row("nbRecords"), + @GridLayout.Row("fieldsInError"), }) public static class Config implements Serializable { @Option - @Documentation("If true, generate some errors.") - private boolean generateErrors = true; + @Documentation("The number of errors to generate..") + private int howManyErrors; @Option @Documentation("Number of generated records.") private int nbRecords = 5; + @Option + @Documentation("Fields in error.") + private List fieldsInError = new ArrayList<>(); + } } diff --git a/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties index 87398b474113c..37eaf22a94654 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties +++ b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties @@ -14,6 +14,8 @@ # Here you can change all your configuration display names to use more explicit labels # You can also translate your configuration by adding one file by local Messages_fr.properties for french for example sampleRecordWithEntriesInError.RecordWithEntriesInErrorEmitter._displayName = Record with entries in error emitter -Config.generateErrors._displayName = Generate some errors +Config.howManyErrors._displayName = Number of errors to generate +Config.howManyErrors._placeholder = Config.nbRecords._displayName = Nb generated records -Config.nbRecords._placeholder = +Config.nbRecords._placeholder = +Config.fieldsInError._displayName = Fields in error \ No newline at end of file From c5e38bb26643179e5830c839ed41668f284aeb53 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 21 Apr 2025 10:23:07 +0800 Subject: [PATCH 27/62] Add withError in Record.Builder --- .../sdk/component/api/record/Record.java | 8 ++++ .../sdk/component/api/record/Schema.java | 5 +++ .../component/api/record/SchemaProperty.java | 8 ++++ .../component/api/service/schema/Schema.java | 4 ++ .../sdk/component/api/record/SchemaTest.java | 3 ++ .../component/runtime/record/RecordImpl.java | 45 +++++++++++++++++++ .../component/runtime/record/SchemaImpl.java | 9 ++++ .../runtime/record/RecordBuilderImplTest.java | 35 +++++++++++++++ 8 files changed, 117 insertions(+) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index b9258587bdbee..d34a27e6ed7fb 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -34,6 +34,8 @@ public interface Record { + String RECORD_ERROR_SUPPORT = "talend.sdk.runtime.record.error.support"; + /** * @return the schema of this record. */ @@ -311,6 +313,10 @@ default Optional getOptionalRecord(final String name) { return ofNullable(get(Record.class, name)); } + default boolean isValid() { + return getSchema().getAllEntries().filter(entry -> !entry.isValid()).findAny().isPresent(); + } + /** * Allows to create a record with a fluent API. This is the unique recommended way to create a record. */ @@ -447,5 +453,7 @@ default Builder withInstant(Schema.Entry entry, Instant value) { Builder withRecord(String name, Record value); Builder withArray(Schema.Entry entry, Collection values); + + Builder withError(String columnName, Object value, String errorMessage, Exception exception); } } diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java index 5050102b4879e..da4b0806638be 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java @@ -232,6 +232,11 @@ interface Entry { */ boolean isMetadata(); + /** + * @return true if the value of this entry is valid; false for invalid value. + */ + boolean isValid(); + /** * @param the default value type. * diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java index 076467334ce3c..95bf16be2d181 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java @@ -37,6 +37,14 @@ public interface SchemaProperty { String ALLOW_SPECIAL_NAME = "field.special.name"; + String ENTRY_IS_ON_ERROR = "entry.on.error"; + + String ENTRY_ERROR_MESSAGE = "entry.error.message"; + + String ENTRY_ERROR_FALLBACK_VALUE = "entry.error.fallback.value"; + + String ERROR_EXCEPTION = "entry.error.exception"; + enum LogicalType { DATE("date"), diff --git a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java index 355e321ebedce..197ea0299d8f7 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java @@ -30,6 +30,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.talend.sdk.component.api.record.SchemaProperty; @Partial("This API should support nested schema but the Studio is not yet ready.\n\n" + "The cloud platform also doesn't use it yet.\n\nAlso prefer to use " @@ -163,6 +164,9 @@ public String getProp(final String property) { public Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder()"); } + + @Override + public boolean isValid() { return true; } } } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java index 0157dabdff610..11c5e325f3c9d 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java @@ -326,6 +326,9 @@ public JsonValue getJsonProp(final String name) { return Entry.super.getJsonProp(name); } + @Override + public boolean isValid() { return true; } + } class SchemaBuilder implements Schema.Builder { diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 47c77ce50c529..9c3f6f06ad3cc 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -64,6 +64,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import org.talend.sdk.component.api.record.SchemaProperty; @EqualsAndHashCode public final class RecordImpl implements Record { @@ -92,6 +93,11 @@ public T get(final Class expectedType, final String name) { return RECORD_CONVERTERS.coerce(expectedType, value, name); } + @Override + public boolean isValid() { + return false; + } + @Override // for debug purposes, don't use it for anything else public String toString() { try (final Jsonb jsonb = JsonbBuilder @@ -513,6 +519,45 @@ public Builder withArray(final Schema.Entry entry, final Collection value return append(entry, values); } + @Override + public Builder withError(String columnName, Object value, String errorMessage, Exception exception) { + final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); + if (!supportError) { + throw new IllegalArgumentException(errorMessage); + } else { + // duplicate the schema instance with a modified Entry + final Entry oldEntry = this.findExistingEntry(columnName); + final Entry updateEntry = oldEntry.toBuilder() + .withName(columnName) + .withNullable(true) + .withType(oldEntry.getType()) + .withProp(SchemaProperty.ENTRY_IS_ON_ERROR, "true") + .withProp(SchemaProperty.ENTRY_ERROR_MESSAGE, errorMessage) + .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) + .withProp(SchemaProperty.ERROR_EXCEPTION, exception == null ? "" : exception.toString()) + .build(); + final Schema.Builder builder = getErrorBuilder(providedSchema.getType()); + providedSchema.getAllEntries() +// .filter(e -> Objects.equals(providedSchema.getEntry(e.getName()), e)) + .forEach(e -> { + if (columnName.equals(e.getName())) { + builder.withEntry(updateEntry); + } else { + builder.withEntry(e); + } + }); + return getErrorBuilder(builder); + } + } + + protected Builder getErrorBuilder(Schema.Builder builder) { + return new BuilderImpl(builder.build()); + } + + protected Schema.Builder getErrorBuilder(Schema.Type type) { + return new SchemaImpl.BuilderImpl().withType(type); + } + private void assertType(final Schema.Type actual, final Schema.Type expected) { if (actual != expected) { throw new IllegalArgumentException("Expected entry type: " + expected + ", got: " + actual); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java index 035aec97f83aa..6348c4d0f69f0 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java @@ -465,6 +465,15 @@ public Map getProps() { return this.props; } + @Override + public boolean isValid() { + String property = this.getProp(SchemaProperty.ENTRY_IS_ON_ERROR); + if (property == null) { + return true; + } + return !Boolean.parseBoolean(property); + } + /** * Plain builder matching {@link Entry} structure. */ diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index dec6e7ceb3260..9cddee216ca8b 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -17,6 +17,7 @@ import static java.util.stream.Collectors.joining; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -248,6 +249,40 @@ void dateTime() { assertThrows(IllegalArgumentException.class, () -> builder2.withDateTime("date", (ZonedDateTime) null)); } + @Test + void withError() { + final Schema schema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("intValue") + .withNullable(false) + .withType(Type.INT) + .build()) + .build(); + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); + final Record record3 = builder3.withError("date", null, "date is null", null) + .withError("intValue", "string", "wrong int value", null) + .build(); + assertFalse(record3.isValid()); + final Entry entry = record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); + assertNotNull(entry); + Assertions.assertFalse(entry.isValid()); + + final Entry entry2 = record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); + assertNotNull(entry2); + Assertions.assertFalse(entry2.isValid()); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); + } + @Test void zonedDateTimeVSInstant() { final Schema schema = new SchemaImpl.BuilderImpl() From 5ba8a4bf4aa10ab023a4aa0933d8352fb4523132 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 21 Apr 2025 18:17:30 +0800 Subject: [PATCH 28/62] Test for AvroRecord --- .../sdk/component/api/record/RecordTest.java | 3 + .../record/RecordBuilderFactoryTest.java | 3 + .../beam/spi/record/AvroRecordBuilder.java | 9 +++ .../spi/record/AvroRecordBuilderTest.java | 62 +++++++++++++++---- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java index c734c2bc50a55..c07eda9008351 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java @@ -135,6 +135,9 @@ public Map getProps() { public String getProp(String property) { return null; } + + @Override + public boolean isValid() { return true; } }; Assertions.assertEquals("value", record.get(String.class, e1)); } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java index 006ffddb6d3d2..a34a6cc58d82f 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java @@ -102,6 +102,9 @@ public String getProp(final String property) { public Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder()"); } + + @Override + public boolean isValid() { return true; } } @RequiredArgsConstructor diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java index 0ce9e2c4c21b6..92232af527616 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java @@ -35,4 +35,13 @@ public AvroRecordBuilder(final Schema providedSchema) { public Record build() { return new AvroRecord(super.build()); } + + protected Record.Builder getErrorBuilder(Schema.Builder builder) { + + return new AvroRecordBuilder(builder.build()); + } + + protected Schema.Builder getErrorBuilder(Schema.Type type) { + return new AvroSchemaBuilder().withType(type); + } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 07fb3da564bf3..7777ef456b03a 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -192,21 +192,59 @@ void avroTest() { @Test void mixedRecordTest() { - final AvroRecordBuilderFactoryProvider recordBuilderFactoryProvider = new AvroRecordBuilderFactoryProvider(); - System.setProperty("talend.component.beam.record.factory.impl", "avro"); - final RecordBuilderFactory recordBuilderFactory = recordBuilderFactoryProvider.apply("test"); - - final RecordBuilderFactory otherFactory = new RecordBuilderFactoryImpl("test"); - final Schema schema = otherFactory - .newSchemaBuilder(RECORD) - .withEntry(otherFactory.newEntryBuilder().withName("e1").withType(INT).build()) + final Schema schema0 = new AvroSchemaBuilder()// + .withType(RECORD) // + .withEntry(dataEntry1) // + .withEntryBefore("data1", meta1) // + .withEntry(dataEntry2) // + .withEntryAfter("meta1", meta2) // .build(); - final Schema arrayType = recordBuilderFactory // - .newSchemaBuilder(Schema.Type.ARRAY) // - .withElementSchema(schema) + final Record.Builder builder0 = factory.newRecordBuilder(schema0); + builder0.withInt("data1", 101) + .withString("data2", "102") + .withInt("meta1", 103) + .withString("meta2", "104"); + final Record record0 = builder0.build(); + assertEquals(101, record0.getInt("data1")); + } + + @Test + void testWithError() { + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + + org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD); + Schema.Entry nameEntry = factory + .newEntryBuilder() + .withName("name") + .withNullable(false) + .withType(Schema.Type.STRING) .build(); - Assertions.assertNotNull(arrayType); + Schema.Entry ageEntry = factory + .newEntryBuilder() + .withName("age") + .withNullable(false) + .withType(Schema.Type.INT) + .build(); + Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(ageEntry).build(); + // record 1 + Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); + Record record1 = recordBuilder.withError("name", null, "Stirng is null", null) + .withError("age", "string", "is not an int", null) + .build(); + assertFalse(record1.isValid()); + + final Schema.Entry entry = record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); + assertNotNull(entry); + Assertions.assertFalse(entry.isValid()); + + final Schema.Entry entry2 = record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); + assertNotNull(entry2); + Assertions.assertFalse(entry2.isValid()); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); } @Test From 2ce8bc81ad9def6ef9b2abe9048407d4bd538e41 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 22 Apr 2025 15:06:11 +0800 Subject: [PATCH 29/62] fix isValid --- .../java/org/talend/sdk/component/api/record/Record.java | 4 +++- .../talend/sdk/component/api/service/schema/Schema.java | 2 +- .../talend/sdk/component/runtime/record/RecordImpl.java | 7 +------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index d34a27e6ed7fb..28a9a3dd6bb6f 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -314,7 +314,9 @@ default Optional getOptionalRecord(final String name) { } default boolean isValid() { - return getSchema().getAllEntries().filter(entry -> !entry.isValid()).findAny().isPresent(); + return !getSchema().getAllEntries() + .filter(entry -> !entry.isValid()).findAny() + .isPresent(); } /** diff --git a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java index 197ea0299d8f7..433673db4b170 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java @@ -30,7 +30,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.talend.sdk.component.api.record.SchemaProperty; @Partial("This API should support nested schema but the Studio is not yet ready.\n\n" + "The cloud platform also doesn't use it yet.\n\nAlso prefer to use " @@ -167,6 +166,7 @@ public Builder toBuilder() { @Override public boolean isValid() { return true; } + } } diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 9c3f6f06ad3cc..82b05b3ed618f 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -93,11 +93,6 @@ public T get(final Class expectedType, final String name) { return RECORD_CONVERTERS.coerce(expectedType, value, name); } - @Override - public boolean isValid() { - return false; - } - @Override // for debug purposes, don't use it for anything else public String toString() { try (final Jsonb jsonb = JsonbBuilder @@ -522,7 +517,7 @@ public Builder withArray(final Schema.Entry entry, final Collection value @Override public Builder withError(String columnName, Object value, String errorMessage, Exception exception) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); - if (!supportError) { + if (!supportError) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry From beca0243ba72390a5525e219ef77e34e14c927e0 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 30 Apr 2025 14:19:02 +0800 Subject: [PATCH 30/62] Add sample-feature\supporterror --- .../sdk/component/api/record/Record.java | 3 +- .../beam/spi/record/AvroRecordBuilder.java | 4 +- .../spi/record/AvroRecordBuilderTest.java | 7 +- sample-parent/sample-features/pom.xml | 1 + .../sample-features/supporterror/pom.xml | 104 +++++++++ .../feature/supporterror/SVersions.java | 23 ++ .../sample/feature/supporterror/Cli.java | 212 ++++++++++++++++++ .../supporterror/SupportErrorInput.java | 131 +++++++++++ .../supporterror/SupportErrorMapper.java | 60 +++++ .../feature/supporterror/package-info.java | 21 ++ ...nent.runtime.serialization.ContainerFinder | 1 + .../src/main/resources/icons/dark/icon.svg | 5 + .../src/main/resources/icons/dark/mapper.svg | 65 ++++++ .../src/main/resources/icons/light/icon.svg | 5 + .../src/main/resources/icons/light/mapper.svg | 65 ++++++ .../feature/supporterror/Messages.properties | 27 +++ 16 files changed, 728 insertions(+), 6 deletions(-) create mode 100644 sample-parent/sample-features/supporterror/pom.xml create mode 100644 sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java create mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg create mode 100644 sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 28a9a3dd6bb6f..8fceaf047897a 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -315,7 +315,8 @@ default Optional getOptionalRecord(final String name) { default boolean isValid() { return !getSchema().getAllEntries() - .filter(entry -> !entry.isValid()).findAny() + .filter(entry -> !entry.isValid()) + .findAny() .isPresent(); } diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java index 92232af527616..35fd6082bd89c 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java @@ -36,12 +36,12 @@ public Record build() { return new AvroRecord(super.build()); } - protected Record.Builder getErrorBuilder(Schema.Builder builder) { + protected Record.Builder getErrorBuilder(final Schema.Builder builder) { return new AvroRecordBuilder(builder.build()); } - protected Schema.Builder getErrorBuilder(Schema.Type type) { + protected Schema.Builder getErrorBuilder(final Schema.Type type) { return new AvroSchemaBuilder().withType(type); } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 7777ef456b03a..bbfc3eda1e6b7 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -54,7 +54,6 @@ import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; -import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.record.SchemaImpl; import org.talend.sdk.component.runtime.record.SchemaImpl.EntryImpl; @@ -236,11 +235,13 @@ void testWithError() { .build(); assertFalse(record1.isValid()); - final Schema.Entry entry = record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); + final Schema.Entry entry = + record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); assertNotNull(entry); Assertions.assertFalse(entry.isValid()); - final Schema.Entry entry2 = record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); + final Schema.Entry entry2 = + record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); diff --git a/sample-parent/sample-features/pom.xml b/sample-parent/sample-features/pom.xml index 3214d3ab785d6..1b1046b48b1b5 100644 --- a/sample-parent/sample-features/pom.xml +++ b/sample-parent/sample-features/pom.xml @@ -32,6 +32,7 @@ aftergroup-lastgroup conditional-outputs checkpoint-runner + supporterror diff --git a/sample-parent/sample-features/supporterror/pom.xml b/sample-parent/sample-features/supporterror/pom.xml new file mode 100644 index 0000000000000..4f1279e001ebb --- /dev/null +++ b/sample-parent/sample-features/supporterror/pom.xml @@ -0,0 +1,104 @@ + + + + 4.0.0 + + + org.talend.sdk.component + sample-features + 1.81.0-SNAPSHOT + + + org.talend.sdk.component.sample.feature + supporterror + jar + + Component Runtime :: Sample Feature :: Supporterror + + + + org.talend.sdk.component + component-api + ${project.version} + compile + + + org.talend.sdk.component + component-runtime-manager + ${project.version} + compile + + + info.picocli + picocli + 4.7.6 + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + + + + true + org.talend.sdk.component.sample.feature.supporterror.Cli + + + + + + maven-shade-plugin + 2.1 + + + + shade + + package + + + + org.talend.sdk.component.sample.feature.supporterror.Cli + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.0.0 + + org.talend.sdk.component.sample.feature.supporterror.Cli + + + + + diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java new file mode 100644 index 0000000000000..0ad88da683be1 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java @@ -0,0 +1,23 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.talend.sdk.component.sample.feature.supporterror; + +public interface SVersions { + + String KIT_VERSION = "${project.version}"; + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java new file mode 100644 index 0000000000000..c39114ce26f44 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -0,0 +1,212 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.supporterror; + + +import static java.util.Optional.ofNullable; +import static lombok.AccessLevel.PRIVATE; +import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +import org.talend.sdk.component.api.record.Record; +import org.talend.sdk.component.api.record.Schema; +import org.talend.sdk.component.api.record.SchemaProperty; +import org.talend.sdk.component.api.service.record.RecordBuilderFactory; +import org.talend.sdk.component.dependencies.maven.Artifact; +import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; +import org.talend.sdk.component.runtime.input.Mapper; +import org.talend.sdk.component.runtime.manager.ComponentManager; +import org.talend.sdk.component.runtime.output.ProcessorImpl; +import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; +import org.talend.sdk.component.runtime.serialization.ContainerFinder; +import org.talend.sdk.component.runtime.serialization.LightContainer; + + +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = PRIVATE) +@Command(name="supportError") +public final class Cli implements Callable { + +// @Parameters(index = "0", description = "The file whose checksum to calculate.") +// private File file; + + //support errors or not. default=false + @Option(names = "-s", defaultValue = "true") + boolean support; + + @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") + File jar; + + @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") + String mapper; + + @Option(names = { "-fl", "--family" },defaultValue = "supporterror") + String family; + + static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + + SVersions.KIT_VERSION; + + @Override + public Integer call() throws Exception { + + try (final ComponentManager manager = manager(jar, GAV)) { + // final JsonObject jsonConfig = readJsonFromFile(configurationFile); + final Map configuration = new HashMap<>(); + + info("support" + String.valueOf(support)); + // create the mapper + final Mapper mpr = manager.findMapper(family, mapper, 1, configuration) + .orElseThrow(() -> new IllegalStateException( + String.format("No mapper found for: %s/%s.", family, manager))); + + final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) + .orElseThrow(() -> new IllegalStateException( + String.format("No Processor found for: %s/%s.", family, manager))); + + info("create input now."); + + List records = ((Supplier>) processor.getDelegate()).get(); + + //final SupportErrorInput input = mpr.create(); + //set the property for support or not +// input.setSupportError(String.valueOf(support)); + info("getting the record."); +// Record data = input.data(); + + for (Record data : records) { + info("Record : " + data.isValid()); + entryout(data, "date"); + entryout(data, "age"); + } + // + info("finished."); + } catch (Exception e) { + error(e); + } + + return 0; + } + + private static void entryout(final Record data, final String column) { + Schema.Entry ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny().get(); + if(ageEntry.isValid()) { + Integer age = data.get(Integer.class, column); + // process the age... + info("Record '"+ column +"': " + age); + } else{ + String errorMessage = ageEntry.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); + info("ERROR: " + errorMessage); + } + } + + public static void main(final String... args) { + int exitCode = new CommandLine(new Cli()).execute(args); + System.exit(exitCode); + } + + static final String ERROR = "[ERROR] "; + + static final String WARN = "[WARN] "; + + static final String INFO = "[INFO] "; + + static MvnCoordinateToFileConverter mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter(); + + + public static ComponentManager manager(final File jar, final String artifact) { + return new ComponentManager(findM2()) { + + final ContainerFinder containerFinder = ContainerFinder.Instance.get(); + + final ComponentManager originalMgr = contextualInstance().get(); + + { + contextualInstance().set(this); + String containerId; + if (jar != null) { + containerId = addPlugin(jar.getAbsolutePath()); + Cli.info(String.format("Manager is using plugin %s from %s.", containerId, jar)); + } else { + final String pluginPath = ofNullable(artifact) + .map(gav -> mvnCoordinateToFileConverter.toArtifact(gav)) + .map(Artifact::toPath) + .orElseThrow(() -> new IllegalArgumentException("Plugin GAV can't be empty")); + String p = findM2().resolve(pluginPath).toAbsolutePath().toString(); + containerId = addPlugin(p); + Cli.info(String.format("Manager is using plugin: %s from GAV %s.", containerId, artifact)); + } + DynamicContainerFinder.SERVICES.put(RecordBuilderFactory.class, + new RecordBuilderFactoryImpl(containerId)); + } + + @Override + public void close() { + DynamicContainerFinder.SERVICES.clear(); + super.close(); + contextualInstance().set(originalMgr); + } + }; + } + + public static class DynamicContainerFinder implements ContainerFinder { + + public static final Map LOADERS = new ConcurrentHashMap<>(); + + public static final Map, Object> SERVICES = new ConcurrentHashMap<>(); + + @Override + public LightContainer find(final String plugin) { + return new LightContainer() { + + @Override + public ClassLoader classloader() { + return LOADERS.get(plugin); + } + + @Override + public T findService(final Class key) { + return key.cast(SERVICES.get(key)); + } + }; + } + } + + public static void info(final String message) { + System.out.println(INFO + message); + } + + public static void warn(final String message) { + System.err.println(WARN + message); + } + + public static void error(final Throwable e) { + System.err.println(ERROR + e.getMessage()); + System.exit(501); + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java new file mode 100644 index 0000000000000..fafd69f1f96bc --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -0,0 +1,131 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.supporterror; + +import java.io.Serializable; +import java.util.Date; + +import javax.annotation.PostConstruct; + +import org.talend.sdk.component.api.component.Icon; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.type.DataSet; +import org.talend.sdk.component.api.configuration.type.DataStore; +import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.Producer; +import org.talend.sdk.component.api.meta.Documentation; +import org.talend.sdk.component.api.record.Record; +import org.talend.sdk.component.api.record.Schema; +import org.talend.sdk.component.runtime.record.RecordImpl; +import org.talend.sdk.component.runtime.record.SchemaImpl; + +import lombok.Data; + +@Version +@Icon(value = Icon.IconType.CUSTOM, custom = "icon") +@Documentation("Support Error Input sample processor connector.") +@Emitter(family = "supporterror", name = "supportErrorInput") +public class SupportErrorInput implements Serializable { + +// private final transient JsonBuilderFactory factory; + + private transient Schema recordSchema; + + private final transient InputConfig configuration; + + public SupportErrorInput(@Option("configuration") final InputConfig config) { + this.configuration = config; + } + + @PostConstruct + public void init() { + recordSchema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("age") + .withNullable(false) + .withType(Schema.Type.INT) + .build()) + .build(); + } + + //set support or not. + public void setSupportError(final String supportError) { + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + } + + @Producer + public Record data() { + final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); + final Record record = builder.withError("date", null, "date is null", null) + .withError("age", "string", "wrong int value", null) + .build(); + + return record; + } + + @Data + @GridLayout(value = { + @GridLayout.Row("dataset"), + }) + @Version + public static class InputConfig { + + @Option + @Documentation("Dataset.") + private Dataset dataset = new Dataset(); + } + + @DataSet + @Data + @GridLayout(value = { @GridLayout.Row("datastore") }) + public static class Dataset implements Serializable { + + @Option + @Documentation("Datastore.") + private Datastore datastore = new Datastore(); + + } + + @DataStore + @Data + @GridLayout(value = { @GridLayout.Row("name"), @GridLayout.Row("age"), + @GridLayout.Row("date")}) + public static class Datastore implements Serializable { + + @Option + @Documentation("Name prop.") + private String name = "test"; + + @Option + @Documentation("Age prop.") + private int age = 0; + + @Option + @Documentation("Date prop.") + private Date date; + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java new file mode 100644 index 0000000000000..3aa4210aae676 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.supporterror; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +import org.talend.sdk.component.api.component.Icon; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.input.Assessor; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.PartitionMapper; +import org.talend.sdk.component.api.input.PartitionSize; +import org.talend.sdk.component.api.input.Split; +import org.talend.sdk.component.api.meta.Documentation; + +@Version(1) +@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") +@PartitionMapper(name = "SupportErrorMapper", infinite = false) +@Documentation("Doc: default SupportErrorMapper documentation without Internationalization.") +public class SupportErrorMapper implements Serializable { + + private SupportErrorInput.InputConfig config; + + public SupportErrorMapper(final @Option("configin") SupportErrorInput.InputConfig config) { + this.config = config; + } + + @Assessor + public long estimateSize() { + return 1500L; + } + + @Split + public List split(@PartitionSize final int desiredNbSplits) { + return Collections.singletonList(this); + } + + @Emitter + public SupportErrorInput createSource() { + + return new SupportErrorInput(this.config); + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java new file mode 100644 index 0000000000000..ff1441932bd2a --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@Components(family = "supporterror", categories = "sample") +@Icon(value = Icon.IconType.CUSTOM, custom = "icon") +package org.talend.sdk.component.sample.feature.supporterror; + +import org.talend.sdk.component.api.component.Components; +import org.talend.sdk.component.api.component.Icon; \ No newline at end of file diff --git a/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder b/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder new file mode 100644 index 0000000000000..0d23def70f392 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder @@ -0,0 +1 @@ +org.talend.sdk.component.runtime.manager.finder.StandaloneContainerFinder diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg new file mode 100644 index 0000000000000..3351e2c0bc9e9 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg @@ -0,0 +1,5 @@ + + + + Checkpoint​Input + \ No newline at end of file diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg new file mode 100644 index 0000000000000..4f0fafff9a769 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg @@ -0,0 +1,65 @@ + + + + + + + + + + + + + diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg new file mode 100644 index 0000000000000..3351e2c0bc9e9 --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg @@ -0,0 +1,5 @@ + + + + Checkpoint​Input + \ No newline at end of file diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg new file mode 100644 index 0000000000000..2958a0056838e --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg @@ -0,0 +1,65 @@ + + + + + + + + + + + + + diff --git a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties new file mode 100644 index 0000000000000..aa81e7a64c5fa --- /dev/null +++ b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties @@ -0,0 +1,27 @@ +# Copyright (C) 2006-2025 Talend Inc. - www.talend.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Here you can change all your configuration display names to use more explicit labels +# You can also translate your configuration by adding one file by local Messages_fr.properties for french for example +supporterror.supportErrorInput._displayName=Support Error Input +supporterror.SupportErrorMapper._displayName=Support Error Mapper +supporterror.datastore.default._displayName=default +supporterror.dataset.default._displayName=dataset +Dataset.datastore._displayName=Datastore +InputConfig.dataset._displayName=Dataset +Datastore.age._displayName=age +Datastore.date._displayName=date +Datastore.name._displayName=name +Datastore.age._placeholder= +Datastore.date._placeholder= +Datastore.name._placeholder= \ No newline at end of file From ad5aeae79a7fbfd058eb0667ecd6afafc4b1c9b6 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 30 Apr 2025 15:39:16 +0800 Subject: [PATCH 31/62] worked --- .../sample/feature/supporterror/Cli.java | 49 +++++++++---------- .../supporterror/SupportErrorInput.java | 7 --- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index c39114ce26f44..608422e334bee 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -26,11 +26,9 @@ import java.io.File; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Supplier; import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; @@ -38,9 +36,8 @@ import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; -import org.talend.sdk.component.runtime.input.Mapper; +import org.talend.sdk.component.runtime.input.PartitionMapperImpl; import org.talend.sdk.component.runtime.manager.ComponentManager; -import org.talend.sdk.component.runtime.output.ProcessorImpl; import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; @@ -52,11 +49,8 @@ @Command(name="supportError") public final class Cli implements Callable { -// @Parameters(index = "0", description = "The file whose checksum to calculate.") -// private File file; - //support errors or not. default=false - @Option(names = "-s", defaultValue = "true") + @Option(names = "-s", defaultValue = "false") boolean support; @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") @@ -69,7 +63,7 @@ public final class Cli implements Callable { String family; static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" - + SVersions.KIT_VERSION; + + Versions.KIT_VERSION; @Override public Integer call() throws Exception { @@ -79,31 +73,30 @@ public Integer call() throws Exception { final Map configuration = new HashMap<>(); info("support" + String.valueOf(support)); + if (support) { + setSupportError(support); + } // create the mapper - final Mapper mpr = manager.findMapper(family, mapper, 1, configuration) + final PartitionMapperImpl mpr = (PartitionMapperImpl)manager.findMapper(family, mapper, 1, configuration) .orElseThrow(() -> new IllegalStateException( String.format("No mapper found for: %s/%s.", family, manager))); - final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) - .orElseThrow(() -> new IllegalStateException( - String.format("No Processor found for: %s/%s.", family, manager))); +// final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) +// .orElseThrow(() -> new IllegalStateException( +// String.format("No Processor found for: %s/%s.", family, manager))); info("create input now."); - List records = ((Supplier>) processor.getDelegate()).get(); + SupportErrorInput seInput = new SupportErrorInput(null); + seInput.init(); - //final SupportErrorInput input = mpr.create(); - //set the property for support or not -// input.setSupportError(String.valueOf(support)); info("getting the record."); -// Record data = input.data(); + Record data = seInput.data(); - for (Record data : records) { - info("Record : " + data.isValid()); - entryout(data, "date"); - entryout(data, "age"); - } - // + info("Record isValid = " + data.isValid()); + entryout(data, "date"); + entryout(data, "age"); + // info("finished."); } catch (Exception e) { error(e); @@ -124,6 +117,13 @@ private static void entryout(final Record data, final String column) { } } + //set support or not. + public void setSupportError(final boolean supportError) { + final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); + final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); + } + public static void main(final String... args) { int exitCode = new CommandLine(new Cli()).execute(args); System.exit(exitCode); @@ -137,7 +137,6 @@ public static void main(final String... args) { static MvnCoordinateToFileConverter mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter(); - public static ComponentManager manager(final File jar, final String artifact) { return new ComponentManager(findM2()) { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index fafd69f1f96bc..99758408a25d8 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -69,13 +69,6 @@ public void init() { .build(); } - //set support or not. - public void setSupportError(final String supportError) { - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); - } - @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); From 53d8c05aa4bfac1d06b64d7cfd35a1e559911811 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 14:44:42 +0800 Subject: [PATCH 32/62] fix bug --- .../beam/spi/record/AvroRecordBuilder.java | 9 ------- .../spi/record/AvroRecordBuilderTest.java | 9 ++++++- .../component/runtime/record/RecordImpl.java | 27 ++++--------------- .../runtime/record/RecordBuilderImplTest.java | 12 +++++++-- .../sample/feature/supporterror/Cli.java | 7 ++--- .../supporterror/SupportErrorInput.java | 8 +++++- .../feature/supporterror/Messages.properties | 3 ++- 7 files changed, 34 insertions(+), 41 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java index 35fd6082bd89c..0ce9e2c4c21b6 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilder.java @@ -35,13 +35,4 @@ public AvroRecordBuilder(final Schema providedSchema) { public Record build() { return new AvroRecord(super.build()); } - - protected Record.Builder getErrorBuilder(final Schema.Builder builder) { - - return new AvroRecordBuilder(builder.build()); - } - - protected Schema.Builder getErrorBuilder(final Schema.Type type) { - return new AvroSchemaBuilder().withType(type); - } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index bbfc3eda1e6b7..c3f5c9e8b5c7e 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -221,16 +221,23 @@ void testWithError() { .withNullable(false) .withType(Schema.Type.STRING) .build(); + Schema.Entry nmEntry = factory + .newEntryBuilder() + .withName("normal") + .withNullable(true) + .withType(Schema.Type.STRING) + .build(); Schema.Entry ageEntry = factory .newEntryBuilder() .withName("age") .withNullable(false) .withType(Schema.Type.INT) .build(); - Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(ageEntry).build(); + Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); // record 1 Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); Record record1 = recordBuilder.withError("name", null, "Stirng is null", null) + .withString("normal", "normal") .withError("age", "string", "is not an int", null) .build(); assertFalse(record1.isValid()); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 82b05b3ed618f..f91d8e0b17e7e 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -61,10 +61,10 @@ import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.EntriesOrder; import org.talend.sdk.component.api.record.Schema.Entry; +import org.talend.sdk.component.api.record.SchemaProperty; import lombok.EqualsAndHashCode; import lombok.Getter; -import org.talend.sdk.component.api.record.SchemaProperty; @EqualsAndHashCode public final class RecordImpl implements Record { @@ -515,9 +515,10 @@ public Builder withArray(final Schema.Entry entry, final Collection value } @Override - public Builder withError(String columnName, Object value, String errorMessage, Exception exception) { + public Builder withError(final String columnName, final Object value, final String errorMessage, + final Exception exception) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); - if (!supportError) { + if (!supportError) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry @@ -531,28 +532,10 @@ public Builder withError(String columnName, Object value, String errorMessage, E .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) .withProp(SchemaProperty.ERROR_EXCEPTION, exception == null ? "" : exception.toString()) .build(); - final Schema.Builder builder = getErrorBuilder(providedSchema.getType()); - providedSchema.getAllEntries() -// .filter(e -> Objects.equals(providedSchema.getEntry(e.getName()), e)) - .forEach(e -> { - if (columnName.equals(e.getName())) { - builder.withEntry(updateEntry); - } else { - builder.withEntry(e); - } - }); - return getErrorBuilder(builder); + return updateEntryByName(columnName, updateEntry); } } - protected Builder getErrorBuilder(Schema.Builder builder) { - return new BuilderImpl(builder.build()); - } - - protected Schema.Builder getErrorBuilder(Schema.Type type) { - return new SchemaImpl.BuilderImpl().withType(type); - } - private void assertType(final Schema.Type actual, final Schema.Type expected) { if (actual != expected) { throw new IllegalArgumentException("Expected entry type: " + expected + ", got: " + actual); diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 9cddee216ca8b..6c8fea7482602 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -258,6 +258,11 @@ void withError() { .withNullable(false) .withType(Schema.Type.DATETIME) .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("normal") + .withNullable(true) + .withType(Type.STRING) + .build()) .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() .withName("intValue") .withNullable(false) @@ -269,14 +274,17 @@ void withError() { final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); final Record record3 = builder3.withError("date", null, "date is null", null) + .withString("normal", "normal") .withError("intValue", "string", "wrong int value", null) .build(); assertFalse(record3.isValid()); - final Entry entry = record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); + final Entry entry = + record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); assertNotNull(entry); Assertions.assertFalse(entry.isValid()); - final Entry entry2 = record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); + final Entry entry2 = + record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 608422e334bee..780d19a007dc2 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -50,7 +50,7 @@ public final class Cli implements Callable { //support errors or not. default=false - @Option(names = "-s", defaultValue = "false") + @Option(names = "-s", defaultValue = "true") boolean support; @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") @@ -81,10 +81,6 @@ public Integer call() throws Exception { .orElseThrow(() -> new IllegalStateException( String.format("No mapper found for: %s/%s.", family, manager))); -// final ProcessorImpl processor = (ProcessorImpl)manager.findMapper(family, "SupportErrorInput", 1, configuration) -// .orElseThrow(() -> new IllegalStateException( -// String.format("No Processor found for: %s/%s.", family, manager))); - info("create input now."); SupportErrorInput seInput = new SupportErrorInput(null); @@ -94,6 +90,7 @@ public Integer call() throws Exception { Record data = seInput.data(); info("Record isValid = " + data.isValid()); + entryout(data, "name"); entryout(data, "date"); entryout(data, "age"); // diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index 99758408a25d8..9982aacf4641d 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -56,6 +56,11 @@ public SupportErrorInput(@Option("configuration") final InputConfig config) { public void init() { recordSchema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("name") + .withNullable(false) + .withType(Schema.Type.STRING) + .build()) .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() .withName("date") .withNullable(false) @@ -72,7 +77,8 @@ public void init() { @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - final Record record = builder.withError("date", null, "date is null", null) + final Record record = builder.withString("name", "example connector") + .withError("date", null, "date is null", null) .withError("age", "string", "wrong int value", null) .build(); diff --git a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties index aa81e7a64c5fa..0e589d31e4429 100644 --- a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties +++ b/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties @@ -14,6 +14,7 @@ # Here you can change all your configuration display names to use more explicit labels # You can also translate your configuration by adding one file by local Messages_fr.properties for french for example supporterror.supportErrorInput._displayName=Support Error Input +supporterror.supportErrorAvroInput._displayName=Support Error Avro Input supporterror.SupportErrorMapper._displayName=Support Error Mapper supporterror.datastore.default._displayName=default supporterror.dataset.default._displayName=dataset @@ -24,4 +25,4 @@ Datastore.date._displayName=date Datastore.name._displayName=name Datastore.age._placeholder= Datastore.date._placeholder= -Datastore.name._placeholder= \ No newline at end of file +Datastore.name._placeholder= From 327161f893ee6499587005c1255409f60a177dbd Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 16:33:26 +0800 Subject: [PATCH 33/62] Add new method to white list. --- .../talend/sdk/component/tools/validator/RecordValidator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java index 8254a0eec55c5..a7fed49240bdc 100644 --- a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java +++ b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java @@ -132,6 +132,7 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.Record$Builder.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", + "org.talend.sdk.component.api.record.Record$Builder.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)", "org.talend.sdk.component.api.record.RecordImpl.withNewSchema(org.talend.sdk.component.api.record.Schema)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.with(org.talend.sdk.component.api.record.Schema$Entry, java.lang.Object)", @@ -161,6 +162,7 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withBoolean(org.talend.sdk.component.api.record.Schema$Entry, boolean)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", - "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)")); + "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", + "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)")); } From 92b02ae8061cd51d25f4cba66c4f78f22779cf94 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 16:47:32 +0800 Subject: [PATCH 34/62] Add "valid":true to serialization --- .../runtime/beam/spi/record/JsonSchemaSerializationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java index 607083f593ef2..1993f10373c98 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java @@ -49,7 +49,7 @@ void toJson() throws Exception { .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { final String json = jsonb.toJson(schema); assertEquals( - "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", json); } } @@ -71,7 +71,7 @@ void toJsonWithMeta() throws Exception { try (final Jsonb jsonb = JsonbBuilder .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { assertEquals( - "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", jsonb.toJson(schema)); } From 042b9aa3b821c012a0fedb483dd81a20a31d8ada Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 6 May 2025 19:59:58 +0800 Subject: [PATCH 35/62] Add "valid":true to serialization --- .../sdk/component/server/front/ActionResourceImplTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java index e357820eecc57..85bb91a1e62c7 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java @@ -199,7 +199,8 @@ void checkSchemaSerialization() { " \"metadata\":[\n ],\n \"props\":{\n\n },\n \"type\":\"STRING\"\n" + " },\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + - " \"props\":{\n\n },\n \"type\":\"ARRAY\"\n }\n ],\n \"metadata\":[\n" + + " \"props\":{\n\n },\n \"type\":\"ARRAY\",\n" + + " \"valid\":true\n }\n ],\n \"metadata\":[\n" + " ],\n \"props\":{\n \"talend.fields.order\":\"array\"\n },\n \"type\":\"RECORD\"\n}"; assertEquals(expected, schema); } @@ -238,7 +239,7 @@ void checkDiscoverProcessorSchema() { }, APPLICATION_JSON_TYPE), JsonObject.class); assertNotNull(guessed); final String expected = - "{\"entries\":[{\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\"},{\"comment\":\"field2 comment\",\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\"},{\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\"},{\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; + "{\"entries\":[{\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"comment\":\"field2 comment\",\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\",\"valid\":true},{\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; assertEquals(expected, guessed.toString()); } From a7b28c21607f9395125439790ee0ce2b4ad2bd80 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 09:49:27 +0800 Subject: [PATCH 36/62] Add "valid":true to serialization --- .../server/front/beam/BeamActionSerializationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java index 553bfe64c4870..e30e2286e2d3c 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java @@ -46,7 +46,8 @@ void checkSchemaSerialization() { + " \"entries\":[\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"STRING\"\n" + " },\n" + " \"metadata\":false,\n" + " \"name\":\"array\",\n" + " \"nullable\":false,\n" - + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\"\n" + " }\n" + " ],\n" + + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\",\n" + + " \"valid\":true\n }\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + " \"talend.fields.order\":\"array\"\n" + " },\n" + " \"type\":\"RECORD\"\n" + "}"; assertEquals(attended, schema); From 68e1f0d4ae3c77e94aae1140d752fcaead71ca7a Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 10:05:38 +0800 Subject: [PATCH 37/62] rename --- .../feature/supporterror/{SVersions.java => Versions.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/{SVersions.java => Versions.java} (87%) diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java similarity index 87% rename from sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java rename to sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java index 0ad88da683be1..8f19afc5c0e3e 100644 --- a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/SVersions.java +++ b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java @@ -16,8 +16,8 @@ package org.talend.sdk.component.sample.feature.supporterror; -public interface SVersions { +public interface Versions { - String KIT_VERSION = "${project.version}"; + String KIT_VERSION = "1.81.0-SNAPSHOT";//"${project.version}"; } From c51e830dc462671cf113a07dd94e83ef235c1838 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 10:21:33 +0800 Subject: [PATCH 38/62] revert junit which no need to change --- .../spi/record/AvroRecordBuilderTest.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index c3f5c9e8b5c7e..797b8b23f06e9 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -54,6 +54,7 @@ import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; +import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.record.SchemaImpl; import org.talend.sdk.component.runtime.record.SchemaImpl.EntryImpl; @@ -191,21 +192,21 @@ void avroTest() { @Test void mixedRecordTest() { - final Schema schema0 = new AvroSchemaBuilder()// - .withType(RECORD) // - .withEntry(dataEntry1) // - .withEntryBefore("data1", meta1) // - .withEntry(dataEntry2) // - .withEntryAfter("meta1", meta2) // + final AvroRecordBuilderFactoryProvider recordBuilderFactoryProvider = new AvroRecordBuilderFactoryProvider(); + System.setProperty("talend.component.beam.record.factory.impl", "avro"); + final RecordBuilderFactory recordBuilderFactory = recordBuilderFactoryProvider.apply("test"); + + final RecordBuilderFactory otherFactory = new RecordBuilderFactoryImpl("test"); + final Schema schema = otherFactory + .newSchemaBuilder(RECORD) + .withEntry(otherFactory.newEntryBuilder().withName("e1").withType(INT).build()) .build(); - final Record.Builder builder0 = factory.newRecordBuilder(schema0); - builder0.withInt("data1", 101) - .withString("data2", "102") - .withInt("meta1", 103) - .withString("meta2", "104"); - final Record record0 = builder0.build(); - assertEquals(101, record0.getInt("data1")); + final Schema arrayType = recordBuilderFactory // + .newSchemaBuilder(Schema.Type.ARRAY) // + .withElementSchema(schema) + .build(); + Assertions.assertNotNull(arrayType); } @Test From a830a9650467017fda301e4c22f17f2703896557 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 10:24:48 +0800 Subject: [PATCH 39/62] fix one problem --- .../sample/feature/supporterror/Cli.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 780d19a007dc2..bb36e2684db20 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -15,7 +15,6 @@ */ package org.talend.sdk.component.sample.feature.supporterror; - import static java.util.Optional.ofNullable; import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; @@ -25,6 +24,7 @@ import picocli.CommandLine.Option; import java.io.File; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; @@ -42,7 +42,6 @@ import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; - import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) @@ -50,7 +49,7 @@ public final class Cli implements Callable { //support errors or not. default=false - @Option(names = "-s", defaultValue = "true") + @Option(names = "-s", defaultValue = "false") boolean support; @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") @@ -90,9 +89,9 @@ public Integer call() throws Exception { Record data = seInput.data(); info("Record isValid = " + data.isValid()); - entryout(data, "name"); - entryout(data, "date"); - entryout(data, "age"); + entryout(data, "name", String.class); + entryout(data, "date", Date.class); + entryout(data, "age", Integer.class); // info("finished."); } catch (Exception e) { @@ -102,12 +101,11 @@ public Integer call() throws Exception { return 0; } - private static void entryout(final Record data, final String column) { + private static void entryout(final Record data, final String column, final Class type) { Schema.Entry ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny().get(); if(ageEntry.isValid()) { - Integer age = data.get(Integer.class, column); - // process the age... - info("Record '"+ column +"': " + age); + Object value = data.get(type, column); + info("Record '"+ column +"': " + value); } else{ String errorMessage = ageEntry.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); info("ERROR: " + errorMessage); From 1d46e1fbaac01454b152da0e460679645d880099 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 15:42:53 +0800 Subject: [PATCH 40/62] fix import order --- .../sdk/component/sample/feature/supporterror/Cli.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index bb36e2684db20..5e95c5fb2008f 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,10 +19,6 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - import java.io.File; import java.util.Date; import java.util.HashMap; @@ -42,6 +38,10 @@ import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) From a537dfd271e9abe11a6bad1185cf230c2611a8b0 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 7 May 2025 16:10:05 +0800 Subject: [PATCH 41/62] fix import order --- .../sdk/component/sample/feature/supporterror/Cli.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 5e95c5fb2008f..bb36e2684db20 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,6 +19,10 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + import java.io.File; import java.util.Date; import java.util.HashMap; @@ -38,10 +42,6 @@ import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) From fb73c8da23936cb4bff235768745354af5522f5f Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 8 May 2025 14:19:52 +0800 Subject: [PATCH 42/62] change command jar --- .../sample-features/supporterror/pom.xml | 9 +-- .../sample/feature/supporterror/Cli.java | 59 ++++++++++--------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/sample-parent/sample-features/supporterror/pom.xml b/sample-parent/sample-features/supporterror/pom.xml index 4f1279e001ebb..b5a57cfc4f32e 100644 --- a/sample-parent/sample-features/supporterror/pom.xml +++ b/sample-parent/sample-features/supporterror/pom.xml @@ -33,19 +33,16 @@ org.talend.sdk.component component-api ${project.version} - compile org.talend.sdk.component component-runtime-manager ${project.version} - compile - info.picocli - picocli - 4.7.6 - compile + org.tomitribe + tomitribe-crest + 0.32 diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index bb36e2684db20..7de7599cc09cd 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,15 +19,11 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; import java.io.File; import java.util.Date; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import org.talend.sdk.component.api.record.Record; @@ -41,31 +37,39 @@ import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; import org.talend.sdk.component.runtime.serialization.LightContainer; +import org.tomitribe.crest.Main; +import org.tomitribe.crest.api.Command; +import org.tomitribe.crest.api.Default; +import org.tomitribe.crest.api.Option; import lombok.NoArgsConstructor; @NoArgsConstructor(access = PRIVATE) -@Command(name="supportError") -public final class Cli implements Callable { - - //support errors or not. default=false - @Option(names = "-s", defaultValue = "false") - boolean support; - - @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") - File jar; - - @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") - String mapper; - - @Option(names = { "-fl", "--family" },defaultValue = "supporterror") - String family; +public final class Cli { + +// //support errors or not. default=false +// @Option(names = "-s", defaultValue = "false") +// boolean support; +// +// @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") +// File jar; +// +// @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") +// String mapper; +// +// @Option(names = { "-fl", "--family" },defaultValue = "supporterror") +// String family; static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + Versions.KIT_VERSION; - @Override - public Integer call() throws Exception { + @Command("supporterror") + public static void runInput( + @Option("gav") @Default(GAV) final String gav, + @Option("s") @Default("false") final boolean support, + @Option("jar") final File jar, + @Option("family") @Default("supporterror") final String family, + @Option("mapper") @Default("SupportErrorMapper") final String mapper) { try (final ComponentManager manager = manager(jar, GAV)) { // final JsonObject jsonConfig = readJsonFromFile(configurationFile); @@ -97,8 +101,6 @@ public Integer call() throws Exception { } catch (Exception e) { error(e); } - - return 0; } private static void entryout(final Record data, final String column, final Class type) { @@ -113,15 +115,18 @@ private static void entryout(final Record data, final String column, final Class } //set support or not. - public void setSupportError(final boolean supportError) { + public static void setSupportError(final boolean supportError) { final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); } - public static void main(final String... args) { - int exitCode = new CommandLine(new Cli()).execute(args); - System.exit(exitCode); + public static void main(final String[] args) throws Exception { + ofNullable(run(args)).ifPresent(System.out::println); + } + + public static Object run(final String[] args) throws Exception { + return new Main(Cli.class).exec(args); } static final String ERROR = "[ERROR] "; From 9a408f4e7ab7b36cb3f0520d26b6180717260a75 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 8 May 2025 15:38:33 +0800 Subject: [PATCH 43/62] fix sonar --- .../sdk/component/api/record/Record.java | 4 +- .../sample/feature/supporterror/Cli.java | 40 ++++++++----------- .../supporterror/SupportErrorInput.java | 5 +-- .../supporterror/SupportErrorMapper.java | 2 +- 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 8fceaf047897a..06a3e4c5956b7 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -315,9 +315,7 @@ default Optional getOptionalRecord(final String name) { default boolean isValid() { return !getSchema().getAllEntries() - .filter(entry -> !entry.isValid()) - .findAny() - .isPresent(); + .anyMatch(entry -> !entry.isValid()); } /** diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 7de7599cc09cd..26019ec792190 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -24,6 +24,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.talend.sdk.component.api.record.Record; @@ -32,7 +33,6 @@ import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; -import org.talend.sdk.component.runtime.input.PartitionMapperImpl; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; @@ -72,18 +72,12 @@ public static void runInput( @Option("mapper") @Default("SupportErrorMapper") final String mapper) { try (final ComponentManager manager = manager(jar, GAV)) { - // final JsonObject jsonConfig = readJsonFromFile(configurationFile); final Map configuration = new HashMap<>(); - info("support" + String.valueOf(support)); + info("support " + support); if (support) { setSupportError(support); } - // create the mapper - final PartitionMapperImpl mpr = (PartitionMapperImpl)manager.findMapper(family, mapper, 1, configuration) - .orElseThrow(() -> new IllegalStateException( - String.format("No mapper found for: %s/%s.", family, manager))); - info("create input now."); SupportErrorInput seInput = new SupportErrorInput(null); @@ -93,9 +87,9 @@ public static void runInput( Record data = seInput.data(); info("Record isValid = " + data.isValid()); - entryout(data, "name", String.class); - entryout(data, "date", Date.class); - entryout(data, "age", Integer.class); + entryOut(data, "name", String.class); + entryOut(data, "date", Date.class); + entryOut(data, "age", Integer.class); // info("finished."); } catch (Exception e) { @@ -103,22 +97,22 @@ public static void runInput( } } - private static void entryout(final Record data, final String column, final Class type) { - Schema.Entry ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny().get(); - if(ageEntry.isValid()) { - Object value = data.get(type, column); - info("Record '"+ column +"': " + value); - } else{ - String errorMessage = ageEntry.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); - info("ERROR: " + errorMessage); + private static void entryOut(final Record data, final String column, final Class type) { + Optional ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny(); + if(ageEntry.isPresent()) { + if (ageEntry.get().isValid()) { + Object value = data.get(type, column); + info("Record '" + column + "': " + value); + } else { + String errorMessage = ageEntry.get().getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); + info("ERROR: " + errorMessage); + } } } //set support or not. public static void setSupportError(final boolean supportError) { - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); } public static void main(final String[] args) throws Exception { @@ -174,9 +168,9 @@ public void close() { public static class DynamicContainerFinder implements ContainerFinder { - public static final Map LOADERS = new ConcurrentHashMap<>(); + static final Map LOADERS = new ConcurrentHashMap<>(); - public static final Map, Object> SERVICES = new ConcurrentHashMap<>(); + static final Map, Object> SERVICES = new ConcurrentHashMap<>(); @Override public LightContainer find(final String plugin) { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index 9982aacf4641d..13fe1001201cc 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -42,7 +42,6 @@ @Emitter(family = "supporterror", name = "supportErrorInput") public class SupportErrorInput implements Serializable { -// private final transient JsonBuilderFactory factory; private transient Schema recordSchema; @@ -77,12 +76,10 @@ public void init() { @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - final Record record = builder.withString("name", "example connector") + return builder.withString("name", "example connector") .withError("date", null, "date is null", null) .withError("age", "string", "wrong int value", null) .build(); - - return record; } @Data diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java index 3aa4210aae676..50f078aec5147 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java @@ -35,7 +35,7 @@ @Documentation("Doc: default SupportErrorMapper documentation without Internationalization.") public class SupportErrorMapper implements Serializable { - private SupportErrorInput.InputConfig config; + private transient SupportErrorInput.InputConfig config; public SupportErrorMapper(final @Option("configin") SupportErrorInput.InputConfig config) { this.config = config; From 5af6128cbb604cd7a9199a843720fdee3f674383 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 8 May 2025 15:44:14 +0800 Subject: [PATCH 44/62] remove comment --- .../component/sample/feature/supporterror/Cli.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index 26019ec792190..c70fce4534add 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -19,7 +19,6 @@ import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; - import java.io.File; import java.util.Date; import java.util.HashMap; @@ -47,19 +46,6 @@ @NoArgsConstructor(access = PRIVATE) public final class Cli { -// //support errors or not. default=false -// @Option(names = "-s", defaultValue = "false") -// boolean support; -// -// @Option(names = { "-f", "--file" }, paramLabel = "ARCHIVE", description = "the jar file") -// File jar; -// -// @Option(names = { "-m", "--mapper" },defaultValue = "SupportErrorMapper") -// String mapper; -// -// @Option(names = { "-fl", "--family" },defaultValue = "supporterror") -// String family; - static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + Versions.KIT_VERSION; From a3b0fda4cf4f3260f0c29031ce49dd10b286da7c Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Fri, 9 May 2025 13:25:00 +0800 Subject: [PATCH 45/62] add readme --- .../sample-features/supporterror/README.md | 89 +++++++++++++++++++ .../sample/feature/supporterror/Versions.java | 2 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 sample-parent/sample-features/supporterror/README.md diff --git a/sample-parent/sample-features/supporterror/README.md b/sample-parent/sample-features/supporterror/README.md new file mode 100644 index 0000000000000..6b5756f4d20ff --- /dev/null +++ b/sample-parent/sample-features/supporterror/README.md @@ -0,0 +1,89 @@ +# Component Runtime :: Sample Feature :: SupportError + +## Table of Contents +- [Overview](#overview) +- [Usage](#usage) + - [How to build the sample connector plugin](#how-to-build-the-sample-connector-plugin) + - [How to run](#how-to-run) + - [How to use](#how-to-use) +- [Plugin artifact](#plugin-artifact) +- [Execution examples](#execution-examples) + - [Run supporterror connector with default behavior](#run-checkpoint-connector-with-default-behavior) + - [Run with framework feature disabled](#run-with-framework-feature-disabled) + + +## Overview +This is a simple TCK connector plugin to test and validate the feature about Improve Record API to add error info for invalid entries. + +This project contains two things: + 1. A sample connector plugin that implements the supporterror feature. + 2. A simple Cli runner that can be used to run the connector plugin and test the supporterror feature. + + +## Usage +### How to build the sample connector plugin +Checkout the code from the repository and build the project using `mvn clean install` +Alternatively build the feature module using `mvn install -am -pl :supporterror` + +### How to run +To run the connector, you need exec the generated artifact `org.talend.sdk.component.sample.feature:supporterror`. +* You can run it directly from `target` folder or repository folder + * `java -jar target/supporterror-1.81.0-SNAPSHOT.jar` +* or you can run it from the maven repository + * `java -jar ~/.m2/repository/org/talend/sdk/component/sample/feature/supporterror/1.81.0-SNAPSHOT/supporterror-1.81.0-SNAPSHOT.jar` + +For later usage, will use the variable `$RUNCMD` as the way you may choose. +⚠️ If you're using jdk17, don't forget to add the `--add-opens` option to the command line +(see profile _jdk9_ in master pom at the repository's root) or use instead jdk11. + +### How to use +Run supporterror with the option "-s" for execution: + +```bash +$ $RUNCMD supporterror + +Usage: supporterror [-s] + +Options: + --s use supporterror or not. + +``` + +## Plugin artifact +There are two ways to run the supporterror runner with a specific plugin artifact: +- Using the `--gav` option to specify the GAV of the plugin artifact. + Syntax: `groupId:artifactId:version[:packaging[:classifier]]` + +- Using the `--jar` option to specify the path to the plugin artifact. + Syntax: `/path/to/plugin.jar` + +⚠️ You cannot use both options at the same time. + +## Execution examples +### Run supporterror connector with default behavior +`java -jar target/supporterror-1.81.0-SNAPSHOT.jar supporterror -s` + +```bash +[INFO] Manager is using plugin: supporterror from GAV org.talend.sdk.component.sample.feature:supporterror:jar:1.81.0-SNAPSHOT. +[INFO] support true +[INFO] create input now. +[INFO] getting the record. +[INFO] Record isValid = false +[INFO] Record 'name': example connector +[INFO] ERROR: date is null +[INFO] ERROR: wrong int value +[INFO] finished. +``` + +### Run with framework feature disabled +In this example we turn off the supporterror capability. +`% java -jar target/supporterror-1.81.0-SNAPSHOT.jar supporterror` + +```bash +[INFO] Manager is using plugin: supporterror from GAV org.talend.sdk.component.sample.feature:supporterror:jar:1.81.0-SNAPSHOT. +[INFO] support false +[INFO] create input now. +[INFO] getting the record. +[ERROR] date is null +``` +We can see that no record is returned because throw errors. diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java index 8f19afc5c0e3e..242683a0ce5e8 100644 --- a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java +++ b/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java @@ -18,6 +18,6 @@ public interface Versions { - String KIT_VERSION = "1.81.0-SNAPSHOT";//"${project.version}"; + String KIT_VERSION = "${project.version}"; } From 96b5897188d5494da4fc00e9ef8f8a60448b6a90 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 12 May 2025 15:11:07 +0800 Subject: [PATCH 46/62] remove withError in record API --- .../sdk/component/api/record/Record.java | 4 +- .../spi/record/AvroRecordBuilderTest.java | 6 +- .../component/runtime/record/RecordImpl.java | 24 +++---- .../runtime/record/RecordBuilderImplTest.java | 70 ++++++++++++++----- .../tools/validator/RecordValidator.java | 4 +- .../sample/feature/supporterror/Cli.java | 3 - .../supporterror/SupportErrorInput.java | 58 ++++++++------- .../supporterror/SupportErrorMapper.java | 1 - 8 files changed, 102 insertions(+), 68 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java index 06a3e4c5956b7..341eb178d0783 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Record.java @@ -34,7 +34,7 @@ public interface Record { - String RECORD_ERROR_SUPPORT = "talend.sdk.runtime.record.error.support"; + String RECORD_ERROR_SUPPORT = "talend.component.record.error.support"; /** * @return the schema of this record. @@ -454,7 +454,5 @@ default Builder withInstant(Schema.Entry entry, Instant value) { Builder withRecord(String name, Record value); Builder withArray(Schema.Entry entry, Collection values); - - Builder withError(String columnName, Object value, String errorMessage, Exception exception); } } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 797b8b23f06e9..4a7078b7b0d91 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -237,9 +237,9 @@ void testWithError() { Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); // record 1 Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record1 = recordBuilder.withError("name", null, "Stirng is null", null) - .withString("normal", "normal") - .withError("age", "string", "is not an int", null) + Record record1 = recordBuilder.with(nameEntry, null) + .with(nmEntry, "normal") + .with(ageEntry, "is not an int") .build(); assertFalse(record1.isValid()); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index f91d8e0b17e7e..841c43abfb187 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -172,9 +172,13 @@ public Object getValue(final String name) { @Override public Builder with(final Entry entry, final Object value) { - validateTypeAgainstProvidedSchema(entry.getName(), entry.getType(), value); + try { + validateTypeAgainstProvidedSchema(entry.getName(), entry.getType(), value); + } catch (Exception e) { + return withError(entry, value, e.getMessage()); + } if (!entry.getType().isCompatible(value)) { - throw new IllegalArgumentException(String + return withError(entry, value, String .format("Entry '%s' of type %s is not compatible with value of type '%s'", entry.getName(), entry.getType(), value.getClass().getName())); } @@ -514,25 +518,21 @@ public Builder withArray(final Schema.Entry entry, final Collection value return append(entry, values); } - @Override - public Builder withError(final String columnName, final Object value, final String errorMessage, - final Exception exception) { + private Builder withError(final Entry entry, final Object value, final String errorMessage) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); if (!supportError) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry - final Entry oldEntry = this.findExistingEntry(columnName); - final Entry updateEntry = oldEntry.toBuilder() - .withName(columnName) + final Entry updateEntry = entry.toBuilder() + .withName(entry.getName()) .withNullable(true) - .withType(oldEntry.getType()) + .withType(entry.getType()) .withProp(SchemaProperty.ENTRY_IS_ON_ERROR, "true") .withProp(SchemaProperty.ENTRY_ERROR_MESSAGE, errorMessage) .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) - .withProp(SchemaProperty.ERROR_EXCEPTION, exception == null ? "" : exception.toString()) - .build(); - return updateEntryByName(columnName, updateEntry); + .build(); + return updateEntryByName(entry.getName(), updateEntry); } } diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 6c8fea7482602..42a3fb0538aa3 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -251,31 +251,43 @@ void dateTime() { @Test void withError() { + Entry dateNull = new EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Type.DATETIME) + .build(); + Entry date = new EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Type.DATETIME) + .build(); + Entry normal = new EntryImpl.BuilderImpl() + .withName("normal") + .withNullable(true) + .withType(Type.STRING) + .build(); + Entry intEntry = new EntryImpl.BuilderImpl() + .withName("intValue") + .withNullable(false) + .withType(Type.INT) + .build(); final Schema schema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Schema.Type.DATETIME) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("normal") - .withNullable(true) - .withType(Type.STRING) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("intValue") - .withNullable(false) - .withType(Type.INT) - .build()) + .withEntry(dateNull) + .withEntry(date) + .withEntry(normal) + .withEntry(intEntry) .build(); final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); - final Record record3 = builder3.withError("date", null, "date is null", null) - .withString("normal", "normal") - .withError("intValue", "string", "wrong int value", null) + + final Record record3 = builder3 + .with(dateNull, null) + .with(date, "not a date") + .with(normal, "normal") + .with(intEntry, "wrong int value") .build(); assertFalse(record3.isValid()); final Entry entry = @@ -288,9 +300,31 @@ void withError() { assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); + final Entry entry3 = + record3.getSchema().getEntries().stream().filter(e -> "normal".equals(e.getName())).findAny().get(); + assertNotNull(entry3); + Assertions.assertTrue(entry3.isValid()); + System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); } + @Test + void testWithWrongEntryType() { + Entry entry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build(); + final Schema schema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(entry) + .build(); + final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(schema); + assertNotNull(builder.getEntry("date")); + + assertThrows(IllegalArgumentException.class, () -> builder.with(entry, "String")); + } + @Test void zonedDateTimeVSInstant() { final Schema schema = new SchemaImpl.BuilderImpl() diff --git a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java index a7fed49240bdc..8254a0eec55c5 100644 --- a/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java +++ b/component-tools/src/main/java/org/talend/sdk/component/tools/validator/RecordValidator.java @@ -132,7 +132,6 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.Record$Builder.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.Record$Builder.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", - "org.talend.sdk.component.api.record.Record$Builder.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)", "org.talend.sdk.component.api.record.RecordImpl.withNewSchema(org.talend.sdk.component.api.record.Schema)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.with(org.talend.sdk.component.api.record.Schema$Entry, java.lang.Object)", @@ -162,7 +161,6 @@ private static String namesOf(final Parameter[] parameters) { "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withBoolean(org.talend.sdk.component.api.record.Schema$Entry, boolean)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(java.lang.String, org.talend.sdk.component.api.record.Record)", "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withRecord(org.talend.sdk.component.api.record.Schema$Entry, org.talend.sdk.component.api.record.Record)", - "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)", - "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withError(java.lang.String, java.lang.Object, java.lang.String, java.lang.Exception)")); + "org.talend.sdk.component.api.record.RecordImpl.BuilderImpl.withArray(org.talend.sdk.component.api.record.Schema$Entry, java.util.Collection)")); } diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java index c70fce4534add..49aa2f1f4a5ab 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java @@ -21,7 +21,6 @@ import java.io.File; import java.util.Date; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -58,8 +57,6 @@ public static void runInput( @Option("mapper") @Default("SupportErrorMapper") final String mapper) { try (final ComponentManager manager = manager(jar, GAV)) { - final Map configuration = new HashMap<>(); - info("support " + support); if (support) { setSupportError(support); diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java index 13fe1001201cc..c9c3d61577ff2 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,9 +42,14 @@ @Emitter(family = "supporterror", name = "supportErrorInput") public class SupportErrorInput implements Serializable { - private transient Schema recordSchema; + private transient Schema.Entry nameEntry; + + private transient Schema.Entry dateEntry; + + private transient Schema.Entry ageEntry; + private final transient InputConfig configuration; public SupportErrorInput(@Option("configuration") final InputConfig config) { @@ -53,32 +58,35 @@ public SupportErrorInput(@Option("configuration") final InputConfig config) { @PostConstruct public void init() { + nameEntry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("name") + .withNullable(false) + .withType(Schema.Type.STRING) + .build(); + dateEntry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withType(Schema.Type.DATETIME) + .build(); + ageEntry = new SchemaImpl.EntryImpl.BuilderImpl() + .withName("age") + .withNullable(false) + .withType(Schema.Type.INT) + .build(); recordSchema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("name") - .withNullable(false) - .withType(Schema.Type.STRING) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Schema.Type.DATETIME) - .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() - .withName("age") - .withNullable(false) - .withType(Schema.Type.INT) - .build()) + .withEntry(nameEntry) + .withEntry(dateEntry) + .withEntry(ageEntry) .build(); } @Producer public Record data() { final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - return builder.withString("name", "example connector") - .withError("date", null, "date is null", null) - .withError("age", "string", "wrong int value", null) + return builder.with(nameEntry, "example connector") + .with(dateEntry, "not a date value") + .with(ageEntry, "wrong int value") .build(); } @@ -86,7 +94,7 @@ public Record data() { @GridLayout(value = { @GridLayout.Row("dataset"), }) - @Version + @Version public static class InputConfig { @Option @@ -96,7 +104,7 @@ public static class InputConfig { @DataSet @Data - @GridLayout(value = { @GridLayout.Row("datastore") }) + @GridLayout(value = {@GridLayout.Row("datastore")}) public static class Dataset implements Serializable { @Option @@ -107,7 +115,7 @@ public static class Dataset implements Serializable { @DataStore @Data - @GridLayout(value = { @GridLayout.Row("name"), @GridLayout.Row("age"), + @GridLayout(value = {@GridLayout.Row("name"), @GridLayout.Row("age"), @GridLayout.Row("date")}) public static class Datastore implements Serializable { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java index 50f078aec5147..db3d0506c9c98 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java +++ b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java @@ -53,7 +53,6 @@ public List split(@PartitionSize final int desiredNbSplits) @Emitter public SupportErrorInput createSource() { - return new SupportErrorInput(this.config); } From c87b92325022003cd86071a266ed1762bfd0cad7 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 12 May 2025 16:13:36 +0800 Subject: [PATCH 47/62] add getErrorMessage,getErrorFallbackValue in Entry API --- .../java/org/talend/sdk/component/api/record/Schema.java | 6 +++++- .../sdk/component/runtime/record/RecordBuilderImplTest.java | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java index da4b0806638be..7d4c814715ef4 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java @@ -233,7 +233,7 @@ interface Entry { boolean isMetadata(); /** - * @return true if the value of this entry is valid; false for invalid value. + * @return true if the value of this entry is valid; false for invalid value. */ boolean isValid(); @@ -301,6 +301,10 @@ default Entry.Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder is not implemented"); } + default String getErrorMessage() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)).orElse(null); } + + default String getErrorFallbackValue() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)).orElse(null); } + /** * Plain builder matching {@link Entry} structure. */ diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 42a3fb0538aa3..94d2d730cf2f5 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -294,11 +294,15 @@ void withError() { record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); assertNotNull(entry); Assertions.assertFalse(entry.isValid()); + assertEquals("Entry 'date' of type DATETIME is not compatible with value of type 'java.lang.String'", + entry.getErrorMessage()); final Entry entry2 = record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); assertNotNull(entry2); Assertions.assertFalse(entry2.isValid()); + assertEquals("Entry 'intValue' of type INT is not compatible with value of type 'java.lang.String'", + entry2.getErrorMessage()); final Entry entry3 = record3.getSchema().getEntries().stream().filter(e -> "normal".equals(e.getName())).findAny().get(); From 05b7b3fbd9415c8090c4dc721049ffe3e0bd23a2 Mon Sep 17 00:00:00 2001 From: ypiel Date: Tue, 13 May 2025 05:21:44 +0200 Subject: [PATCH 48/62] feat(QTDI-1305): Fix RecordBuilder builder pattern +some improvements. (#1042) * feat(QTDI-1305): Fix RecordBuilder builder pattern +some improvements. * feat(QTDI-1305): rename sample supporterror to entry-with-error. --- .../sdk/component/api/record/Schema.java | 15 +- .../component/api/service/schema/Schema.java | 9 +- .../sdk/component/api/record/RecordTest.java | 9 +- .../sdk/component/api/record/SchemaTest.java | 9 +- .../record/RecordBuilderFactoryTest.java | 12 +- .../component/runtime/record/RecordImpl.java | 52 ++++-- .../component/runtime/record/SchemaImpl.java | 20 +++ .../runtime/record/RecordBuilderImplTest.java | 94 ++++++----- .../README.md | 2 +- .../pom.xml | 49 +++++- .../feature/entrywitherror}/Versions.java | 2 +- .../sample/feature/entrywitherror}/Cli.java | 92 +++++++---- .../RecordWithEntriesInErrorEmitter.java | 150 ++++++++++++++++++ .../feature/entrywitherror}/package-info.java | 6 +- ...nent.runtime.serialization.ContainerFinder | 0 .../src/main/resources/icons/dark/icon.svg | 0 .../src/main/resources/icons/dark/mapper.svg | 0 .../src/main/resources/icons/light/icon.svg | 0 .../src/main/resources/icons/light/mapper.svg | 0 .../entrywitherror}/Messages.properties | 17 +- sample-parent/sample-features/pom.xml | 2 +- .../supporterror/SupportErrorInput.java | 135 ---------------- .../supporterror/SupportErrorMapper.java | 59 ------- 23 files changed, 427 insertions(+), 307 deletions(-) rename sample-parent/sample-features/{supporterror => entry-with-error}/README.md (98%) rename sample-parent/sample-features/{supporterror => entry-with-error}/pom.xml (59%) rename sample-parent/sample-features/{supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror}/Versions.java (91%) rename sample-parent/sample-features/{supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror}/Cli.java (65%) create mode 100644 sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java rename sample-parent/sample-features/{supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror}/package-info.java (79%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/dark/icon.svg (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/dark/mapper.svg (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/light/icon.svg (100%) rename sample-parent/sample-features/{supporterror => entry-with-error}/src/main/resources/icons/light/mapper.svg (100%) rename sample-parent/sample-features/{supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror => entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror}/Messages.properties (58%) delete mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java delete mode 100644 sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java index 7d4c814715ef4..210d903cca1e4 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java @@ -232,6 +232,11 @@ interface Entry { */ boolean isMetadata(); + /** + * @return Is this entry can be in error. + */ + boolean isErrorCapable(); + /** * @return true if the value of this entry is valid; false for invalid value. */ @@ -301,9 +306,13 @@ default Entry.Builder toBuilder() { throw new UnsupportedOperationException("#toBuilder is not implemented"); } - default String getErrorMessage() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)).orElse(null); } + default String getErrorMessage() { + return getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); + } - default String getErrorFallbackValue() { return Optional.ofNullable(getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)).orElse(null); } + default String getErrorFallbackValue() { + return getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE); + } /** * Plain builder matching {@link Entry} structure. @@ -326,6 +335,8 @@ default Builder withLogicalType(String logicalType) { Builder withNullable(boolean nullable); + Builder withErrorCapable(boolean errorCapable); + Builder withMetadata(boolean metadata); Builder withDefaultValue(T value); diff --git a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java index 433673db4b170..5192dc9f5c4c5 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java @@ -129,6 +129,11 @@ public boolean isNullable() { return true; } + @Override + public boolean isErrorCapable() { + return false; + } + @Override public boolean isMetadata() { return false; @@ -165,7 +170,9 @@ public Builder toBuilder() { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java index c07eda9008351..63a9852487f4b 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java @@ -106,6 +106,11 @@ public boolean isNullable() { return false; } + @Override + public boolean isErrorCapable() { + return false; + } + @Override public boolean isMetadata() { return false; @@ -137,7 +142,9 @@ public String getProp(String property) { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } }; Assertions.assertEquals("value", record.get(String.class, e1)); } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java index 11c5e325f3c9d..d1b8e6bb8094b 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java @@ -291,6 +291,11 @@ public boolean isNullable() { throw new UnsupportedOperationException("#isNullable()"); } + @Override + public boolean isErrorCapable() { + throw new UnsupportedOperationException("#isErrorCapable()"); + } + @Override public boolean isMetadata() { throw new UnsupportedOperationException("#isMetadata()"); @@ -327,7 +332,9 @@ public JsonValue getJsonProp(final String name) { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } } diff --git a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java index a34a6cc58d82f..ff2c4fba06b14 100644 --- a/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java +++ b/component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java @@ -78,6 +78,8 @@ static class MockEntry implements Schema.Entry { private final boolean nullable; + private final boolean errorCapable; + private final boolean metadata; private final Object defaultVal; @@ -104,7 +106,9 @@ public Builder toBuilder() { } @Override - public boolean isValid() { return true; } + public boolean isValid() { + return true; + } } @RequiredArgsConstructor @@ -138,6 +142,12 @@ public Entry.Builder withNullable(boolean nullable) { return this; } + @Override + public Entry.Builder withErrorCapable(boolean errorCapable) { + this.builder.withErrorCapable(errorCapable); + return this; + } + @Override public Entry.Builder withMetadata(boolean metadata) { this.builder.withMetadata(metadata); diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 841c43abfb187..baa0ba2571401 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -130,6 +130,8 @@ public static class BuilderImpl implements Builder { private OrderState orderState; + private Map entriesInError = new HashMap<>(); + public BuilderImpl() { this(null); } @@ -151,7 +153,7 @@ private void initOrderState() { } else { final List fields = this.providedSchema.naturalOrder() .getFieldsOrder() - .map(this.providedSchema::getEntry) + .map(n -> this.getEntryWithErrorIfAny(this.providedSchema.getEntry(n))) .collect(Collectors.toList()); this.orderState = new OrderState(fields); } @@ -179,8 +181,9 @@ public Builder with(final Entry entry, final Object value) { } if (!entry.getType().isCompatible(value)) { return withError(entry, value, String - .format("Entry '%s' of type %s is not compatible with value of type '%s'", entry.getName(), - entry.getType(), value.getClass().getName())); + .format("Entry '%s' of type %s is not compatible with given value of type '%s': '%s'.", + entry.getName(), + entry.getType(), value.getClass().getName(), value)); } if (entry.getType() == Schema.Type.DATETIME) { @@ -328,25 +331,53 @@ private Schema.Entry validateTypeAgainstProvidedSchema(final String name, final return entry; } + /** + * This method return the updated entry with error information if any. + * + * @param e The entry to check. + * @return The entry updated with error information or the given one. + */ + private Entry getEntryWithErrorIfAny(final Entry e) { + if (!e.isErrorCapable()) { + // The entry doesn't support error management + return e; + } + + return entriesInError.getOrDefault(e.getOriginalFieldName(), e); + } + public Record build() { final Schema currentSchema; if (this.providedSchema != null) { final String missing = this.providedSchema .getAllEntries() + .map(this::getEntryWithErrorIfAny) .filter(it -> !it.isNullable() && !values.containsKey(it.getName())) .map(Schema.Entry::getName) .collect(joining(", ")); if (!missing.isEmpty()) { throw new IllegalArgumentException("Missing entries: " + missing); } + + Schema schemaWithErrors = this.providedSchema; + if (!this.entriesInError.isEmpty()) { + Schema.Builder schemaBuilder = new SchemaImpl.BuilderImpl() + .withType(this.providedSchema.getType()); + this.providedSchema.getEntries() + .stream() + .map(this::getEntryWithErrorIfAny) + .forEach(schemaBuilder::withEntry); + schemaWithErrors = schemaBuilder.build(); + } + if (orderState != null && orderState.isOverride()) { - currentSchema = this.providedSchema.toBuilder().build(this.orderState.buildComparator()); + currentSchema = schemaWithErrors.toBuilder().build(this.orderState.buildComparator()); } else { - currentSchema = this.providedSchema; + currentSchema = schemaWithErrors; } } else { final Schema.Builder builder = new SchemaImpl.BuilderImpl().withType(RECORD); - this.entries.forEachValue(builder::withEntry); + this.entries.streams().map(this::getEntryWithErrorIfAny).forEach(builder::withEntry); initOrderState(); currentSchema = builder.build(orderState.buildComparator()); } @@ -520,19 +551,20 @@ public Builder withArray(final Schema.Entry entry, final Collection value private Builder withError(final Entry entry, final Object value, final String errorMessage) { final boolean supportError = Boolean.parseBoolean(System.getProperty(RECORD_ERROR_SUPPORT, "false")); - if (!supportError) { + if (!supportError || !entry.isErrorCapable()) { throw new IllegalArgumentException(errorMessage); } else { // duplicate the schema instance with a modified Entry - final Entry updateEntry = entry.toBuilder() + final Entry updatedEntry = entry.toBuilder() .withName(entry.getName()) .withNullable(true) .withType(entry.getType()) .withProp(SchemaProperty.ENTRY_IS_ON_ERROR, "true") .withProp(SchemaProperty.ENTRY_ERROR_MESSAGE, errorMessage) .withProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE, String.valueOf(value)) - .build(); - return updateEntryByName(entry.getName(), updateEntry); + .build(); + this.entriesInError.put(updatedEntry.getOriginalFieldName(), updatedEntry); + return this; } } diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java index 6348c4d0f69f0..41886e2ee38b2 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/SchemaImpl.java @@ -352,6 +352,7 @@ private EntryImpl(final EntryImpl.BuilderImpl builder) { this.type = builder.type; } this.nullable = builder.nullable; + this.errorCapable = builder.errorCapable; this.metadata = builder.metadata; this.defaultValue = builder.defaultValue; this.elementSchema = builder.elementSchema; @@ -379,6 +380,11 @@ private EntryImpl(final EntryImpl.BuilderImpl builder) { */ private final boolean nullable; + /** + * Is this entry can be in error. + */ + private final boolean errorCapable; + /** * Is this entry a metadata entry. */ @@ -440,6 +446,11 @@ public boolean isNullable() { return this.nullable; } + @Override + public boolean isErrorCapable() { + return this.errorCapable; + } + @Override public boolean isMetadata() { return this.metadata; @@ -487,6 +498,8 @@ public static class BuilderImpl implements Entry.Builder { private boolean nullable; + private boolean errorCapable; + private boolean metadata = false; private Object defaultValue; @@ -506,6 +519,7 @@ private BuilderImpl(final Entry entry) { this.name = entry.getName(); this.rawName = entry.getRawName(); this.nullable = entry.isNullable(); + this.errorCapable = entry.isErrorCapable(); this.type = entry.getType(); this.comment = entry.getComment(); this.elementSchema = entry.getElementSchema(); @@ -549,6 +563,12 @@ public Builder withNullable(final boolean nullable) { return this; } + @Override + public Builder withErrorCapable(final boolean errorCapable) { + this.errorCapable = errorCapable; + return this; + } + @Override public Builder withMetadata(final boolean metadata) { this.metadata = metadata; diff --git a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java index 94d2d730cf2f5..85e233722a835 100644 --- a/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java +++ b/component-runtime-impl/src/test/java/org/talend/sdk/component/runtime/record/RecordBuilderImplTest.java @@ -249,67 +249,79 @@ void dateTime() { assertThrows(IllegalArgumentException.class, () -> builder2.withDateTime("date", (ZonedDateTime) null)); } + @Test + void withErrorWhenNotSupported() { + Assertions.assertThrows(IllegalArgumentException.class, () -> withError("false")); + } + @Test void withError() { - Entry dateNull = new EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Type.DATETIME) - .build(); - Entry date = new EntryImpl.BuilderImpl() + Record record = withError("true"); + + assertFalse(record.isValid()); + + final Entry retrievedDateEntry = record.getSchema().getEntry("date"); + assertNotNull(retrievedDateEntry); + Assertions.assertFalse(retrievedDateEntry.isValid()); + assertEquals( + "Entry 'date' of type DATETIME is not compatible with given value of type 'java.lang.String': 'not a date'.", + retrievedDateEntry.getErrorMessage()); + Assertions.assertNull(record.getDateTime("date")); + + final Entry retrievedIntEntry = record.getSchema().getEntry("intValue"); + assertNotNull(retrievedIntEntry); + Assertions.assertFalse(retrievedIntEntry.isValid()); + assertEquals( + "Entry 'intValue' of type INT is not compatible with given value of type 'java.lang.String': 'wrong int value'.", + retrievedIntEntry.getErrorMessage()); + Assertions.assertNull(record.getDateTime("intValue")); + + final Entry retrievedStringEntry = record.getSchema().getEntry("normal"); + assertNotNull(retrievedStringEntry); + Assertions.assertTrue(retrievedStringEntry.isValid()); + Assertions.assertEquals("No error", record.getString("normal")); + + } + + private Record withError(final String supported) { + final String errorSupportBackup = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, supported); + + Entry dateEntry = new EntryImpl.BuilderImpl() .withName("date") .withNullable(false) + .withErrorCapable(true) .withType(Type.DATETIME) .build(); - Entry normal = new EntryImpl.BuilderImpl() + Entry stringEntry = new EntryImpl.BuilderImpl() .withName("normal") .withNullable(true) + .withErrorCapable(true) .withType(Type.STRING) .build(); Entry intEntry = new EntryImpl.BuilderImpl() .withName("intValue") .withNullable(false) + .withErrorCapable(true) .withType(Type.INT) .build(); final Schema schema = new SchemaImpl.BuilderImpl() .withType(Schema.Type.RECORD) - .withEntry(dateNull) - .withEntry(date) - .withEntry(normal) + .withEntry(dateEntry) + .withEntry(stringEntry) .withEntry(intEntry) .build(); - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); - final RecordImpl.BuilderImpl builder3 = new RecordImpl.BuilderImpl(schema); - final Record record3 = builder3 - .with(dateNull, null) - .with(date, "not a date") - .with(normal, "normal") - .with(intEntry, "wrong int value") - .build(); - assertFalse(record3.isValid()); - final Entry entry = - record3.getSchema().getEntries().stream().filter(e -> "date".equals(e.getName())).findAny().get(); - assertNotNull(entry); - Assertions.assertFalse(entry.isValid()); - assertEquals("Entry 'date' of type DATETIME is not compatible with value of type 'java.lang.String'", - entry.getErrorMessage()); - - final Entry entry2 = - record3.getSchema().getEntries().stream().filter(e -> "intValue".equals(e.getName())).findAny().get(); - assertNotNull(entry2); - Assertions.assertFalse(entry2.isValid()); - assertEquals("Entry 'intValue' of type INT is not compatible with value of type 'java.lang.String'", - entry2.getErrorMessage()); - - final Entry entry3 = - record3.getSchema().getEntries().stream().filter(e -> "normal".equals(e.getName())).findAny().get(); - assertNotNull(entry3); - Assertions.assertTrue(entry3.isValid()); - - System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); + final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(schema); + + builder.with(stringEntry, "No error"); + builder.with(dateEntry, "not a date"); + builder.with(intEntry, "wrong int value"); + final Record record = builder.build(); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, errorSupportBackup == null ? "false" : errorSupportBackup); + + return record; } @Test diff --git a/sample-parent/sample-features/supporterror/README.md b/sample-parent/sample-features/entry-with-error/README.md similarity index 98% rename from sample-parent/sample-features/supporterror/README.md rename to sample-parent/sample-features/entry-with-error/README.md index 6b5756f4d20ff..a37bef27b3eeb 100644 --- a/sample-parent/sample-features/supporterror/README.md +++ b/sample-parent/sample-features/entry-with-error/README.md @@ -1,4 +1,4 @@ -# Component Runtime :: Sample Feature :: SupportError +# Component Runtime :: Sample Feature :: Entry with error ## Table of Contents - [Overview](#overview) diff --git a/sample-parent/sample-features/supporterror/pom.xml b/sample-parent/sample-features/entry-with-error/pom.xml similarity index 59% rename from sample-parent/sample-features/supporterror/pom.xml rename to sample-parent/sample-features/entry-with-error/pom.xml index b5a57cfc4f32e..b136e3b977d73 100644 --- a/sample-parent/sample-features/supporterror/pom.xml +++ b/sample-parent/sample-features/entry-with-error/pom.xml @@ -26,7 +26,7 @@ supporterror jar - Component Runtime :: Sample Feature :: Supporterror + Component Runtime :: Sample Feature :: Entry with error support @@ -56,6 +56,47 @@ 1.8 + + org.talend.sdk.component + talend-component-maven-plugin + ${project.version} + + + talend-component-validate + + validate + + process-classes + + true + true + false + true + true + true + true + true + true + false + + false + + true + true + true + true + true + true + true + true + true + true + true + true + + + + org.apache.maven.plugins maven-jar-plugin @@ -63,7 +104,7 @@ true - org.talend.sdk.component.sample.feature.supporterror.Cli + org.talend.sdk.component.sample.feature.entrywitherror.Cli @@ -80,7 +121,7 @@ - org.talend.sdk.component.sample.feature.supporterror.Cli + org.talend.sdk.component.sample.feature.entrywitherror.Cli @@ -93,7 +134,7 @@ exec-maven-plugin 3.0.0 - org.talend.sdk.component.sample.feature.supporterror.Cli + org.talend.sdk.component.sample.feature.entrywitherror.Cli diff --git a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java b/sample-parent/sample-features/entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror/Versions.java similarity index 91% rename from sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java rename to sample-parent/sample-features/entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror/Versions.java index 242683a0ce5e8..fef316911253f 100644 --- a/sample-parent/sample-features/supporterror/src/main/java-templates/org/talend/sdk/component/sample/feature/supporterror/Versions.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java-templates/org/talend/sdk/component/sample/feature/entrywitherror/Versions.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.talend.sdk.component.sample.feature.supporterror; +package org.talend.sdk.component.sample.feature.entrywitherror; public interface Versions { diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java similarity index 65% rename from sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java rename to sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 49aa2f1f4a5ab..9570ec019fc37 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -13,24 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.talend.sdk.component.sample.feature.supporterror; +package org.talend.sdk.component.sample.feature.entrywitherror; import static java.util.Optional.ofNullable; import static lombok.AccessLevel.PRIVATE; import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; import java.io.File; -import java.util.Date; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.talend.sdk.component.api.record.Record; -import org.talend.sdk.component.api.record.Schema; -import org.talend.sdk.component.api.record.SchemaProperty; +import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; +import org.talend.sdk.component.runtime.input.InputImpl; +import org.talend.sdk.component.runtime.input.Mapper; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl; import org.talend.sdk.component.runtime.serialization.ContainerFinder; @@ -51,49 +52,74 @@ public final class Cli { @Command("supporterror") public static void runInput( @Option("gav") @Default(GAV) final String gav, - @Option("s") @Default("false") final boolean support, + @Option("support") @Default("false") final boolean support, + @Option("gen-some-errors") @Default("true") final boolean genErrors, + @Option("gen-nbrecords") @Default("10") final int nbRecords, @Option("jar") final File jar, @Option("family") @Default("supporterror") final String family, @Option("mapper") @Default("SupportErrorMapper") final String mapper) { - try (final ComponentManager manager = manager(jar, GAV)) { - info("support " + support); - if (support) { - setSupportError(support); - } - info("create input now."); - - SupportErrorInput seInput = new SupportErrorInput(null); - seInput.init(); + info("support " + support); + if (support) { + setSupportError(support); + } - info("getting the record."); - Record data = seInput.data(); + Map config = new HashMap<>(); + config.put("configuration.generateErrors", String.valueOf(genErrors)); + config.put("configuration.nbRecords", String.valueOf(nbRecords)); + run(jar, gav, config, "sampleRecordWithEntriesInError", "RecordWithEntriesInErrorEmitter"); + } - info("Record isValid = " + data.isValid()); - entryOut(data, "name", String.class); - entryOut(data, "date", Date.class); - entryOut(data, "age", Integer.class); - // + private static void run(final File jar, final String gav, final Map configuration, + final String family, final String mapper) { + try (final ComponentManager manager = manager(jar, gav)) { + info("configuration: " + configuration); + + // create the mapper + final Mapper mpr = manager.findMapper(family, mapper, 1, configuration) + .orElseThrow(() -> new IllegalStateException( + String.format("No mapper found for: %s/%s.", family, manager))); + + List mappers = mpr.split(1); + Record data; + + int count = 0; + for (Mapper currentMapper : mappers) { + final InputImpl input = InputImpl.class.cast(currentMapper.create()); + input.start(); + while ((data = (Record) input.next()) != null) { + count++; + recordOut(count, data); + } + input.stop(); + } info("finished."); } catch (Exception e) { error(e); } } - private static void entryOut(final Record data, final String column, final Class type) { - Optional ageEntry = data.getSchema().getEntries().stream().filter(e -> column.equals(e.getName())).findAny(); - if(ageEntry.isPresent()) { - if (ageEntry.get().isValid()) { - Object value = data.get(type, column); - info("Record '" + column + "': " + value); - } else { - String errorMessage = ageEntry.get().getProp(SchemaProperty.ENTRY_ERROR_MESSAGE); - info("ERROR: " + errorMessage); - } + private static void recordOut(final int count, final Record record) { + System.out.printf("Record no %s is valid ? %s\n", count, record.isValid() ? "yes" : "no"); + System.out.printf("\tName: %s\n", record.getString("name")); + Entry date = record.getSchema().getEntry("date"); + if (date.isValid()) { + System.out.printf("\tDate: %s\n", record.getDateTime("date")); + } else { + System.out.printf("\tDate is on error: \n\t\tMessage:%s\n\t\tFallback value: %s\n", + date.getErrorMessage(), date.getErrorFallbackValue()); + } + + Entry age = record.getSchema().getEntry("age"); + if (age.isValid()) { + System.out.printf("\tAge: %s\n", record.getInt("age")); + } else { + System.out.printf("\tAge is on error: \n\t\tMessage:%s\n\t\tFallback value: %s\n", + age.getErrorMessage(), age.getErrorFallbackValue()); } } - //set support or not. + // set support or not. public static void setSupportError(final boolean supportError) { System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportError)); } diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java new file mode 100644 index 0000000000000..576732e75298f --- /dev/null +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2006-2025 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.sample.feature.entrywitherror; + +import java.io.Serializable; +import java.time.Month; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.function.Function; + +import javax.annotation.PostConstruct; + +import org.talend.sdk.component.api.component.Icon; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.Producer; +import org.talend.sdk.component.api.meta.Documentation; +import org.talend.sdk.component.api.record.Record; +import org.talend.sdk.component.api.record.Record.Builder; +import org.talend.sdk.component.api.record.Schema; +import org.talend.sdk.component.api.record.Schema.Entry; +import org.talend.sdk.component.api.service.record.RecordBuilderFactory; +import org.talend.sdk.component.runtime.record.SchemaImpl; + +import lombok.Data; + +@Version +@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") +@Emitter(name = "RecordWithEntriesInErrorEmitter") +@Documentation("Generated record with entries in error.") +public class RecordWithEntriesInErrorEmitter implements Serializable { + + private final RecordBuilderFactory recordBuilderFactory; + + private final Config config; + + private transient Schema recordSchema; + + private transient Function createRecordFunction; + + private transient int index; + + public RecordWithEntriesInErrorEmitter( + final RecordBuilderFactory recordBuilderFactory, + final @Option("configuration") Config config) { + this.recordBuilderFactory = recordBuilderFactory; + this.config = config; + } + + @PostConstruct + public void init() { + recordSchema = new SchemaImpl.BuilderImpl() + .withType(Schema.Type.RECORD) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("name") + .withNullable(false) + .withErrorCapable(true) + .withType(Schema.Type.STRING) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("date") + .withNullable(false) + .withErrorCapable(true) + .withType(Schema.Type.DATETIME) + .build()) + .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withName("age") + .withNullable(false) + .withErrorCapable(true) + .withType(Schema.Type.INT) + .build()) + .build(); + + createRecordFunction = i -> { + Builder builder = recordBuilderFactory.newRecordBuilder(recordSchema).withString("name", "name " + i); + + // Generate error only on odd generated records + boolean generateErrors = config.isGenerateErrors() && i % 2 == 0; + + if (generateErrors) { + Entry dateEntry = recordSchema.getEntry("date"); + builder.with(dateEntry, "789-555"); + } else { + ZonedDateTime dateTime = ZonedDateTime.of( + 2025, // Year + Month.APRIL.getValue(), // Month + 1 + i, // Day + 15, // Hours + 30, // Minutes + 0, // seconds + 0, // nanoseconds + ZoneId.of("UTC") // Timezone + ); + builder.withDateTime("date", dateTime); + } + + if (generateErrors) { + Entry ageEntry = recordSchema.getEntry("age"); + builder.with(ageEntry, "-78"); + } else { + builder.withInt("age", 50 + i); + } + + return builder.build(); + }; + } + + @Producer + public Record data() { + index++; + if (index <= config.getNbRecords()) { + return createRecordFunction.apply(index); + } + + return null; + } + + @Data + @GridLayout(value = { + @GridLayout.Row("generateErrors"), + @GridLayout.Row("nbRecords"), + }) + public static class Config implements Serializable { + + @Option + @Documentation("If true, generate some errors.") + private boolean generateErrors = true; + + @Option + @Documentation("Number of generated records.") + private int nbRecords = 5; + + } + +} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/package-info.java similarity index 79% rename from sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java rename to sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/package-info.java index ff1441932bd2a..0668f6bcaa05c 100644 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/package-info.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/package-info.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Components(family = "supporterror", categories = "sample") +@Components(family = "sampleRecordWithEntriesInError", categories = "sample") @Icon(value = Icon.IconType.CUSTOM, custom = "icon") -package org.talend.sdk.component.sample.feature.supporterror; +package org.talend.sdk.component.sample.feature.entrywitherror; import org.talend.sdk.component.api.component.Components; -import org.talend.sdk.component.api.component.Icon; \ No newline at end of file +import org.talend.sdk.component.api.component.Icon; diff --git a/sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder b/sample-parent/sample-features/entry-with-error/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder rename to sample-parent/sample-features/entry-with-error/src/main/resources/META-INF/services/org.talend.sdk.component.runtime.serialization.ContainerFinder diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/dark/icon.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/dark/mapper.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/light/icon.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg similarity index 100% rename from sample-parent/sample-features/supporterror/src/main/resources/icons/light/mapper.svg rename to sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg diff --git a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties similarity index 58% rename from sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties rename to sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties index 0e589d31e4429..87398b474113c 100644 --- a/sample-parent/sample-features/supporterror/src/main/resources/org/talend/sdk/component/sample/feature/supporterror/Messages.properties +++ b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties @@ -13,16 +13,7 @@ # limitations under the License. # Here you can change all your configuration display names to use more explicit labels # You can also translate your configuration by adding one file by local Messages_fr.properties for french for example -supporterror.supportErrorInput._displayName=Support Error Input -supporterror.supportErrorAvroInput._displayName=Support Error Avro Input -supporterror.SupportErrorMapper._displayName=Support Error Mapper -supporterror.datastore.default._displayName=default -supporterror.dataset.default._displayName=dataset -Dataset.datastore._displayName=Datastore -InputConfig.dataset._displayName=Dataset -Datastore.age._displayName=age -Datastore.date._displayName=date -Datastore.name._displayName=name -Datastore.age._placeholder= -Datastore.date._placeholder= -Datastore.name._placeholder= +sampleRecordWithEntriesInError.RecordWithEntriesInErrorEmitter._displayName = Record with entries in error emitter +Config.generateErrors._displayName = Generate some errors +Config.nbRecords._displayName = Nb generated records +Config.nbRecords._placeholder = diff --git a/sample-parent/sample-features/pom.xml b/sample-parent/sample-features/pom.xml index 1b1046b48b1b5..4b0d77c93bc41 100644 --- a/sample-parent/sample-features/pom.xml +++ b/sample-parent/sample-features/pom.xml @@ -32,7 +32,7 @@ aftergroup-lastgroup conditional-outputs checkpoint-runner - supporterror + entry-with-error diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java deleted file mode 100644 index c9c3d61577ff2..0000000000000 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorInput.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.talend.sdk.component.sample.feature.supporterror; - -import java.io.Serializable; -import java.util.Date; - -import javax.annotation.PostConstruct; - -import org.talend.sdk.component.api.component.Icon; -import org.talend.sdk.component.api.component.Version; -import org.talend.sdk.component.api.configuration.Option; -import org.talend.sdk.component.api.configuration.type.DataSet; -import org.talend.sdk.component.api.configuration.type.DataStore; -import org.talend.sdk.component.api.configuration.ui.layout.GridLayout; -import org.talend.sdk.component.api.input.Emitter; -import org.talend.sdk.component.api.input.Producer; -import org.talend.sdk.component.api.meta.Documentation; -import org.talend.sdk.component.api.record.Record; -import org.talend.sdk.component.api.record.Schema; -import org.talend.sdk.component.runtime.record.RecordImpl; -import org.talend.sdk.component.runtime.record.SchemaImpl; - -import lombok.Data; - -@Version -@Icon(value = Icon.IconType.CUSTOM, custom = "icon") -@Documentation("Support Error Input sample processor connector.") -@Emitter(family = "supporterror", name = "supportErrorInput") -public class SupportErrorInput implements Serializable { - - private transient Schema recordSchema; - - private transient Schema.Entry nameEntry; - - private transient Schema.Entry dateEntry; - - private transient Schema.Entry ageEntry; - - private final transient InputConfig configuration; - - public SupportErrorInput(@Option("configuration") final InputConfig config) { - this.configuration = config; - } - - @PostConstruct - public void init() { - nameEntry = new SchemaImpl.EntryImpl.BuilderImpl() - .withName("name") - .withNullable(false) - .withType(Schema.Type.STRING) - .build(); - dateEntry = new SchemaImpl.EntryImpl.BuilderImpl() - .withName("date") - .withNullable(false) - .withType(Schema.Type.DATETIME) - .build(); - ageEntry = new SchemaImpl.EntryImpl.BuilderImpl() - .withName("age") - .withNullable(false) - .withType(Schema.Type.INT) - .build(); - recordSchema = new SchemaImpl.BuilderImpl() - .withType(Schema.Type.RECORD) - .withEntry(nameEntry) - .withEntry(dateEntry) - .withEntry(ageEntry) - .build(); - } - - @Producer - public Record data() { - final RecordImpl.BuilderImpl builder = new RecordImpl.BuilderImpl(recordSchema); - return builder.with(nameEntry, "example connector") - .with(dateEntry, "not a date value") - .with(ageEntry, "wrong int value") - .build(); - } - - @Data - @GridLayout(value = { - @GridLayout.Row("dataset"), - }) - @Version - public static class InputConfig { - - @Option - @Documentation("Dataset.") - private Dataset dataset = new Dataset(); - } - - @DataSet - @Data - @GridLayout(value = {@GridLayout.Row("datastore")}) - public static class Dataset implements Serializable { - - @Option - @Documentation("Datastore.") - private Datastore datastore = new Datastore(); - - } - - @DataStore - @Data - @GridLayout(value = {@GridLayout.Row("name"), @GridLayout.Row("age"), - @GridLayout.Row("date")}) - public static class Datastore implements Serializable { - - @Option - @Documentation("Name prop.") - private String name = "test"; - - @Option - @Documentation("Age prop.") - private int age = 0; - - @Option - @Documentation("Date prop.") - private Date date; - } - -} diff --git a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java b/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java deleted file mode 100644 index db3d0506c9c98..0000000000000 --- a/sample-parent/sample-features/supporterror/src/main/java/org/talend/sdk/component/sample/feature/supporterror/SupportErrorMapper.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.talend.sdk.component.sample.feature.supporterror; - -import java.io.Serializable; -import java.util.Collections; -import java.util.List; - -import org.talend.sdk.component.api.component.Icon; -import org.talend.sdk.component.api.component.Version; -import org.talend.sdk.component.api.configuration.Option; -import org.talend.sdk.component.api.input.Assessor; -import org.talend.sdk.component.api.input.Emitter; -import org.talend.sdk.component.api.input.PartitionMapper; -import org.talend.sdk.component.api.input.PartitionSize; -import org.talend.sdk.component.api.input.Split; -import org.talend.sdk.component.api.meta.Documentation; - -@Version(1) -@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") -@PartitionMapper(name = "SupportErrorMapper", infinite = false) -@Documentation("Doc: default SupportErrorMapper documentation without Internationalization.") -public class SupportErrorMapper implements Serializable { - - private transient SupportErrorInput.InputConfig config; - - public SupportErrorMapper(final @Option("configin") SupportErrorInput.InputConfig config) { - this.config = config; - } - - @Assessor - public long estimateSize() { - return 1500L; - } - - @Split - public List split(@PartitionSize final int desiredNbSplits) { - return Collections.singletonList(this); - } - - @Emitter - public SupportErrorInput createSource() { - return new SupportErrorInput(this.config); - } - -} From 684f1eec9ee26a8e549fc0116ecfbd3b0242e4e0 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Tue, 13 May 2025 15:19:23 +0200 Subject: [PATCH 49/62] feat(QTDI-1305): set RecordImpl#BuilderImpl#entriesInError final. --- .../org/talend/sdk/component/runtime/record/RecordImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index baa0ba2571401..1bc5d45484878 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -130,7 +130,7 @@ public static class BuilderImpl implements Builder { private OrderState orderState; - private Map entriesInError = new HashMap<>(); + private final Map entriesInError = new HashMap<>(); public BuilderImpl() { this(null); From a39f2be35a22919f8cc9d1b1cf46b22de05384d9 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Tue, 13 May 2025 18:05:05 +0200 Subject: [PATCH 50/62] feat(QTDI-1305): Fix entry with error in AvroRecord + unit test. --- .../org/talend/sdk/component/api/record/SchemaProperty.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java index 95bf16be2d181..25954bba2adb9 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java @@ -43,8 +43,6 @@ public interface SchemaProperty { String ENTRY_ERROR_FALLBACK_VALUE = "entry.error.fallback.value"; - String ERROR_EXCEPTION = "entry.error.exception"; - enum LogicalType { DATE("date"), From 41fb00b8cc9a1d2bab955d692fe967c61fe7f1b3 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 14 May 2025 16:08:15 +0800 Subject: [PATCH 51/62] add transfer "errorcapable" between Avro.Field and Schema.Entry. --- .../sdk/component/runtime/beam/spi/record/AvroSchema.java | 1 + .../runtime/beam/spi/record/AvroSchemaBuilder.java | 4 +++- .../runtime/beam/spi/record/KeysForAvroProperty.java | 2 ++ .../runtime/beam/spi/record/AvroRecordBuilderTest.java | 6 ++++-- .../beam/spi/record/JsonSchemaSerializationTest.java | 4 ++-- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java index 0a976b980ab81..e6f08b6b55eb2 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java @@ -222,6 +222,7 @@ private static Entry buildFromAvro(final Field field, final Type type, .withRawName(field.getProp(KeysForAvroProperty.LABEL)) // .withType(type) // .withNullable(field.schema().getType() == UNION) // + .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.ERROR_CAPABLE))) .withMetadata(AvroSchema.isMetadata(field)) // .withDefaultValue(field.defaultVal()) // .withElementSchema(elementSchema) // diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java index 948501a242d33..00a72e93686d3 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java @@ -463,7 +463,9 @@ public static Field toField(final org.apache.avro.Schema schema, final Schema.En for (Map.Entry e : entry.getProps().entrySet()) { field.addProp(e.getKey(), e.getValue()); } - + if (entry.isErrorCapable()) { + field.addProp(KeysForAvroProperty.ERROR_CAPABLE, String.valueOf(entry.isErrorCapable())); + } return field; } } diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java index a5d5b541cbaae..69860b99b9406 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java @@ -21,4 +21,6 @@ public interface KeysForAvroProperty { // alias that indicate field is metadata if present. String METADATA_ALIAS_NAME = "talend.field.__METADATA__"; + + String ERROR_CAPABLE = "entry.errorCapable"; } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 4a7078b7b0d91..6cb91a48e12f2 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -213,13 +213,13 @@ void mixedRecordTest() { void testWithError() { final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); - final String val2 = System.getProperty(Record.RECORD_ERROR_SUPPORT); org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD); Schema.Entry nameEntry = factory .newEntryBuilder() .withName("name") .withNullable(false) + .withErrorCapable(true) .withType(Schema.Type.STRING) .build(); Schema.Entry nmEntry = factory @@ -232,13 +232,15 @@ void testWithError() { .newEntryBuilder() .withName("age") .withNullable(false) + .withErrorCapable(true) .withType(Schema.Type.INT) .build(); Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); // record 1 Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record1 = recordBuilder.with(nameEntry, null) + Record record1 = recordBuilder .with(nmEntry, "normal") + .with(nameEntry, null) .with(ageEntry, "is not an int") .build(); assertFalse(record1.isValid()); diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java index 1993f10373c98..607083f593ef2 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java @@ -49,7 +49,7 @@ void toJson() throws Exception { .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { final String json = jsonb.toJson(schema); assertEquals( - "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", json); } } @@ -71,7 +71,7 @@ void toJsonWithMeta() throws Exception { try (final Jsonb jsonb = JsonbBuilder .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { assertEquals( - "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", jsonb.toJson(schema)); } From 8ca768183e175a3948b2720ec8420c4ba6e58090 Mon Sep 17 00:00:00 2001 From: ypiel Date: Thu, 15 May 2025 10:09:08 +0200 Subject: [PATCH 52/62] feat(QTDI-1305): Fix some unit test + improve sample CLI * feat(QTDI-1305): Complete unit test for entry with error in beam impl.. * feat(QTDI-1305): Fix beam impl unit test. * feat(QTDI-1305): Fix beam impl unit test bis. * feat(QTDI-1305): Fix unit test after adding Entry#isErrorCapable attribute. * feat(QTDI-1305): Imprive sample feature CLI to let user define the number of generated errors.. * feat(QTDI-1305): the CLI support --use-avro-impl option. * feat(QTDI-1305): the CLI support --fields-in-error to be able to decide which fields are in error. --- .../runtime/beam/spi/record/AvroSchema.java | 7 +- .../beam/spi/record/AvroSchemaBuilder.java | 2 +- .../beam/spi/record/KeysForAvroProperty.java | 2 +- .../spi/record/AvroRecordBuilderTest.java | 82 +++++++++++----- .../record/JsonSchemaSerializationTest.java | 4 +- .../server/front/ActionResourceImplTest.java | 5 +- .../beam/BeamActionSerializationTest.java | 5 +- .../sample-features/entry-with-error/pom.xml | 42 ++++----- .../sample/feature/entrywitherror/Cli.java | 93 ++++++++++++------- .../RecordWithEntriesInErrorEmitter.java | 39 +++++--- .../entrywitherror/Messages.properties | 6 +- 11 files changed, 182 insertions(+), 105 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java index e6f08b6b55eb2..a6dc888ba0128 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java @@ -216,13 +216,14 @@ private Entry fromAvro(final Field field) { } private static Entry buildFromAvro(final Field field, final Type type, - final Optional logicalType, final AvroSchema elementSchema) { + final Optional logicalType, + final AvroSchema elementSchema) { Entry.Builder builder = new EntryImpl.BuilderImpl() // .withName(field.name()) // .withRawName(field.getProp(KeysForAvroProperty.LABEL)) // .withType(type) // .withNullable(field.schema().getType() == UNION) // - .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.ERROR_CAPABLE))) + .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.IS_ERROR_CAPABLE))) .withMetadata(AvroSchema.isMetadata(field)) // .withDefaultValue(field.defaultVal()) // .withElementSchema(elementSchema) // @@ -230,6 +231,8 @@ private static Entry buildFromAvro(final Field field, final Type type, .withProps(field.getObjectProps() .entrySet() .stream() + // KeysForAvroProperty.IS_ERROR_CAPABLE is already managed above + .filter(p -> !p.getKey().equals(KeysForAvroProperty.IS_ERROR_CAPABLE)) .collect(toMap(Map.Entry::getKey, e -> String.valueOf(e.getValue())))); logicalType.ifPresent(builder::withLogicalType); diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java index 00a72e93686d3..e234a3a031ba7 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java @@ -464,7 +464,7 @@ public static Field toField(final org.apache.avro.Schema schema, final Schema.En field.addProp(e.getKey(), e.getValue()); } if (entry.isErrorCapable()) { - field.addProp(KeysForAvroProperty.ERROR_CAPABLE, String.valueOf(entry.isErrorCapable())); + field.addProp(KeysForAvroProperty.IS_ERROR_CAPABLE, String.valueOf(entry.isErrorCapable())); } return field; } diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java index 69860b99b9406..937fbeeea3734 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java @@ -22,5 +22,5 @@ public interface KeysForAvroProperty { // alias that indicate field is metadata if present. String METADATA_ALIAS_NAME = "talend.field.__METADATA__"; - String ERROR_CAPABLE = "entry.errorCapable"; + String IS_ERROR_CAPABLE = "entry.errorCapable"; } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 6cb91a48e12f2..265c713ba2d8e 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -41,6 +41,7 @@ import javax.json.JsonArray; import javax.json.JsonObject; +import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.IndexedRecord; import org.apache.avro.io.EncoderFactory; @@ -51,6 +52,7 @@ import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.EntriesOrder; +import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; @@ -209,53 +211,91 @@ void mixedRecordTest() { Assertions.assertNotNull(arrayType); } + @Test + void testWithoutErrorSupport() { + Assertions.assertThrows(IllegalArgumentException.class, () -> testWithError("false")); + } + @Test void testWithError() { - final String val = System.getProperty(Record.RECORD_ERROR_SUPPORT); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "true"); + Record record = testWithError("true"); + assertFalse(record.isValid()); + + final Schema.Entry entry = record.getSchema().getEntry("name"); + assertNotNull(entry); + Assertions.assertFalse(entry.isValid()); + Assertions.assertNull(record.getString("name")); + + final Schema.Entry entry2 = record.getSchema().getEntry("age"); + assertNotNull(entry2); + Assertions.assertFalse(entry2.isValid()); + Assertions.assertNull(record.get(Integer.class, "age")); + + IndexedRecord unwrap = ((AvroRecord) record).unwrap(IndexedRecord.class); + + Field nameField = unwrap.getSchema().getFields().get(0); + Assertions.assertEquals("true", + nameField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + Assertions.assertEquals("Entry 'name' is not nullable", + nameField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); + Assertions.assertEquals("null", + nameField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); + + Field ageField = unwrap.getSchema().getFields().get(2); + Assertions.assertEquals("true", + ageField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + Assertions.assertEquals("Entry 'age' of type INT is not compatible with given value of type " + + "'java.lang.String': 'is not an int'.", + ageField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); + Assertions.assertEquals("is not an int", + ageField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); + + Field noErrorField = unwrap.getSchema().getFields().get(1); + Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); + Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); + + } + + private Record testWithError(final String supported) { + final String errorSupportBackup = System.getProperty(Record.RECORD_ERROR_SUPPORT); + System.setProperty(Record.RECORD_ERROR_SUPPORT, supported); org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD); Schema.Entry nameEntry = factory .newEntryBuilder() .withName("name") - .withNullable(false) .withErrorCapable(true) + .withNullable(false) .withType(Schema.Type.STRING) .build(); - Schema.Entry nmEntry = factory + Schema.Entry noErrorEntry = factory .newEntryBuilder() .withName("normal") + .withErrorCapable(true) .withNullable(true) .withType(Schema.Type.STRING) .build(); Schema.Entry ageEntry = factory .newEntryBuilder() .withName("age") - .withNullable(false) .withErrorCapable(true) + .withNullable(false) .withType(Schema.Type.INT) .build(); - Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(nmEntry).withEntry(ageEntry).build(); - // record 1 + Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(noErrorEntry).withEntry(ageEntry).build(); + + Entry age = customerSchema.getEntry("age"); + Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record1 = recordBuilder - .with(nmEntry, "normal") - .with(nameEntry, null) + Record record = recordBuilder.with(nameEntry, null) + .with(noErrorEntry, "normal") .with(ageEntry, "is not an int") .build(); - assertFalse(record1.isValid()); - - final Schema.Entry entry = - record1.getSchema().getEntries().stream().filter(e -> "name".equals(e.getName())).findAny().get(); - assertNotNull(entry); - Assertions.assertFalse(entry.isValid()); - final Schema.Entry entry2 = - record1.getSchema().getEntries().stream().filter(e -> "age".equals(e.getName())).findAny().get(); - assertNotNull(entry2); - Assertions.assertFalse(entry2.isValid()); + System.setProperty(Record.RECORD_ERROR_SUPPORT, errorSupportBackup == null ? "false" : errorSupportBackup); - System.setProperty(Record.RECORD_ERROR_SUPPORT, "false"); + return record; } @Test diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java index 607083f593ef2..791d653b0eb1c 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/JsonSchemaSerializationTest.java @@ -49,7 +49,7 @@ void toJson() throws Exception { .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { final String json = jsonb.toJson(schema); assertEquals( - "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"errorCapable\":false,\"metadata\":false,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", json); } } @@ -71,7 +71,7 @@ void toJsonWithMeta() throws Exception { try (final Jsonb jsonb = JsonbBuilder .create(new JsonbConfig().withPropertyOrderStrategy(PropertyOrderStrategy.LEXICOGRAPHICAL))) { assertEquals( - "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\"}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", + "{\"entries\":[],\"metadata\":[{\"elementSchema\":{\"entries\":[],\"metadata\":[],\"props\":{},\"type\":\"STRING\"},\"errorCapable\":false,\"metadata\":true,\"name\":\"array\",\"nullable\":true,\"props\":{\"talend.component.label\":\"array\"},\"rawName\":\"array\",\"type\":\"ARRAY\",\"valid\":true}],\"props\":{\"talend.fields.order\":\"array\"},\"type\":\"RECORD\"}", jsonb.toJson(schema)); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java index 85bb91a1e62c7..9233c5bdc82d4 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ActionResourceImplTest.java @@ -198,7 +198,8 @@ void checkSchemaSerialization() { "{\n \"entries\":[\n {\n \"elementSchema\":{\n \"entries\":[\n ],\n" + " \"metadata\":[\n ],\n \"props\":{\n\n },\n \"type\":\"STRING\"\n" + - " },\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + + " },\n \"errorCapable\":false," + + "\n \"metadata\":false,\n \"name\":\"array\",\n \"nullable\":false,\n" + " \"props\":{\n\n },\n \"type\":\"ARRAY\",\n" + " \"valid\":true\n }\n ],\n \"metadata\":[\n" + " ],\n \"props\":{\n \"talend.fields.order\":\"array\"\n },\n \"type\":\"RECORD\"\n}"; @@ -239,7 +240,7 @@ void checkDiscoverProcessorSchema() { }, APPLICATION_JSON_TYPE), JsonObject.class); assertNotNull(guessed); final String expected = - "{\"entries\":[{\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"comment\":\"field2 comment\",\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\",\"valid\":true},{\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; + "{\"entries\":[{\"errorCapable\":false,\"metadata\":false,\"name\":\"field1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"comment\":\"field2 comment\",\"errorCapable\":false,\"metadata\":false,\"name\":\"field2\",\"nullable\":false,\"props\":{},\"type\":\"LONG\",\"valid\":true},{\"errorCapable\":false,\"metadata\":false,\"name\":\"V1\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true},{\"errorCapable\":false,\"metadata\":false,\"name\":\"driver\",\"nullable\":false,\"props\":{},\"type\":\"STRING\",\"valid\":true}],\"metadata\":[],\"props\":{\"talend.fields.order\":\"field1,field2,V1,driver\"},\"type\":\"RECORD\"}"; assertEquals(expected, guessed.toString()); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java index e30e2286e2d3c..d4111f8379d3d 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/beam/BeamActionSerializationTest.java @@ -45,8 +45,9 @@ void checkSchemaSerialization() { final String attended = "{\n" + " \"entries\":[\n" + " {\n" + " \"elementSchema\":{\n" + " \"entries\":[\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"STRING\"\n" + " },\n" - + " \"metadata\":false,\n" + " \"name\":\"array\",\n" + " \"nullable\":false,\n" - + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\",\n" + + + " \"errorCapable\":false,\n \"metadata\":false,\n" + + " \"name\":\"array\",\n" + " \"nullable\":false,\n" + + " \"props\":{\n" + "\n" + " },\n" + " \"type\":\"ARRAY\",\n" + " \"valid\":true\n }\n" + " ],\n" + " \"metadata\":[\n" + " ],\n" + " \"props\":{\n" + " \"talend.fields.order\":\"array\"\n" + " },\n" + " \"type\":\"RECORD\"\n" + "}"; diff --git a/sample-parent/sample-features/entry-with-error/pom.xml b/sample-parent/sample-features/entry-with-error/pom.xml index b136e3b977d73..dcccec81bae86 100644 --- a/sample-parent/sample-features/entry-with-error/pom.xml +++ b/sample-parent/sample-features/entry-with-error/pom.xml @@ -23,7 +23,7 @@ org.talend.sdk.component.sample.feature - supporterror + entrywitherror jar Component Runtime :: Sample Feature :: Entry with error support @@ -39,6 +39,15 @@ component-runtime-manager ${project.version} + + org.talend.sdk.component + component-runtime-beam + ${project.version} + + + org.apache.avro + avro + org.tomitribe tomitribe-crest @@ -97,18 +106,6 @@ - - org.apache.maven.plugins - maven-jar-plugin - - - - true - org.talend.sdk.component.sample.feature.entrywitherror.Cli - - - - maven-shade-plugin 2.1 @@ -124,19 +121,20 @@ org.talend.sdk.component.sample.feature.entrywitherror.Cli + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + - - - org.codehaus.mojo - exec-maven-plugin - 3.0.0 - - org.talend.sdk.component.sample.feature.entrywitherror.Cli - - diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 9570ec019fc37..f9183843a1925 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -24,12 +24,15 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.avro.generic.IndexedRecord; import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; +import org.talend.sdk.component.runtime.beam.spi.record.AvroRecord; import org.talend.sdk.component.runtime.input.InputImpl; import org.talend.sdk.component.runtime.input.Mapper; import org.talend.sdk.component.runtime.manager.ComponentManager; @@ -46,32 +49,43 @@ @NoArgsConstructor(access = PRIVATE) public final class Cli { - static final String GAV = "org.talend.sdk.component.sample.feature:supporterror:jar:" + static final String GAV = "org.talend.sdk.component.sample.feature:entrywitherror:jar:" + Versions.KIT_VERSION; - @Command("supporterror") + @Command("entry-with-error") public static void runInput( @Option("gav") @Default(GAV) final String gav, - @Option("support") @Default("false") final boolean support, - @Option("gen-some-errors") @Default("true") final boolean genErrors, - @Option("gen-nbrecords") @Default("10") final int nbRecords, + @Option("support-entry-with-error") @Default("false") final boolean supportEntryWithError, + @Option("use-avro-impl") @Default("false") final boolean useAvroImpl, + @Option("how-many-errors") @Default("0") final int howManyErrors, + @Option("gen-nb-records") @Default("10") final int nbRecords, + @Option("fields-in-error") @Default("age,date") final String fieldsInError, @Option("jar") final File jar, - @Option("family") @Default("supporterror") final String family, - @Option("mapper") @Default("SupportErrorMapper") final String mapper) { + @Option("family") @Default("sampleRecordWithEntriesInError") final String family, + @Option("mapper") @Default("RecordWithEntriesInErrorEmitter") final String mapper) { - info("support " + support); - if (support) { - setSupportError(support); - } + System.out.printf( + "Parameters:%n\tgav: %s%n\tsupport-entry-with-error: %s%n\tuse-avro-impl: %s%n\thow-many-errors: %d%n" + + "\tgen-nb-records: %d%n\tgfields-in-error: %s%n\tjar: %s%n\tfamily: %s%n\tmapper: %s%n", + gav, supportEntryWithError, useAvroImpl, howManyErrors, nbRecords, fieldsInError, jar, family, mapper); + + System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportEntryWithError)); + System.setProperty("talend.component.beam.record.factory.impl", useAvroImpl ? "avro" : "default"); Map config = new HashMap<>(); - config.put("configuration.generateErrors", String.valueOf(genErrors)); + config.put("configuration.howManyErrors", String.valueOf(howManyErrors)); config.put("configuration.nbRecords", String.valueOf(nbRecords)); - run(jar, gav, config, "sampleRecordWithEntriesInError", "RecordWithEntriesInErrorEmitter"); + + String[] fields = fieldsInError.split(","); + for (int i = 0; i < fields.length; i++) { + config.put("configuration.fieldsInError[" + i + "]", fields[i]); + } + + run(jar, gav, config, family, mapper, useAvroImpl); } private static void run(final File jar, final String gav, final Map configuration, - final String family, final String mapper) { + final String family, final String mapper, final boolean avro) { try (final ComponentManager manager = manager(jar, gav)) { info("configuration: " + configuration); @@ -89,39 +103,52 @@ private static void run(final File jar, final String gav, final Map { + String props = f.getObjectProps() + .entrySet() + .stream() + .map(es -> "\t\t\t" + es.getKey() + " = " + es.getValue()) + .collect(Collectors.joining("\n")); + System.out.printf("\t\tField '%s', properties: %n%s%n", f.name(), props); + }); + } + } public static void main(final String[] args) throws Exception { @@ -132,9 +159,7 @@ public static Object run(final String[] args) throws Exception { return new Main(Cli.class).exec(args); } - static final String ERROR = "[ERROR] "; - - static final String WARN = "[WARN] "; + static final String EXCEPTION = "[EXCEPTION] "; static final String INFO = "[INFO] "; @@ -202,12 +227,8 @@ public static void info(final String message) { System.out.println(INFO + message); } - public static void warn(final String message) { - System.err.println(WARN + message); - } - - public static void error(final Throwable e) { - System.err.println(ERROR + e.getMessage()); + public static void exception(final Throwable e) { + System.err.println(EXCEPTION + e.getMessage()); System.exit(501); } diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java index 576732e75298f..8a8558dc4625a 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java @@ -19,6 +19,8 @@ import java.time.Month; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; import java.util.function.Function; import javax.annotation.PostConstruct; @@ -35,7 +37,6 @@ import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; -import org.talend.sdk.component.runtime.record.SchemaImpl; import lombok.Data; @@ -64,21 +65,20 @@ public RecordWithEntriesInErrorEmitter( @PostConstruct public void init() { - recordSchema = new SchemaImpl.BuilderImpl() - .withType(Schema.Type.RECORD) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + recordSchema = recordBuilderFactory.newSchemaBuilder(Schema.Type.RECORD) + .withEntry(recordBuilderFactory.newEntryBuilder() .withName("name") .withNullable(false) .withErrorCapable(true) .withType(Schema.Type.STRING) .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withEntry(recordBuilderFactory.newEntryBuilder() .withName("date") .withNullable(false) .withErrorCapable(true) .withType(Schema.Type.DATETIME) .build()) - .withEntry(new SchemaImpl.EntryImpl.BuilderImpl() + .withEntry(recordBuilderFactory.newEntryBuilder() .withName("age") .withNullable(false) .withErrorCapable(true) @@ -87,12 +87,18 @@ public void init() { .build(); createRecordFunction = i -> { - Builder builder = recordBuilderFactory.newRecordBuilder(recordSchema).withString("name", "name " + i); + Builder builder = recordBuilderFactory.newRecordBuilder(recordSchema); - // Generate error only on odd generated records - boolean generateErrors = config.isGenerateErrors() && i % 2 == 0; + boolean generateErrors = config.getHowManyErrors() >= i; - if (generateErrors) { + if (generateErrors && config.getFieldsInError().contains("name")) { + Entry nameEntry = recordSchema.getEntry("name"); + builder.with(nameEntry, null); + } else { + builder.withString("name", "name " + i); + } + + if (generateErrors && config.getFieldsInError().contains("date")) { Entry dateEntry = recordSchema.getEntry("date"); builder.with(dateEntry, "789-555"); } else { @@ -109,7 +115,7 @@ public void init() { builder.withDateTime("date", dateTime); } - if (generateErrors) { + if (generateErrors && config.getFieldsInError().contains("age")) { Entry ageEntry = recordSchema.getEntry("age"); builder.with(ageEntry, "-78"); } else { @@ -132,19 +138,24 @@ public Record data() { @Data @GridLayout(value = { - @GridLayout.Row("generateErrors"), + @GridLayout.Row("howManyErrors"), @GridLayout.Row("nbRecords"), + @GridLayout.Row("fieldsInError"), }) public static class Config implements Serializable { @Option - @Documentation("If true, generate some errors.") - private boolean generateErrors = true; + @Documentation("The number of errors to generate..") + private int howManyErrors; @Option @Documentation("Number of generated records.") private int nbRecords = 5; + @Option + @Documentation("Fields in error.") + private List fieldsInError = new ArrayList<>(); + } } diff --git a/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties index 87398b474113c..37eaf22a94654 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties +++ b/sample-parent/sample-features/entry-with-error/src/main/resources/org/talend/sdk/component/sample/feature/entrywitherror/Messages.properties @@ -14,6 +14,8 @@ # Here you can change all your configuration display names to use more explicit labels # You can also translate your configuration by adding one file by local Messages_fr.properties for french for example sampleRecordWithEntriesInError.RecordWithEntriesInErrorEmitter._displayName = Record with entries in error emitter -Config.generateErrors._displayName = Generate some errors +Config.howManyErrors._displayName = Number of errors to generate +Config.howManyErrors._placeholder = Config.nbRecords._displayName = Nb generated records -Config.nbRecords._placeholder = +Config.nbRecords._placeholder = +Config.fieldsInError._displayName = Fields in error \ No newline at end of file From 493d65a31b124c4cd16813848375936d35c21917 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 10:59:14 +0200 Subject: [PATCH 53/62] feat(QTDI-1305): the CLI display error in 'name' attribute. --- .../component/sample/feature/entrywitherror/Cli.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index f9183843a1925..05e793d5b82cf 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -119,7 +119,15 @@ private static void recordOut(final int count, final Record record, final boolea System.out.println("-----------------------------------------------------"); System.out.printf("Record (%s) no %s is valid ? %s%n", record.getClass().getSimpleName(), count, record.isValid() ? "yes" : "no"); - System.out.printf("\tName: %s%n", record.getString("name")); + + Entry name = record.getSchema().getEntry("name"); + if (name.isValid()) { + System.out.printf("\tName: %s%n", record.getString("name")); + } else { + System.out.printf("\tName is on error: %n\t\tMessage:%s%n\t\tFallback value: %s%n", + name.getErrorMessage(), name.getErrorFallbackValue()); + } + Entry date = record.getSchema().getEntry("date"); if (date.isValid()) { System.out.printf("\tDate: %s%n", record.getDateTime("date")); From a31416edb385accefc8ef53b23ee34cffbfaca5d Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 13:52:24 +0200 Subject: [PATCH 54/62] feat(QTDI-1305): the CLI display value even when error. --- .../sample/feature/entrywitherror/Cli.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 05e793d5b82cf..8970a915d1c76 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -20,6 +20,7 @@ import static org.talend.sdk.component.runtime.manager.ComponentManager.findM2; import java.io.File; +import java.time.ZonedDateTime; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -121,27 +122,33 @@ private static void recordOut(final int count, final Record record, final boolea record.isValid() ? "yes" : "no"); Entry name = record.getSchema().getEntry("name"); + String nameValue = record.getString("name"); if (name.isValid()) { - System.out.printf("\tName: %s%n", record.getString("name")); + System.out.printf("\tName: %s%n", nameValue); } else { - System.out.printf("\tName is on error: %n\t\tMessage:%s%n\t\tFallback value: %s%n", - name.getErrorMessage(), name.getErrorFallbackValue()); + System.out.printf( + "\tName is on error: %n\t\tValue (should be null): %s%n\t\tMessage:%s%n\t\tFallback value: %s%n", + nameValue, name.getErrorMessage(), name.getErrorFallbackValue()); } Entry date = record.getSchema().getEntry("date"); + ZonedDateTime dateValue = record.getDateTime("date"); if (date.isValid()) { - System.out.printf("\tDate: %s%n", record.getDateTime("date")); + System.out.printf("\tDate: %s%n", dateValue); } else { - System.out.printf("\tDate is on error: %n\t\tMessage:%s%n\t\tFallback value: %s%n", - date.getErrorMessage(), date.getErrorFallbackValue()); + System.out.printf( + "\tDate is on error: %n\t\tValue (should be null): %s%n\t\tMessage:%s%n\t\tFallback value: %s%n", + dateValue, date.getErrorMessage(), date.getErrorFallbackValue()); } Entry age = record.getSchema().getEntry("age"); + Integer ageValue = record.get(Integer.class, "age"); if (age.isValid()) { - System.out.printf("\tAge: %s%n", record.getInt("age")); + System.out.printf("\tAge: %s%n", ageValue); } else { - System.out.printf("\tAge is on error: %n\t\tMessage:%s%n\t\tFallback value: %s%n", - age.getErrorMessage(), age.getErrorFallbackValue()); + System.out.printf( + "\tAge is on error: %n\t\tValue (should be null): %s%n\t\tMessage:%s%n\t\tFallback value: %s%n", + ageValue, age.getErrorMessage(), age.getErrorFallbackValue()); } if (avro) { @@ -176,8 +183,6 @@ public static Object run(final String[] args) throws Exception { public static ComponentManager manager(final File jar, final String artifact) { return new ComponentManager(findM2()) { - final ContainerFinder containerFinder = ContainerFinder.Instance.get(); - final ComponentManager originalMgr = contextualInstance().get(); { @@ -236,7 +241,7 @@ public static void info(final String message) { } public static void exception(final Throwable e) { - System.err.println(EXCEPTION + e.getMessage()); + System.err.printf("%s%s%n%s%n", EXCEPTION, e.getMessage(), e); System.exit(501); } From 2e24616b49187c9da6cadd9ebe5bada0d263a49d Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 14:39:54 +0200 Subject: [PATCH 55/62] feat(QTDI-1305): Fix icons. --- .../RecordWithEntriesInErrorEmitter.java | 2 +- .../src/main/resources/icons/dark/icon.svg | 2 +- .../src/main/resources/icons/dark/mapper.svg | 65 ------------------- .../src/main/resources/icons/light/icon.svg | 2 +- .../src/main/resources/icons/light/mapper.svg | 65 ------------------- 5 files changed, 3 insertions(+), 133 deletions(-) delete mode 100644 sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg delete mode 100644 sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java index 8a8558dc4625a..798554e08b47c 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java @@ -41,7 +41,7 @@ import lombok.Data; @Version -@Icon(value = Icon.IconType.CUSTOM, custom = "mapper") +@Icon(value = Icon.IconType.CUSTOM, custom = "icon") @Emitter(name = "RecordWithEntriesInErrorEmitter") @Documentation("Generated record with entries in error.") public class RecordWithEntriesInErrorEmitter implements Serializable { diff --git a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg index 3351e2c0bc9e9..7932a76568dae 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg +++ b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/icon.svg @@ -1,5 +1,5 @@ - Checkpoint​Input + Entry​with error \ No newline at end of file diff --git a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg deleted file mode 100644 index 4f0fafff9a769..0000000000000 --- a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/dark/mapper.svg +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - - - - - diff --git a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg index 3351e2c0bc9e9..7932a76568dae 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg +++ b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/icon.svg @@ -1,5 +1,5 @@ - Checkpoint​Input + Entry​with error \ No newline at end of file diff --git a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg b/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg deleted file mode 100644 index 2958a0056838e..0000000000000 --- a/sample-parent/sample-features/entry-with-error/src/main/resources/icons/light/mapper.svg +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - - - - - From d0b2bf9638a15c5ac1633fb68965e1dc574c830e Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 14:53:25 +0200 Subject: [PATCH 56/62] feat(QTDI-1305): Improve properties keys. --- .../org/talend/sdk/component/api/record/SchemaProperty.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java index 25954bba2adb9..ea32ea06afa26 100644 --- a/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java +++ b/component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java @@ -37,11 +37,11 @@ public interface SchemaProperty { String ALLOW_SPECIAL_NAME = "field.special.name"; - String ENTRY_IS_ON_ERROR = "entry.on.error"; + String ENTRY_IS_ON_ERROR = "record.value.on.error"; - String ENTRY_ERROR_MESSAGE = "entry.error.message"; + String ENTRY_ERROR_MESSAGE = "record.value.on.error.message"; - String ENTRY_ERROR_FALLBACK_VALUE = "entry.error.fallback.value"; + String ENTRY_ERROR_FALLBACK_VALUE = "record.value.on.error.fallback_value"; enum LogicalType { From 7b847055028fd2a91bda4720e1872b1a4975a1e9 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 15:00:22 +0200 Subject: [PATCH 57/62] feat(QTDI-1305): Clean code. --- .../runtime/beam/spi/record/AvroRecordBuilderTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 265c713ba2d8e..d94ce0023285d 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -285,8 +285,6 @@ private Record testWithError(final String supported) { .build(); Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(noErrorEntry).withEntry(ageEntry).build(); - Entry age = customerSchema.getEntry("age"); - Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); Record record = recordBuilder.with(nameEntry, null) .with(noErrorEntry, "normal") From c7c3f49ae9314d4c3e966ea95719d8bbe98718b9 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 15:15:22 +0200 Subject: [PATCH 58/62] feat(QTDI-1305): Clean code. --- .../sample/feature/entrywitherror/Cli.java | 20 +++++++++---------- .../RecordWithEntriesInErrorEmitter.java | 4 +--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 8970a915d1c76..93df05335ac7d 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -116,13 +116,13 @@ private static void run(final File jar, final String gav, final Map { String props = f.getObjectProps() diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java index 798554e08b47c..e6fc5b30ed418 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/RecordWithEntriesInErrorEmitter.java @@ -50,8 +50,6 @@ public class RecordWithEntriesInErrorEmitter implements Serializable { private final Config config; - private transient Schema recordSchema; - private transient Function createRecordFunction; private transient int index; @@ -65,7 +63,7 @@ public RecordWithEntriesInErrorEmitter( @PostConstruct public void init() { - recordSchema = recordBuilderFactory.newSchemaBuilder(Schema.Type.RECORD) + final Schema recordSchema = recordBuilderFactory.newSchemaBuilder(Schema.Type.RECORD) .withEntry(recordBuilderFactory.newEntryBuilder() .withName("name") .withNullable(false) From aaf7453cb54bf0caaf24de2e2b74ce106dbfb038 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Thu, 15 May 2025 15:42:13 +0200 Subject: [PATCH 59/62] feat(QTDI-1305): Fix readme. --- .../entry-with-error/README.md | 153 ++++++++++++++---- .../sample/feature/entrywitherror/Cli.java | 2 +- 2 files changed, 119 insertions(+), 36 deletions(-) diff --git a/sample-parent/sample-features/entry-with-error/README.md b/sample-parent/sample-features/entry-with-error/README.md index a37bef27b3eeb..dad382d0da918 100644 --- a/sample-parent/sample-features/entry-with-error/README.md +++ b/sample-parent/sample-features/entry-with-error/README.md @@ -13,44 +13,52 @@ ## Overview -This is a simple TCK connector plugin to test and validate the feature about Improve Record API to add error info for invalid entries. +This is a simple TCK connector plugin to test and validate the feature about improve record API to add error info for +invalid entries. This project contains two things: - 1. A sample connector plugin that implements the supporterror feature. - 2. A simple Cli runner that can be used to run the connector plugin and test the supporterror feature. + 1. A sample connector plugin that implements the entry-with-error feature. + 2. A simple Cli runner that can be used to run the connector plugin and test the entry-with-error feature. ## Usage ### How to build the sample connector plugin Checkout the code from the repository and build the project using `mvn clean install` -Alternatively build the feature module using `mvn install -am -pl :supporterror` +Alternatively build the feature module using `mvn install -am -pl :entrywitherror` ### How to run -To run the connector, you need exec the generated artifact `org.talend.sdk.component.sample.feature:supporterror`. +To run the connector, you need exec the generated artifact `org.talend.sdk.component.sample.feature:entrywitherror`. * You can run it directly from `target` folder or repository folder - * `java -jar target/supporterror-1.81.0-SNAPSHOT.jar` + * `java -jar target/entrywitherror-1.81.0-SNAPSHOT.jar ` * or you can run it from the maven repository - * `java -jar ~/.m2/repository/org/talend/sdk/component/sample/feature/supporterror/1.81.0-SNAPSHOT/supporterror-1.81.0-SNAPSHOT.jar` + * `java -jar ~/.m2/repository/org/talend/sdk/component/sample/feature/entrywitherror/1.81.0-SNAPSHOT/entrywitherror-1.81.0-SNAPSHOT.jar` For later usage, will use the variable `$RUNCMD` as the way you may choose. ⚠️ If you're using jdk17, don't forget to add the `--add-opens` option to the command line (see profile _jdk9_ in master pom at the repository's root) or use instead jdk11. ### How to use -Run supporterror with the option "-s" for execution: +Run `entry-with-error` command: ```bash -$ $RUNCMD supporterror - -Usage: supporterror [-s] - -Options: - --s use supporterror or not. - +$ $RUNCMD entry-with-error + +Usage: entry-with-error [options] + +Options: + --family= default: sampleRecordWithEntriesInError + --fields-in-error= default: age,date + --gav= default: org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT + --gen-nb-records= default: 10 + --how-many-errors= default: 0 + --jar= + --mapper= default: RecordWithEntriesInErrorEmitter + --support-entry-with-error + --use-avro-impl ``` ## Plugin artifact -There are two ways to run the supporterror runner with a specific plugin artifact: +There are two ways to run the entry-with-error runner with a specific plugin artifact: - Using the `--gav` option to specify the GAV of the plugin artifact. Syntax: `groupId:artifactId:version[:packaging[:classifier]]` @@ -60,30 +68,105 @@ There are two ways to run the supporterror runner with a specific plugin artifac ⚠️ You cannot use both options at the same time. ## Execution examples -### Run supporterror connector with default behavior -`java -jar target/supporterror-1.81.0-SNAPSHOT.jar supporterror -s` +### Run entry-with-error sample with default behavior +To execute with legacy behavior, just don't add the `--support-entry-with-error` option. The command +`$RUNCMD entry-with-error --gen-nb-records=4` will use default parameters values. It will generate 10 records with 0 errors. ```bash -[INFO] Manager is using plugin: supporterror from GAV org.talend.sdk.component.sample.feature:supporterror:jar:1.81.0-SNAPSHOT. -[INFO] support true -[INFO] create input now. -[INFO] getting the record. -[INFO] Record isValid = false -[INFO] Record 'name': example connector -[INFO] ERROR: date is null -[INFO] ERROR: wrong int value +Parameters: + gav: org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT + support-entry-with-error: false + use-avro-impl: false + how-many-errors: 0 + gen-nb-records: 4 + fields-in-error: age,date + jar: null + family: sampleRecordWithEntriesInError + mapper: RecordWithEntriesInErrorEmitter +[INFO] Manager is using plugin: entrywitherror from GAV org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT. +[INFO] configuration: {configuration.fieldsInError[0]=age, configuration.fieldsInError[1]=date, configuration.nbRecords=4, configuration.howManyErrors=0} +----------------------------------------------------- +Record (RecordImpl) no 1 is valid ? yes + Name: name 1 + Date: 2025-04-02T15:30Z[UTC] + Age: 51 +----------------------------------------------------- +Record (RecordImpl) no 2 is valid ? yes + Name: name 2 + Date: 2025-04-03T15:30Z[UTC] + Age: 52 +----------------------------------------------------- +Record (RecordImpl) no 3 is valid ? yes + Name: name 3 + Date: 2025-04-04T15:30Z[UTC] + Age: 53 +----------------------------------------------------- +Record (RecordImpl) no 4 is valid ? yes + Name: name 4 + Date: 2025-04-05T15:30Z[UTC] + Age: 54 +----------------------------------------------------- [INFO] finished. + +``` +The following command do the same, but an error is generated on the `age` attribute of the first record. So, an exception is raised: +```bash +$RUNCMD entry-with-error --gen-nb-records=4 --how-many-errors=1 --fields-in-error=age + +Parameters: + gav: org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT + support-entry-with-error: false use-avro-impl: false + how-many-errors: 1 + gen-nb-records: 4 gfields-in-error: age + jar: null + family: sampleRecordWithEntriesInError + mapper: RecordWithEntriesInErrorEmitter +[INFO] Manager is using plugin: entrywitherror from GAV org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT. +[INFO] configuration: {configuration.fieldsInError[0]=age, configuration.nbRecords=4, configuration.howManyErrors=1} +[EXCEPTION] Entry 'age' of type INT is not compatible with given value of type 'java.lang.String': '-78'. +java.lang.IllegalArgumentException: Entry 'age' of type INT is not compatible with given value of type 'java.lang.String': '-78'. ``` -### Run with framework feature disabled -In this example we turn off the supporterror capability. -`% java -jar target/supporterror-1.81.0-SNAPSHOT.jar supporterror` +### Run with entry-wit-error enable +In this example we turn on the `entry-with-error` capability with the option `--support-entry-with-error`. ```bash -[INFO] Manager is using plugin: supporterror from GAV org.talend.sdk.component.sample.feature:supporterror:jar:1.81.0-SNAPSHOT. -[INFO] support false -[INFO] create input now. -[INFO] getting the record. -[ERROR] date is null +$RUNCMD entry-with-error --gen-nb-records=4 --how-many-errors=1 --fields-in-error=age --support-entry-with-error + +Parameters: + gav: org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT support-entry-with-error: true + use-avro-impl: false + how-many-errors: 1 gen-nb-records: 4 + gfields-in-error: age + jar: null + family: sampleRecordWithEntriesInError + mapper: RecordWithEntriesInErrorEmitter +[INFO] Manager is using plugin: entrywitherror from GAV org.talend.sdk.component.sample.feature:entrywitherror:jar:1.81.0-SNAPSHOT. +[INFO] configuration: {configuration.fieldsInError[0]=age, configuration.nbRecords=4, configuration.howManyErrors=1} +----------------------------------------------------- +Record (RecordImpl) no 1 is valid ? no + Name: name 1 + Date: 2025-04-02T15:30Z[UTC] + Age is on error: + Value (should be null): null + Message:Entry 'age' of type INT is not compatible with given value of type 'java.lang.String': '-78'. + Fallback value: -78 +----------------------------------------------------- +Record (RecordImpl) no 2 is valid ? yes + Name: name 2 + Date: 2025-04-03T15:30Z[UTC] + Age: 52 +----------------------------------------------------- +Record (RecordImpl) no 3 is valid ? yes + Name: name 3 + Date: 2025-04-04T15:30Z[UTC] + Age: 53 +----------------------------------------------------- +Record (RecordImpl) no 4 is valid ? yes + Name: name 4 + Date: 2025-04-05T15:30Z[UTC] + Age: 54 +----------------------------------------------------- +[INFO] finished. ``` -We can see that no record is returned because throw errors. +The first record is on error, bu, bo exception is raise and following records are well generated. diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 93df05335ac7d..029940410fcf8 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -67,7 +67,7 @@ public static void runInput( System.out.printf( "Parameters:%n\tgav: %s%n\tsupport-entry-with-error: %s%n\tuse-avro-impl: %s%n\thow-many-errors: %d%n" + - "\tgen-nb-records: %d%n\tgfields-in-error: %s%n\tjar: %s%n\tfamily: %s%n\tmapper: %s%n", + "\tgen-nb-records: %d%n\tfields-in-error: %s%n\tjar: %s%n\tfamily: %s%n\tmapper: %s%n", gav, supportEntryWithError, useAvroImpl, howManyErrors, nbRecords, fieldsInError, jar, family, mapper); System.setProperty(Record.RECORD_ERROR_SUPPORT, String.valueOf(supportEntryWithError)); From a1840c9d65c45816164cf85664ad77669f1fefec Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Fri, 16 May 2025 13:42:54 +0200 Subject: [PATCH 60/62] feat(QTDI-1305): Adda property on IndexedRecord/schema to check if the record is valid. --- .../sdk/component/runtime/beam/spi/record/AvroRecord.java | 2 ++ .../runtime/beam/spi/record/KeysForAvroProperty.java | 4 +++- .../runtime/beam/spi/record/AvroRecordBuilderTest.java | 1 - .../sdk/component/sample/feature/entrywitherror/Cli.java | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecord.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecord.java index b275246fe6ffb..41b0b164426dd 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecord.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecord.java @@ -83,6 +83,8 @@ public AvroRecord(final Record record) { this.schema = AvroSchema.toAvroSchema(record.getSchema()); this.delegate = new GenericData.Record(this.schema.getActualDelegate()); + this.delegate.getSchema().addProp(KeysForAvroProperty.RECORD_IN_ERROR, String.valueOf(!record.isValid())); + record .getSchema() .getAllEntries() diff --git a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java index 937fbeeea3734..976a40984b3ba 100644 --- a/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java +++ b/component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java @@ -22,5 +22,7 @@ public interface KeysForAvroProperty { // alias that indicate field is metadata if present. String METADATA_ALIAS_NAME = "talend.field.__METADATA__"; - String IS_ERROR_CAPABLE = "entry.errorCapable"; + String IS_ERROR_CAPABLE = "talend.component.record.entry.errorCapable"; + + String RECORD_IN_ERROR = "talend.component.record.value.on.error"; } diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index d94ce0023285d..49b56e419683b 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -52,7 +52,6 @@ import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.EntriesOrder; -import org.talend.sdk.component.api.record.Schema.Entry; import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.runtime.beam.spi.AvroRecordBuilderFactoryProvider; diff --git a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java index 029940410fcf8..89c13e726bcda 100644 --- a/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java +++ b/sample-parent/sample-features/entry-with-error/src/main/java/org/talend/sdk/component/sample/feature/entrywitherror/Cli.java @@ -34,6 +34,7 @@ import org.talend.sdk.component.dependencies.maven.Artifact; import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; import org.talend.sdk.component.runtime.beam.spi.record.AvroRecord; +import org.talend.sdk.component.runtime.beam.spi.record.KeysForAvroProperty; import org.talend.sdk.component.runtime.input.InputImpl; import org.talend.sdk.component.runtime.input.Mapper; import org.talend.sdk.component.runtime.manager.ComponentManager; @@ -153,6 +154,8 @@ private static void recordOut(final int count, final Record aRecord, final boole if (avro) { IndexedRecord unwrap = ((AvroRecord) aRecord).unwrap(IndexedRecord.class); + System.out.printf("\tAvro IndexedRecord on error ? %s=%s%n", KeysForAvroProperty.RECORD_IN_ERROR, + unwrap.getSchema().getProp(KeysForAvroProperty.RECORD_IN_ERROR)); System.out.println("\tAvro fields properties:"); unwrap.getSchema().getFields().stream().forEach(f -> { String props = f.getObjectProps() From d5e09dff54c34a59fdad147c6849412bd3cc48c4 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Fri, 16 May 2025 14:14:42 +0200 Subject: [PATCH 61/62] feat(QTDI-1305): Display entries original name in error. --- .../org/talend/sdk/component/runtime/record/RecordImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java index 1bc5d45484878..31e2884860881 100644 --- a/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java +++ b/component-runtime-impl/src/main/java/org/talend/sdk/component/runtime/record/RecordImpl.java @@ -182,7 +182,7 @@ public Builder with(final Entry entry, final Object value) { if (!entry.getType().isCompatible(value)) { return withError(entry, value, String .format("Entry '%s' of type %s is not compatible with given value of type '%s': '%s'.", - entry.getName(), + entry.getOriginalFieldName(), entry.getType(), value.getClass().getName(), value)); } @@ -323,10 +323,10 @@ private Schema.Entry validateTypeAgainstProvidedSchema(final String name, final final Schema.Entry entry = this.findExistingEntry(name); if (entry.getType() != type) { throw new IllegalArgumentException( - "Entry '" + name + "' expected to be a " + entry.getType() + ", got a " + type); + "Entry '" + entry.getOriginalFieldName() + "' expected to be a " + entry.getType() + ", got a " + type); } if (value == null && !entry.isNullable()) { - throw new IllegalArgumentException("Entry '" + name + "' is not nullable"); + throw new IllegalArgumentException("Entry '" + entry.getOriginalFieldName() + "' is not nullable"); } return entry; } From 2780de79374fb02d293175bb7deeebc205fc0930 Mon Sep 17 00:00:00 2001 From: Yves Piel Date: Fri, 16 May 2025 15:34:36 +0200 Subject: [PATCH 62/62] feat(QTDI-1305): Add a unit test. --- .../spi/record/AvroRecordBuilderTest.java | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java index 49b56e419683b..05a7544f3cb6d 100644 --- a/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java +++ b/component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2025 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -212,12 +212,27 @@ void mixedRecordTest() { @Test void testWithoutErrorSupport() { - Assertions.assertThrows(IllegalArgumentException.class, () -> testWithError("false")); + Assertions.assertThrows(IllegalArgumentException.class, () -> testWithError("false", true)); } @Test - void testWithError() { - Record record = testWithError("true"); + void testWithErrorSupportButNoError() { + Record record = testWithError("true", false); + + IndexedRecord unwrap = ((AvroRecord) record).unwrap(IndexedRecord.class); + + Assertions.assertTrue(unwrap.getSchema().getProp(KeysForAvroProperty.RECORD_IN_ERROR).equals("false")); + + Field nameField = unwrap.getSchema().getFields().get(0); + Assertions.assertNull(nameField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + + Field ageField = unwrap.getSchema().getFields().get(2); + Assertions.assertNull(ageField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); + } + + @Test + void testWithErrorSupport() { + Record record = testWithError("true", true); assertFalse(record.isValid()); final Schema.Entry entry = record.getSchema().getEntry("name"); @@ -232,6 +247,8 @@ void testWithError() { IndexedRecord unwrap = ((AvroRecord) record).unwrap(IndexedRecord.class); + Assertions.assertTrue(unwrap.getSchema().getProp(KeysForAvroProperty.RECORD_IN_ERROR).equals("true")); + Field nameField = unwrap.getSchema().getFields().get(0); Assertions.assertEquals("true", nameField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); @@ -244,7 +261,7 @@ void testWithError() { Assertions.assertEquals("true", ageField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR)); Assertions.assertEquals("Entry 'age' of type INT is not compatible with given value of type " + - "'java.lang.String': 'is not an int'.", + "'java.lang.String': 'is not an int'.", ageField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE)); Assertions.assertEquals("is not an int", ageField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE)); @@ -256,7 +273,7 @@ void testWithError() { } - private Record testWithError(final String supported) { + private Record testWithError(final String supported, final boolean genError) { final String errorSupportBackup = System.getProperty(Record.RECORD_ERROR_SUPPORT); System.setProperty(Record.RECORD_ERROR_SUPPORT, supported); @@ -285,9 +302,9 @@ private Record testWithError(final String supported) { Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(noErrorEntry).withEntry(ageEntry).build(); Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema); - Record record = recordBuilder.with(nameEntry, null) + Record record = recordBuilder.with(nameEntry, genError ? null : "a string") .with(noErrorEntry, "normal") - .with(ageEntry, "is not an int") + .with(ageEntry, genError ? "is not an int" : 10) .build(); System.setProperty(Record.RECORD_ERROR_SUPPORT, errorSupportBackup == null ? "false" : errorSupportBackup); @@ -606,8 +623,8 @@ private Schema.Entry newEntry(final String name, Schema.Type type) { } private Schema.Entry newEntry(final String name, String rawname, Schema.Type type, boolean nullable, - Object defaultValue, - String comment) { + Object defaultValue, + String comment) { return new EntryImpl.BuilderImpl() .withName(name) .withRawName(rawname) @@ -623,7 +640,7 @@ private Schema.Entry newMetaEntry(final String name, Schema.Type type) { } private Schema.Entry newMetaEntry(final String name, String rawname, Schema.Type type, boolean nullable, - Object defaultValue, String comment) { + Object defaultValue, String comment) { return new EntryImpl.BuilderImpl() .withName(name) .withRawName(rawname)