feat(QTDI-1305): improve error in record

component-api/src/main/java/org/talend/sdk/component/api/record/Record.java

@@ -34,6 +34,8 @@
 
 public interface Record {
 
+    String RECORD_ERROR_SUPPORT = "talend.component.record.error.support";
+
     /**
      * @return the schema of this record.
      */
@@ -311,6 +313,11 @@ default Optional<Record> getOptionalRecord(final String name) {
         return ofNullable(get(Record.class, name));
     }
 
+    default boolean isValid() {
+        return !getSchema().getAllEntries()
+                .anyMatch(entry -> !entry.isValid());
+    }
+
     /**
      * Allows to create a record with a fluent API. This is the unique recommended way to create a record.
      */

component-api/src/main/java/org/talend/sdk/component/api/record/Schema.java

@@ -232,6 +232,16 @@ interface Entry {
          */
         boolean isMetadata();
 
+        /**
+         * @return Is this entry can be in error.
+         */
+        boolean isErrorCapable();
+
+        /**
+         * @return true if the value of this entry is valid; false for invalid value.
+         */
+        boolean isValid();
+
         /**
          * @param <T> the default value type.
          *
@@ -296,6 +306,14 @@ default Entry.Builder toBuilder() {
             throw new UnsupportedOperationException("#toBuilder is not implemented");
         }
 
+        default String getErrorMessage() {
+            return getProp(SchemaProperty.ENTRY_ERROR_MESSAGE);
+        }
+
+        default String getErrorFallbackValue() {
+            return getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE);
+        }
+
         /**
          * Plain builder matching {@link Entry} structure.
          */
@@ -317,6 +335,8 @@ default Builder withLogicalType(String logicalType) {
 
             Builder withNullable(boolean nullable);
 
+            Builder withErrorCapable(boolean errorCapable);
+
             Builder withMetadata(boolean metadata);
 
             <T> Builder withDefaultValue(T value);

component-api/src/main/java/org/talend/sdk/component/api/record/SchemaProperty.java

@@ -37,6 +37,12 @@ public interface SchemaProperty {
 
     String ALLOW_SPECIAL_NAME = "field.special.name";
 
+    String ENTRY_IS_ON_ERROR = "record.value.on.error";
+
+    String ENTRY_ERROR_MESSAGE = "record.value.on.error.message";
+
+    String ENTRY_ERROR_FALLBACK_VALUE = "record.value.on.error.fallback_value";
+
     enum LogicalType {
 
         DATE("date"),

component-api/src/main/java/org/talend/sdk/component/api/service/schema/Schema.java

@@ -129,6 +129,11 @@ public boolean isNullable() {
             return true;
         }
 
+        @Override
+        public boolean isErrorCapable() {
+            return false;
+        }
+
         @Override
         public boolean isMetadata() {
             return false;
@@ -163,6 +168,12 @@ public String getProp(final String property) {
         public Builder toBuilder() {
             throw new UnsupportedOperationException("#toBuilder()");
         }
+
+        @Override
+        public boolean isValid() {
+            return true;
+        }
+
     }
 
 }

component-api/src/test/java/org/talend/sdk/component/api/record/RecordTest.java

@@ -106,6 +106,11 @@ public boolean isNullable() {
                 return false;
             }
 
+            @Override
+            public boolean isErrorCapable() {
+                return false;
+            }
+
             @Override
             public boolean isMetadata() {
                 return false;
@@ -135,6 +140,11 @@ public Map<String, String> getProps() {
             public String getProp(String property) {
                 return null;
             }
+
+            @Override
+            public boolean isValid() {
+                return true;
+            }
         };
         Assertions.assertEquals("value", record.get(String.class, e1));
     }

component-api/src/test/java/org/talend/sdk/component/api/record/SchemaTest.java

@@ -291,6 +291,11 @@ public boolean isNullable() {
             throw new UnsupportedOperationException("#isNullable()");
         }
 
+        @Override
+        public boolean isErrorCapable() {
+            throw new UnsupportedOperationException("#isErrorCapable()");
+        }
+
         @Override
         public boolean isMetadata() {
             throw new UnsupportedOperationException("#isMetadata()");
@@ -326,6 +331,11 @@ public JsonValue getJsonProp(final String name) {
             return Entry.super.getJsonProp(name);
         }
 
+        @Override
+        public boolean isValid() {
+            return true;
+        }
+
     }
 
     class SchemaBuilder implements Schema.Builder {

component-api/src/test/java/org/talend/sdk/component/api/service/record/RecordBuilderFactoryTest.java

@@ -78,6 +78,8 @@ static class MockEntry implements Schema.Entry {
 
         private final boolean nullable;
 
+        private final boolean errorCapable;
+
         private final boolean metadata;
 
         private final Object defaultVal;
@@ -102,6 +104,11 @@ public String getProp(final String property) {
         public Builder toBuilder() {
             throw new UnsupportedOperationException("#toBuilder()");
         }
+
+        @Override
+        public boolean isValid() {
+            return true;
+        }
     }
 
     @RequiredArgsConstructor
@@ -135,6 +142,12 @@ public Entry.Builder withNullable(boolean nullable) {
             return this;
         }
 
+        @Override
+        public Entry.Builder withErrorCapable(boolean errorCapable) {
+            this.builder.withErrorCapable(errorCapable);
+            return this;
+        }
+
         @Override
         public Entry.Builder withMetadata(boolean metadata) {
             this.builder.withMetadata(metadata);

component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecord.java

@@ -83,6 +83,8 @@ public AvroRecord(final Record record) {
         this.schema = AvroSchema.toAvroSchema(record.getSchema());
         this.delegate = new GenericData.Record(this.schema.getActualDelegate());
 
+        this.delegate.getSchema().addProp(KeysForAvroProperty.RECORD_IN_ERROR, String.valueOf(!record.isValid()));
+
         record
                 .getSchema()
                 .getAllEntries()

component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchema.java

@@ -216,19 +216,23 @@ private Entry fromAvro(final Field field) {
     }
 
     private static Entry buildFromAvro(final Field field, final Type type,
-            final Optional<SchemaProperty.LogicalType> logicalType, final AvroSchema elementSchema) {
+            final Optional<SchemaProperty.LogicalType> logicalType,
+            final AvroSchema elementSchema) {
         Entry.Builder builder = new EntryImpl.BuilderImpl() //
                 .withName(field.name()) //
                 .withRawName(field.getProp(KeysForAvroProperty.LABEL)) //
                 .withType(type) //
                 .withNullable(field.schema().getType() == UNION) //
+                .withErrorCapable(Boolean.parseBoolean(field.getProp(KeysForAvroProperty.IS_ERROR_CAPABLE)))
                 .withMetadata(AvroSchema.isMetadata(field)) //
                 .withDefaultValue(field.defaultVal()) //
                 .withElementSchema(elementSchema) //
                 .withComment(field.doc()) //
                 .withProps(field.getObjectProps()
                         .entrySet()
                         .stream()
+                        // KeysForAvroProperty.IS_ERROR_CAPABLE is already managed above
+                        .filter(p -> !p.getKey().equals(KeysForAvroProperty.IS_ERROR_CAPABLE))
                         .collect(toMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))));
 
         logicalType.ifPresent(builder::withLogicalType);

component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/AvroSchemaBuilder.java

@@ -463,7 +463,9 @@ public static Field toField(final org.apache.avro.Schema schema, final Schema.En
             for (Map.Entry<String, String> e : entry.getProps().entrySet()) {
                 field.addProp(e.getKey(), e.getValue());
             }
-
+            if (entry.isErrorCapable()) {
+                field.addProp(KeysForAvroProperty.IS_ERROR_CAPABLE, String.valueOf(entry.isErrorCapable()));
+            }
             return field;
         }
     }

component-runtime-beam/src/main/java/org/talend/sdk/component/runtime/beam/spi/record/KeysForAvroProperty.java

@@ -21,4 +21,8 @@ public interface KeysForAvroProperty {
 
     // alias that indicate field is metadata if present.
     String METADATA_ALIAS_NAME = "talend.field.__METADATA__";
+
+    String IS_ERROR_CAPABLE = "talend.component.record.entry.errorCapable";
+
+    String RECORD_IN_ERROR = "talend.component.record.value.on.error";
 }

component-runtime-beam/src/test/java/org/talend/sdk/component/runtime/beam/spi/record/AvroRecordBuilderTest.java

@@ -1,12 +1,12 @@
 /**
  * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
- *
+ * <p>
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -41,6 +41,7 @@
 import javax.json.JsonArray;
 import javax.json.JsonObject;
 
+import org.apache.avro.Schema.Field;
 import org.apache.avro.generic.GenericDatumWriter;
 import org.apache.avro.generic.IndexedRecord;
 import org.apache.avro.io.EncoderFactory;
@@ -209,6 +210,108 @@
         Assertions.assertNotNull(arrayType);
     }
 
+    @Test
+    void testWithoutErrorSupport() {
+        Assertions.assertThrows(IllegalArgumentException.class, () -> testWithError("false", true));
+    }
+
+    @Test
+    void testWithErrorSupportButNoError() {
+        Record record = testWithError("true", false);
+
+        IndexedRecord unwrap = ((AvroRecord) record).unwrap(IndexedRecord.class);
+
+        Assertions.assertTrue(unwrap.getSchema().getProp(KeysForAvroProperty.RECORD_IN_ERROR).equals("false"));
+
+        Field nameField = unwrap.getSchema().getFields().get(0);
+        Assertions.assertNull(nameField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR));
+
+        Field ageField = unwrap.getSchema().getFields().get(2);
+        Assertions.assertNull(ageField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR));
+    }
+
+    @Test
+    void testWithErrorSupport() {
+        Record record = testWithError("true", true);
+        assertFalse(record.isValid());
+
+        final Schema.Entry entry = record.getSchema().getEntry("name");
+        assertNotNull(entry);
+        Assertions.assertFalse(entry.isValid());
+        Assertions.assertNull(record.getString("name"));
+
+        final Schema.Entry entry2 = record.getSchema().getEntry("age");
+        assertNotNull(entry2);
+        Assertions.assertFalse(entry2.isValid());
+        Assertions.assertNull(record.get(Integer.class, "age"));
+
+        IndexedRecord unwrap = ((AvroRecord) record).unwrap(IndexedRecord.class);
+
+        Assertions.assertTrue(unwrap.getSchema().getProp(KeysForAvroProperty.RECORD_IN_ERROR).equals("true"));
+
+        Field nameField = unwrap.getSchema().getFields().get(0);
+        Assertions.assertEquals("true",
+                nameField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR));
+        Assertions.assertEquals("Entry 'name' is not nullable",
+                nameField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE));
+        Assertions.assertEquals("null",
+                nameField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE));
+
+        Field ageField = unwrap.getSchema().getFields().get(2);
+        Assertions.assertEquals("true",
+                ageField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR));
+        Assertions.assertEquals("Entry 'age' of type INT is not compatible with given value of type " +
+                        "'java.lang.String': 'is not an int'.",
+                ageField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE));
+        Assertions.assertEquals("is not an int",
+                ageField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE));
+
+        Field noErrorField = unwrap.getSchema().getFields().get(1);
+        Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_IS_ON_ERROR));
+        Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_ERROR_MESSAGE));
+        Assertions.assertNull(noErrorField.getProp(SchemaProperty.ENTRY_ERROR_FALLBACK_VALUE));
+
+    }
+
+    private Record testWithError(final String supported, final boolean genError) {
+        final String errorSupportBackup = System.getProperty(Record.RECORD_ERROR_SUPPORT);
+        System.setProperty(Record.RECORD_ERROR_SUPPORT, supported);
+
+        org.talend.sdk.component.api.record.Schema.Builder schemaBuilder = factory.newSchemaBuilder(Schema.Type.RECORD);
+        Schema.Entry nameEntry = factory
+                .newEntryBuilder()
+                .withName("name")
+                .withErrorCapable(true)
+                .withNullable(false)
+                .withType(Schema.Type.STRING)
+                .build();
+        Schema.Entry noErrorEntry = factory
+                .newEntryBuilder()
+                .withName("normal")
+                .withErrorCapable(true)
+                .withNullable(true)
+                .withType(Schema.Type.STRING)
+                .build();
+        Schema.Entry ageEntry = factory
+                .newEntryBuilder()
+                .withName("age")
+                .withErrorCapable(true)
+                .withNullable(false)
+                .withType(Schema.Type.INT)
+                .build();
+        Schema customerSchema = schemaBuilder.withEntry(nameEntry).withEntry(noErrorEntry).withEntry(ageEntry).build();
+
+        Record.Builder recordBuilder = factory.newRecordBuilder(customerSchema);
+        Record record = recordBuilder.with(nameEntry, genError ? null : "a string")
+                .with(noErrorEntry, "normal")
+                .with(ageEntry, genError ? "is not an int" : 10)
+                .build();
+
+        System.setProperty(Record.RECORD_ERROR_SUPPORT, errorSupportBackup == null ? "false" : errorSupportBackup);
+
+        return record;
+    }
+
     @Test
     void recordWithNewSchema() {
         final Schema schema0 = new AvroSchemaBuilder()//
@@ -520,8 +623,8 @@
     }
 
     private Schema.Entry newEntry(final String name, String rawname, Schema.Type type, boolean nullable,
-            Object defaultValue,
-            String comment) {
+                                  Object defaultValue,
+                                  String comment) {
         return new EntryImpl.BuilderImpl()
                 .withName(name)
                 .withRawName(rawname)
@@ -537,7 +640,7 @@
     }
 
     private Schema.Entry newMetaEntry(final String name, String rawname, Schema.Type type, boolean nullable,
-            Object defaultValue, String comment) {
+                                      Object defaultValue, String comment) {
         return new EntryImpl.BuilderImpl()
                 .withName(name)
                 .withRawName(rawname)