diff --git a/cardano-db-sync/app/http-get-json-metadata.hs b/cardano-db-sync/app/http-get-json-metadata.hs index 760a20222..e9e20f1b3 100644 --- a/cardano-db-sync/app/http-get-json-metadata.hs +++ b/cardano-db-sync/app/http-get-json-metadata.hs @@ -143,11 +143,12 @@ runGetVote :: Text.Text -> Maybe VoteMetaHash -> DB.AnchorType -> IO () runGetVote file mExpectedHash vtype = do respBs <- BS.readFile (Text.unpack file) let respLBs = fromStrict respBs - (ocvd, val, hsh, mWarning) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash vtype Nothing - print ocvd + (mocvd, val, hsh, mWarning, isValidJson) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash vtype Nothing + print mocvd print val print $ bsBase16Encode hsh print mWarning + putStrLn $ "Is valid JSON: " ++ show isValidJson -- ------------------------------------------------------------------------------------------------ diff --git a/cardano-db-sync/cardano-db-sync.cabal b/cardano-db-sync/cardano-db-sync.cabal index d3e136fa6..a3a6c49ef 100644 --- a/cardano-db-sync/cardano-db-sync.cabal +++ b/cardano-db-sync/cardano-db-sync.cabal @@ -16,6 +16,7 @@ category: Cryptocurrency build-type: Custom extra-source-files: CHANGELOG.md schema/*.sql + test/testfiles/*.jsonld custom-setup setup-depends: @@ -334,6 +335,7 @@ test-suite test Cardano.DbSync.Era.Shelley.Generic.ScriptDataTest Cardano.DbSync.Era.Shelley.Generic.ScriptTest Cardano.DbSync.Gen + Cardano.DbSync.OffChain.VoteTest Cardano.DbSync.Util.AddressTest Cardano.DbSync.Util.Bech32Test Cardano.DbSync.Util.CborTest diff --git a/cardano-db-sync/src/Cardano/DbSync/OffChain.hs b/cardano-db-sync/src/Cardano/DbSync/OffChain.hs index ee94a64ec..ebe44ea62 100644 --- a/cardano-db-sync/src/Cardano/DbSync/OffChain.hs +++ b/cardano-db-sync/src/Cardano/DbSync/OffChain.hs @@ -349,27 +349,54 @@ fetchOffChainVoteData gateways time oVoteWorkQ = convert eres = case eres of Right sVoteData -> - let - offChainData = sovaOffChainVoteData sVoteData - minimalBody = Vote.getMinimalBody offChainData - vdt = - DB.OffChainVoteData - { DB.offChainVoteDataLanguage = Vote.getLanguage offChainData - , DB.offChainVoteDataComment = Vote.textValue <$> Vote.comment minimalBody - , DB.offChainVoteDataBytes = sovaBytes sVoteData - , DB.offChainVoteDataHash = sovaHash sVoteData - , DB.offChainVoteDataJson = sovaJson sVoteData - , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ - , DB.offChainVoteDataWarning = sovaWarning sVoteData - , DB.offChainVoteDataIsValid = Nothing - } - gaF ocvdId = mkGovAction ocvdId offChainData - drepF ocvdId = mkDrep ocvdId offChainData - authorsF ocvdId = map (mkAuthor ocvdId) $ Vote.getAuthors offChainData - referencesF ocvdId = map (mkReference ocvdId) $ mListToList $ Vote.references minimalBody - externalUpdatesF ocvdId = map (mkexternalUpdates ocvdId) $ mListToList $ Vote.externalUpdates minimalBody - in - OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF) + case (sovaIsValidJson sVoteData, sovaOffChainVoteData sVoteData) of + -- Scenario 1: Valid JSON + Valid CIP schema + (True, Just offChainData) -> + let + minimalBody = Vote.getMinimalBody offChainData + vdt = + DB.OffChainVoteData + { DB.offChainVoteDataLanguage = Vote.getLanguage offChainData + , DB.offChainVoteDataComment = Vote.textValue <$> Vote.comment minimalBody + , DB.offChainVoteDataBytes = sovaBytes sVoteData + , DB.offChainVoteDataHash = sovaHash sVoteData + , DB.offChainVoteDataJson = sovaJson sVoteData + , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ + , DB.offChainVoteDataWarning = sovaWarning sVoteData + , DB.offChainVoteDataIsValid = Just True + } + gaF ocvdId = mkGovAction ocvdId offChainData + drepF ocvdId = mkDrep ocvdId offChainData + authorsF ocvdId = map (mkAuthor ocvdId) $ Vote.getAuthors offChainData + referencesF ocvdId = map (mkReference ocvdId) $ mListToList $ Vote.references minimalBody + externalUpdatesF ocvdId = map (mkexternalUpdates ocvdId) $ mListToList $ Vote.externalUpdates minimalBody + in + OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF) + -- Scenario 2 & 3: Valid JSON but invalid CIP schema OR Invalid JSON + (_, _) -> + let + vdt = + DB.OffChainVoteData + { DB.offChainVoteDataLanguage = "" + , DB.offChainVoteDataComment = Nothing + , DB.offChainVoteDataBytes = sovaBytes sVoteData + , DB.offChainVoteDataHash = sovaHash sVoteData + , DB.offChainVoteDataJson = sovaJson sVoteData + , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ + , DB.offChainVoteDataWarning = sovaWarning sVoteData + , -- Just False if valid JSON but invalid schema, NULL if unparseable JSON + DB.offChainVoteDataIsValid = + if sovaIsValidJson sVoteData + then Just False + else Nothing + } + gaF _ = Nothing + drepF _ = Nothing + authorsF _ = [] + referencesF _ = [] + externalUpdatesF _ = [] + in + OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF) Left err -> OffChainVoteResultError $ DB.OffChainVoteFetchError diff --git a/cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs b/cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs index 4252a0f27..9c576b87c 100644 --- a/cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs +++ b/cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs @@ -108,36 +108,46 @@ httpGetOffChainVoteDataSingle vurl metaHash anchorType = do let req = httpGetBytes manager request 3000000 3000000 url httpRes <- handleExceptT (convertHttpException url) req (respBS, respLBS, mContentType) <- hoistEither httpRes - (ocvd, decodedValue, metadataHash, mWarning) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl) + (mocvd, decodedValue, metadataHash, mWarning, isValidJson) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl) pure $ SimplifiedOffChainVoteData { sovaHash = metadataHash , sovaBytes = respBS , sovaJson = Text.decodeUtf8 $ LBS.toStrict (Aeson.encode decodedValue) , sovaContentType = mContentType - , sovaOffChainVoteData = ocvd + , sovaOffChainVoteData = mocvd , sovaWarning = mWarning + , sovaIsValidJson = isValidJson } where url = OffChainVoteUrl vurl -parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text) +parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Maybe Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text, Bool) parseAndValidateVoteData bs lbs metaHash anchorType murl = do let metadataHash = Crypto.digest (Proxy :: Proxy Crypto.Blake2b_256) bs + -- First check if hash matches - this is critical and must fail if mismatch (hsh, mWarning) <- case unVoteMetaHash <$> metaHash of Just expectedMetaHashBs | metadataHash /= expectedMetaHashBs -> left $ OCFErrHashMismatch murl (renderByteArray expectedMetaHashBs) (renderByteArray metadataHash) _ -> pure (metadataHash, Nothing) - decodedValue <- + -- Hash matches, now try to decode as generic JSON + -- If this fails, we still want to store the data with is_valid = NULL and an error message + (decodedValue, isValidJson) <- case Aeson.eitherDecode' @Aeson.Value lbs of - Left err -> left $ OCFErrJsonDecodeFail murl (Text.pack err) - Right res -> pure res - ocvd <- - case Vote.eitherDecodeOffChainVoteData lbs anchorType of - Left err -> left $ OCFErrJsonDecodeFail murl (Text.pack err) - Right res -> pure res - pure (ocvd, decodedValue, hsh, mWarning) + Left err -> + -- Not valid JSON - create an error message object + pure (Aeson.object [("error", Aeson.String "Content is not valid JSON. See bytes column for raw data."), ("parse_error", Aeson.String $ Text.pack err)], False) + Right res -> pure (res, True) + -- Try to decode into strongly-typed vote data structure (only if JSON was valid) + -- If this fails (e.g., doNotList is string instead of bool), we still store with is_valid = false + let ocvd = + if isValidJson + then case Vote.eitherDecodeOffChainVoteData lbs anchorType of + Left _err -> Nothing -- Don't fail, just return Nothing (will set is_valid = false) + Right res -> Just res + else Nothing -- Not valid JSON, so can't parse as CIP + pure (ocvd, decodedValue, hsh, mWarning, isValidJson) httpGetBytes :: Http.Manager -> diff --git a/cardano-db-sync/src/Cardano/DbSync/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Types.hs index 5dc2cc58e..ae50eb596 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Types.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Types.hs @@ -193,8 +193,9 @@ data SimplifiedOffChainVoteData = SimplifiedOffChainVoteData , sovaBytes :: !ByteString , sovaJson :: !Text , sovaContentType :: !(Maybe ByteString) - , sovaOffChainVoteData :: !Vote.OffChainVoteData + , sovaOffChainVoteData :: !(Maybe Vote.OffChainVoteData) , sovaWarning :: !(Maybe Text) + , sovaIsValidJson :: !Bool } data Retry = Retry diff --git a/cardano-db-sync/test/Cardano/DbSync/OffChain/VoteTest.hs b/cardano-db-sync/test/Cardano/DbSync/OffChain/VoteTest.hs new file mode 100644 index 000000000..e53e40164 --- /dev/null +++ b/cardano-db-sync/test/Cardano/DbSync/OffChain/VoteTest.hs @@ -0,0 +1,146 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE NoImplicitPrelude #-} + +module Cardano.DbSync.OffChain.VoteTest (tests) where + +import qualified Cardano.Db as DB +import Cardano.DbSync.Error (runOrThrowIO) +import Cardano.DbSync.OffChain.Http (parseAndValidateVoteData) +import Cardano.Prelude hiding ((%)) +import qualified Data.Aeson as Aeson +import qualified Data.Aeson.Key as AesonKey +import qualified Data.Aeson.KeyMap as KeyMap +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as LBS +import qualified Data.Text as Text +import Hedgehog + +tests :: IO Bool +tests = + checkParallel $ + Group + "Cardano.DbSync.OffChain.Vote" + [ ("parseAndValidateVoteData handles valid CIP-119 format", prop_parseValidCIPFormat) + , ("parseAndValidateVoteData handles invalid CIP format (type error)", prop_parseInvalidCIPFormat) + , ("parseAndValidateVoteData handles valid JSON but invalid structure", prop_parseValidJsonInvalidStructure) + , ("parseAndValidateVoteData handles unparseable JSON", prop_parseUnparseableJson) + ] + +-- | Test that we can parse valid CIP-119 format correctly +-- Scenario: Valid JSON + Valid CIP schema -> is_valid = true +prop_parseValidCIPFormat :: Property +prop_parseValidCIPFormat = withTests 1 $ property $ do + -- Read the test file with valid CIP-119 format + fileContent <- liftIO $ BS.readFile "test/testfiles/valid-vote-minimal.jsonld" + let lbsContent = LBS.fromStrict fileContent + + -- Run the parser + result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData fileContent lbsContent Nothing DB.DrepAnchor Nothing + + let (mocvd, val, _hash, _warning, isValidJson) = result + + -- Should succeed in parsing generic JSON + annotate "Successfully parsed as generic JSON" + assert isValidJson + + -- Should successfully parse into strongly-typed OffChainVoteData + case mocvd of + Just _ocvd -> do + annotate "Successfully parsed into OffChainVoteData" + success + Nothing -> do + annotate "Failed to parse into OffChainVoteData" + failure + + -- Should have valid Aeson.Value + case Aeson.toJSON val of + Aeson.Object _obj -> do + annotate "Has valid JSON object" + success + _ -> do + annotate "Expected JSON object" + failure + +-- | Test that we can parse JSON with incorrect field types (e.g., doNotList as string instead of bool) +-- This is based on the issue https://github.com/IntersectMBO/cardano-db-sync/issues/1995 +-- Scenario: Valid JSON but invalid CIP schema -> is_valid = false +prop_parseInvalidCIPFormat :: Property +prop_parseInvalidCIPFormat = withTests 1 $ property $ do + -- Read the test file with invalid doNotList field (string instead of bool) + fileContent <- liftIO $ BS.readFile "test/testfiles/invalid-vote-type-error.jsonld" + let lbsContent = LBS.fromStrict fileContent + + -- Run the parser + result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData fileContent lbsContent Nothing DB.DrepAnchor Nothing + + let (mocvd, val, _hash, _warning, isValidJson) = result + + -- Should succeed in parsing generic JSON + annotate "Successfully parsed as generic JSON" + assert isValidJson + + -- Should fail to parse into strongly-typed OffChainVoteData + assert $ isNothing mocvd + + -- But should have valid Aeson.Value + case Aeson.toJSON val of + Aeson.Object _obj -> do + annotate "Has valid JSON object" + success + _ -> do + annotate "Expected JSON object" + failure + +-- | Test with completely valid JSON but not matching the CIP schema +-- Scenario: Valid JSON but invalid CIP schema -> is_valid = false +prop_parseValidJsonInvalidStructure :: Property +prop_parseValidJsonInvalidStructure = withTests 1 $ property $ do + -- Read the test file with valid JSON but wrong structure + fileContent <- liftIO $ BS.readFile "test/testfiles/invalid-vote-wrong-structure.jsonld" + let lbsContent = LBS.fromStrict fileContent + + -- This should succeed because it's valid JSON, just not matching the schema + result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData fileContent lbsContent Nothing DB.DrepAnchor Nothing + + let (mocvd, _val, _hash, _warning, isValidJson) = result + + annotate "Successfully parsed generic JSON" + assert isValidJson + -- Should not parse into OffChainVoteData + assert $ isNothing mocvd + +-- | Test with completely unparseable content (not valid JSON at all) +-- Scenario: Invalid JSON but hash matches -> is_valid = NULL +prop_parseUnparseableJson :: Property +prop_parseUnparseableJson = withTests 1 $ property $ do + -- Read the test file with malformed JSON + fileContent <- liftIO $ BS.readFile "test/testfiles/invalid-vote-malformed-json.jsonld" + let lbsContent = LBS.fromStrict fileContent + + -- This should not fail, but instead return an error message in the JSON field + result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData fileContent lbsContent Nothing DB.DrepAnchor Nothing + + let (mocvd, val, _hash, _warning, isValidJson) = result + + annotate "Content is not valid JSON" + -- Should flag as invalid JSON + assert $ not isValidJson + + -- Should not parse into OffChainVoteData + assert $ isNothing mocvd + + -- Should have an error message in the JSON value + case val of + Aeson.Object obj -> do + annotate "Has error message object" + -- Check that error field exists + case KeyMap.lookup (AesonKey.fromString "error") obj of + Just (Aeson.String msg) -> do + annotate $ "Error message: " <> show msg + assert $ Text.isInfixOf "not valid JSON" msg + _ -> do + annotate "Expected error field with string value" + failure + _ -> do + annotate "Expected JSON object with error message" + failure diff --git a/cardano-db-sync/test/Main.hs b/cardano-db-sync/test/Main.hs index b629870a1..b32b9ce61 100644 --- a/cardano-db-sync/test/Main.hs +++ b/cardano-db-sync/test/Main.hs @@ -4,6 +4,7 @@ import qualified Cardano.DbSync.ApiTest as Api import qualified Cardano.DbSync.Config.TypesTest as Types import qualified Cardano.DbSync.Era.Shelley.Generic.ScriptDataTest as ScriptData import qualified Cardano.DbSync.Era.Shelley.Generic.ScriptTest as Script +import qualified Cardano.DbSync.OffChain.VoteTest as VoteTest import qualified Cardano.DbSync.Util.AddressTest as Address import qualified Cardano.DbSync.Util.Bech32Test as Bech32 import qualified Cardano.DbSync.Util.CborTest as Cbor @@ -23,4 +24,5 @@ main = , DbSync.tests , Types.tests , Api.tests + , VoteTest.tests ] diff --git a/cardano-db-sync/test/testfiles/invalid-vote-malformed-json.jsonld b/cardano-db-sync/test/testfiles/invalid-vote-malformed-json.jsonld new file mode 100644 index 000000000..ad07c7392 --- /dev/null +++ b/cardano-db-sync/test/testfiles/invalid-vote-malformed-json.jsonld @@ -0,0 +1,4 @@ +{ + "givenName": "Test DRep", + "objectives": "Missing closing quote and brace + "motivations": "This JSON is malformed diff --git a/cardano-db-sync/test/testfiles/invalid-vote-type-error.jsonld b/cardano-db-sync/test/testfiles/invalid-vote-type-error.jsonld new file mode 100644 index 000000000..c4288706d --- /dev/null +++ b/cardano-db-sync/test/testfiles/invalid-vote-type-error.jsonld @@ -0,0 +1,68 @@ +{ + "@context": { + "CIP100": "https://github.com/cardano-foundation/CIPs/blob/master/CIP-0100/README.md#", + "CIP119": "https://github.com/cardano-foundation/CIPs/blob/master/CIP-0119/README.md#", + "hashAlgorithm": "CIP100:hashAlgorithm", + "body": { + "@id": "CIP119:body", + "@context": { + "references": { + "@id": "CIP119:references", + "@container": "@set", + "@context": { + "GovernanceMetadata": "CIP100:GovernanceMetadataReference", + "Identity": "CIP119:IdentityReference", + "Link": "CIP119:LinkReference", + "Other": "CIP100:OtherReference", + "label": "CIP100:reference-label", + "uri": "CIP100:reference-uri" + } + }, + "paymentAddress": "CIP119:paymentAddress", + "givenName": "CIP119:givenName", + "image": "CIP119:image", + "objectives": "CIP119:objectives", + "motivations": "CIP119:motivations", + "qualifications": "CIP119:qualifications", + "doNotList": "CIP119:doNotList" + } + }, + "authors": { + "@id": "CIP100:authors", + "@container": "@set", + "@context": { + "name": "http://xmlns.com/foaf/0.1/name", + "witness": { + "@id": "CIP100:witness", + "@context": { + "witnessAlgorithm": "CIP100:witnessAlgorithm", + "publicKey": "CIP100:publicKey", + "signature": "CIP100:signature" + } + } + } + } + }, + "authors": [], + "hashAlgorithm": "blake2b-256", + "body": { + "givenName": "Test DRep", + "doNotList": "false", + "motivations": "Test motivations for DRep registration", + "objectives": "Test objectives for governance participation", + "qualifications": "Test qualifications and experience", + "paymentAddress": "addr1test000000000000000000000000000000000000000000000000000000", + "references": [ + { + "@type": "Link", + "label": "Test Website", + "uri": "https://example.com/test" + }, + { + "@type": "Identity", + "label": "Test Identity", + "uri": "https://example.com/identity" + } + ] + } +} diff --git a/cardano-db-sync/test/testfiles/invalid-vote-wrong-structure.jsonld b/cardano-db-sync/test/testfiles/invalid-vote-wrong-structure.jsonld new file mode 100644 index 000000000..6fee94e85 --- /dev/null +++ b/cardano-db-sync/test/testfiles/invalid-vote-wrong-structure.jsonld @@ -0,0 +1,17 @@ +{ + "title": "Some Random Document", + "description": "This is valid JSON but doesn't match CIP-100/119 schema at all", + "version": "1.0", + "data": { + "field1": "value1", + "field2": 42, + "field3": true, + "nested": { + "items": [1, 2, 3], + "metadata": { + "timestamp": "2025-01-01T00:00:00Z" + } + } + }, + "tags": ["test", "example", "invalid-schema"] +} diff --git a/cardano-db-sync/test/testfiles/valid-vote-minimal.jsonld b/cardano-db-sync/test/testfiles/valid-vote-minimal.jsonld new file mode 100644 index 000000000..b6a4701ca --- /dev/null +++ b/cardano-db-sync/test/testfiles/valid-vote-minimal.jsonld @@ -0,0 +1,68 @@ +{ + "@context": { + "CIP100": "https://github.com/cardano-foundation/CIPs/blob/master/CIP-0100/README.md#", + "CIP119": "https://github.com/cardano-foundation/CIPs/blob/master/CIP-0119/README.md#", + "hashAlgorithm": "CIP100:hashAlgorithm", + "body": { + "@id": "CIP119:body", + "@context": { + "references": { + "@id": "CIP119:references", + "@container": "@set", + "@context": { + "GovernanceMetadata": "CIP100:GovernanceMetadataReference", + "Identity": "CIP119:IdentityReference", + "Link": "CIP119:LinkReference", + "Other": "CIP100:OtherReference", + "label": "CIP100:reference-label", + "uri": "CIP100:reference-uri" + } + }, + "paymentAddress": "CIP119:paymentAddress", + "givenName": "CIP119:givenName", + "image": "CIP119:image", + "objectives": "CIP119:objectives", + "motivations": "CIP119:motivations", + "qualifications": "CIP119:qualifications", + "doNotList": "CIP119:doNotList" + } + }, + "authors": { + "@id": "CIP100:authors", + "@container": "@set", + "@context": { + "name": "http://xmlns.com/foaf/0.1/name", + "witness": { + "@id": "CIP100:witness", + "@context": { + "witnessAlgorithm": "CIP100:witnessAlgorithm", + "publicKey": "CIP100:publicKey", + "signature": "CIP100:signature" + } + } + } + } + }, + "authors": [], + "hashAlgorithm": "blake2b-256", + "body": { + "givenName": "Test DRep", + "doNotList": false, + "motivations": "Test motivations for DRep registration", + "objectives": "Test objectives for governance participation", + "qualifications": "Test qualifications and experience", + "paymentAddress": "addr1test000000000000000000000000000000000000000000000000000000", + "references": [ + { + "@type": "Link", + "label": "Test Website", + "uri": "https://example.com/test" + }, + { + "@type": "Identity", + "label": "Test Identity", + "uri": "https://example.com/identity" + } + ] + } +} diff --git a/cardano-db/src/Cardano/Db/Schema/Core/OffChain.hs b/cardano-db/src/Cardano/Db/Schema/Core/OffChain.hs index d7c345fec..a4e539a4d 100644 --- a/cardano-db/src/Cardano/Db/Schema/Core/OffChain.hs +++ b/cardano-db/src/Cardano/Db/Schema/Core/OffChain.hs @@ -90,7 +90,15 @@ offChainPoolFetchErrorEncoder = -- | -- Table Name: off_chain_vote_data --- Description: +-- Description: Stores off-chain voting anchor data with validation status +-- +-- The is_valid column indicates the parsing status: +-- • TRUE: Content is valid JSON AND conforms to CIP schema +-- All related tables are populated +-- • FALSE: Content is valid JSON BUT does not conform to CIP schema +-- The json column contains the actual JSON, but related tables are empty +-- • NULL: Content is not valid JSON at all +-- The json column contains an error message, bytes column has raw data data OffChainVoteData = OffChainVoteData { offChainVoteDataVotingAnchorId :: !Id.VotingAnchorId -- noreference , offChainVoteDataHash :: !ByteString diff --git a/doc/schema.md b/doc/schema.md index 14c6481bd..133a93518 100644 --- a/doc/schema.md +++ b/doc/schema.md @@ -1014,7 +1014,14 @@ A table containing pool offchain data fetch errors. ### `off_chain_vote_data` -The table with the offchain metadata related to Vote Anchors. It accepts metadata in a more lenient way than what's decribed in CIP-100. New in 13.2-Conway. +Stores off-chain voting anchor data with validation status. The table accepts metadata in a more lenient way than what's described in CIP-100. Only data with hash matches are stored here; hash mismatches are stored in off_chain_vote_fetch_error for retry. + +The is_valid column indicates the parsing status: + • TRUE: Content is valid JSON AND conforms to CIP-100 schema. All related fields (language, comment) and related tables (off_chain_vote_gov_action_data, off_chain_vote_drep_data, off_chain_vote_author, off_chain_vote_reference, off_chain_vote_external_update) are populated. + • FALSE: Hash matches and content is valid JSON BUT does not conform to CIP-100 schema. The json column contains the actual JSON, but language/comment fields and related tables remain empty. + • NULL: Hash matches but content is not valid JSON at all. The json column contains an error message, bytes column has raw data. Language/comment fields and related tables remain empty. + +New in 13.2-Conway. * Primary Id: `id` @@ -1028,7 +1035,7 @@ The table with the offchain metadata related to Vote Anchors. It accepts metadat | `json` | jsonb | The payload as JSON. | | `bytes` | bytea | The raw bytes of the payload. | | `warning` | string | A warning that occured while validating the metadata. | -| `is_valid` | boolean | False if the data is found invalid. db-sync leaves this field null since it normally populates off_chain_vote_fetch_error for invalid data. It can be used manually to mark some metadata invalid by clients. | +| `is_valid` | boolean | Indicates validation status: TRUE for valid JSON conforming to CIP-100 schema, FALSE for valid JSON not conforming to CIP-100 schema, NULL for content that is not valid JSON at all. | ### `off_chain_vote_gov_action_data` diff --git a/flake.nix b/flake.nix index 734c9d5a8..dc07f879b 100644 --- a/flake.nix +++ b/flake.nix @@ -218,6 +218,8 @@ # Override extra-source-files packages.cardano-db-sync.package.extraSrcFiles = [ "../schema/*.sql" ]; + packages.cardano-db-sync.components.tests.test.extraSrcFiles = + [ "test/testfiles/*.jsonld" ]; packages.cardano-db.package.extraSrcFiles = ["../config/pgpass-testnet"]; packages.cardano-db.components.tests.schema-rollback.extraSrcFiles =