cardano-rpc | Add decoded PlutusData and NativeScript in proto definition

.gitattributes

@@ -0,0 +1,3 @@
+cardano-rpc/gen/** linguist-generated=true
+cardano-rpc/gen/** no-prettify
+

.github/workflows/haskell.yml

@@ -86,7 +86,12 @@ jobs:
 
     - name: "[Windows] Install grpc dependencies"
       if: runner.os == 'Windows'
-      run: /usr/bin/pacman --noconfirm -S mingw-w64-x86_64-snappy mingw-w64-x86_64-protobuf
+      run: |
+        /usr/bin/pacman --noconfirm -S mingw-w64-x86_64-snappy mingw-w64-x86_64-protobuf
+        cat <<EOF >> $GITHUB_ENV
+        LIBRARY_PATH=/mingw64/lib
+        CPATH=/mingw64/include
+        EOF
 
     - name: "[macOS] Install grpc dependencies"
       if: runner.os == 'macOS'
@@ -107,6 +112,34 @@ jobs:
         cabal-store: ${{ steps.setup-haskell.outputs.cabal-store }}
         cache-version: ${{ env.CABAL_CACHE_VERSION }}
 
+
+    - name: '[Linux] [cardano-rpc] Install buf'
+      if: runner.os == 'Linux'
+      run: |
+        curl -sSL "https://github.com/bufbuild/buf/releases/latest/download/buf-Linux-x86_64" -o "/usr/local/bin/buf"
+        chmod +x /usr/local/bin/buf
+
+    - name: '[Linux] [cardano-rpc] Install proto-lens-protoc'
+      if: runner.os == 'Linux'
+      run: |
+        cabal install proto-lens-protoc --installdir=$HOME/.local/bin
+
+    - name: '[Linux] [cardano-rpc] Generate protobuf code'
+      if: runner.os == 'Linux'
+      working-directory: cardano-rpc
+      run: buf generate proto
+
+    - name: '[Linux] [cardano-rpc] Check that generated files from proto definitions are up to date'
+      if: runner.os == 'Linux'
+      run: |
+        git add cardano-rpc/gen
+        if ! git diff --staged --quiet -- cardano-rpc/gen; then
+          echo "Generated files differ from repository files:"
+          git diff --staged --name-status -- cardano-rpc/gen
+          exit 1
+        fi
+        echo "All generated files match repository files"
+
     # Now we build.
     - name: Build all
       run: cabal build all --enable-tests

.github/workflows/hls.yml

@@ -119,6 +119,4 @@ jobs:
           cradle:
             cabal:
           EOF
-          # Remove the following line after `cardano-rpc` project is enabled in `cabal.project`
-          mv cardano-rpc ../
           haskell-language-server

.hlint.yaml

@@ -60,6 +60,9 @@
 # Allow snake_case naming in test modules (following Haskell test property convention)
 - ignore: {name: Use camelCase, within: [Test.Cardano.Api.**, Test.Golden.Cardano.Api.**]}
 
+# Ignore all files in cardano-rpc/gen (generated code)
+- ignore: {within: [cardano-rpc/gen/**]}
+
 - ignore: {name: Eta reduce}
 - ignore: {name: Use + directly}
 - ignore: {name: Use fmap}

cabal.project

@@ -20,15 +20,9 @@ packages:
     cardano-api
     cardano-api-gen
     cardano-wasm
-    -- TODO fix potential issues with build-type: Custom and protoc and reenable
-    -- When uncommenting this line, remember to reenable in .github/workflows/hls.yml
-    -- by removing the line that moves it away before calling `haskell-language-server`.
-    -- cardano-rpc
+    cardano-rpc
 
-extra-packages: Cabal, process
-
-if impl(ghc < 9.8)
-  constraints: interpolatedstring-perl6:setup.time source
+extra-packages: Cabal
 
 -- It may slow down build plan preparation, but without it cabal has problems
 -- with solving constraints. Remove this when not needed anymore.

cardano-api/src/Cardano/Api/HasTypeProxy.hs

@@ -13,10 +13,12 @@ module Cardano.Api.HasTypeProxy
 where
 
 import Data.ByteString qualified as BS
+import Data.ByteString.Lazy qualified as BSL
 import Data.Kind (Constraint, Type)
 import Data.Proxy (Proxy (..))
 import Data.Typeable (Typeable)
 import Data.Word (Word16, Word8)
+import Numeric.Natural (Natural)
 
 class Typeable t => HasTypeProxy t where
   -- | A family of singleton types used in this API to indicate which type to
@@ -35,10 +37,18 @@ instance HasTypeProxy Word16 where
   data AsType Word16 = AsWord16
   proxyToAsType _ = AsWord16
 
+instance HasTypeProxy Natural where
+  data AsType Natural = AsNatural
+  proxyToAsType _ = AsNatural
+
 instance HasTypeProxy BS.ByteString where
   data AsType BS.ByteString = AsByteString
   proxyToAsType _ = AsByteString
 
+instance HasTypeProxy BSL.ByteString where
+  data AsType BSL.ByteString = AsByteStringLazy
+  proxyToAsType _ = AsByteStringLazy
+
 data FromSomeType (c :: Type -> Constraint) b where
   FromSomeType :: c a => AsType a -> (a -> b) -> FromSomeType c b
 

cardano-api/src/Cardano/Api/Serialise/Raw.hs

@@ -26,13 +26,16 @@ import Data.Bits (Bits (..))
 import Data.ByteString qualified as BS
 import Data.ByteString.Base16 qualified as Base16
 import Data.ByteString.Builder qualified as BSB
-import Data.ByteString.Char8 as BSC
+import Data.ByteString.Char8 (ByteString)
+import Data.ByteString.Char8 qualified as BSC
+import Data.ByteString.Lazy qualified as BSL
 import Data.Data (typeRep)
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Data.Text.Encoding qualified as Text
 import Data.Typeable (TypeRep, Typeable)
 import Data.Word (Word16, Word8)
+import Numeric.Natural (Natural)
 
 class (HasTypeProxy a, Typeable a) => SerialiseAsRawBytes a where
   serialiseToRawBytes :: a -> ByteString
@@ -60,10 +63,23 @@ instance SerialiseAsRawBytes Word16 where
       throwError . SerialiseAsRawBytesError $
         "Cannot decode Word16 from (hex): " <> BSC.unpack (Base16.encode bs)
 
+instance SerialiseAsRawBytes Natural where
+  serialiseToRawBytes 0 = BS.singleton 0x00
+  serialiseToRawBytes n = BS.toStrict . BSB.toLazyByteString $ go n mempty
+   where
+    go 0 acc = acc
+    go x acc = go (x `shiftR` 8) (BSB.word8 (fromIntegral (x .&. 0xFF)) <> acc)
+  deserialiseFromRawBytes AsNatural "\x00" = pure 0
+  deserialiseFromRawBytes AsNatural input = pure . Prelude.foldl' (\acc byte -> acc `shiftL` 8 .|. fromIntegral byte) 0 $ BS.unpack input
+
 instance SerialiseAsRawBytes BS.ByteString where
   serialiseToRawBytes = id
   deserialiseFromRawBytes AsByteString = pure
 
+instance SerialiseAsRawBytes BSL.ByteString where
+  serialiseToRawBytes = BSL.toStrict
+  deserialiseFromRawBytes AsByteStringLazy = pure . BSL.fromStrict
+
 serialiseToRawBytesHex :: SerialiseAsRawBytes a => a -> ByteString
 serialiseToRawBytesHex = Base16.encode . serialiseToRawBytes
 

cardano-api/src/Cardano/Api/Serialise/SerialiseUsing.hs

@@ -17,8 +17,10 @@ import Cardano.Api.Serialise.Json
 import Cardano.Api.Serialise.Raw
 
 import Data.Aeson.Types qualified as Aeson
+import Data.ByteString qualified as B
 import Data.Text.Encoding qualified as Text
 import Data.Typeable (tyConName, typeRep, typeRepTyCon)
+import Numeric (showBin)
 
 -- | For use with @deriving via@, to provide 'ToCBOR' and 'FromCBOR' instances,
 -- based on the 'SerialiseAsRawBytes' instance.
@@ -39,6 +41,10 @@ instance SerialiseAsRawBytes a => FromCBOR (UsingRawBytes a) where
     ttoken = proxyToAsType (Proxy :: Proxy a)
     tname = (tyConName . typeRepTyCon . typeRep) (Proxy :: Proxy a)
 
+-- | Prints the representation in binary format, quoted
+instance SerialiseAsRawBytes a => Show (UsingRawBytes a) where
+  showsPrec _ (UsingRawBytes x) = showChar '"' . mconcat (map showBin . B.unpack $ serialiseToRawBytes x) . showChar '"'
+
 -- | For use with @deriving via@, to provide instances for any\/all of 'Show',
 -- 'ToJSON', 'FromJSON', 'ToJSONKey', FromJSONKey' using a hex
 -- encoding, based on the 'SerialiseAsRawBytes' instance.

cardano-rpc/README.md

@@ -2,7 +2,7 @@
 
 ## What's this
 
-The `cardano-rpc` package provides client and server haskell modules for gRPC interface of `cardano-node`. 
+The `cardano-rpc` package provides client and server haskell modules for gRPC interface of `cardano-node`.
 It implements [UTxO RPC](https://utxorpc.org/introduction) protobuf communication protocol specification.
 
 ## Building
@@ -14,7 +14,27 @@ You need the following dependencies installed on your system:
 - [`snappy`](https://github.com/google/snappy) development files (`libsnappy-dev` in Ubuntu)
 - [`protobuf`](https://developers.google.com/protocol-buffers/) compiler (`protobuf-compiler` in Ubuntu)
 
-Then do:
+### Generating the Haskell code from proto definitions
+
+You need to install `buf` and `proto-lens-protoc`.
+1. Follow the `buf` installation guide at: https://buf.build/docs/cli/installation/
+1. To install Haskell protobuf code compiler:
+```bash
+cabal install proto-lens-protoc
+```
+
+Then you can generate Haskell code using:
+```bash
+( cd cardano-rpc/ ; buf generate proto )
+```
+
+
+This will output the Haskell code into `cardano-rpc/
+
+### Building the haskell code
+
+To build the package use the following command:
+
 ```bash
 cabal build cardano-rpc
 ```

cardano-rpc/buf.gen.yaml

@@ -0,0 +1,26 @@
+version: v1
+
+managed:
+  enabled: true
+
+plugins:
+  # node
+  # - plugin: es
+  #   out: gen/node/src
+  #   opt:
+  #     - target=ts
+  #
+  # - plugin: connect-es
+  #   out: gen/node/src
+  #   opt:
+  #     - target=ts
+  #
+  # - plugin: buf.build/bufbuild/protoschema-jsonschema
+  #   out: gen/jsonschema/schema
+
+  # haskell
+
+  - plugin: haskell-protolens
+    path: proto-lens-protoc
+    out: gen
+

cardano-rpc/cardano-rpc.cabal

@@ -17,26 +17,17 @@ license-files:
   LICENSE
   NOTICE
 
-build-type: Custom
+build-type: Simple
 extra-doc-files:
   CHANGELOG.md
   README.md
 
-extra-source-files: proto/**/*.proto
-
-custom-setup
-  setup-depends:
-    Cabal >=3.0 && <3.13,
-    base >=4.14 && <5,
-    proto-lens-setup >=0.4 && <0.5,
-
 common project-config
   default-language: Haskell2010
   default-extensions:
     ImportQualifiedPost
     OverloadedStrings
 
-  build-depends: base >=4.14 && <5
   ghc-options:
     -Wall
     -Wcompat
@@ -50,7 +41,6 @@ common project-config
 library
   import: project-config
   hs-source-dirs: src
-  build-tool-depends: proto-lens-protoc:proto-lens-protoc
   exposed-modules:
     Cardano.Rpc.Client
     Cardano.Rpc.Proto.Api.Node
@@ -65,29 +55,13 @@ library
     Cardano.Rpc.Server.Internal.UtxoRpc.Query
     Cardano.Rpc.Server.Internal.UtxoRpc.Submit
     Cardano.Rpc.Server.Internal.UtxoRpc.Type
-    Proto.Cardano.Rpc.Node
-    Proto.Cardano.Rpc.Node_Fields
-    Proto.Utxorpc.V1alpha.Cardano.Cardano
-    Proto.Utxorpc.V1alpha.Cardano.Cardano_Fields
-    Proto.Utxorpc.V1alpha.Query.Query
-    Proto.Utxorpc.V1alpha.Query.Query_Fields
-    Proto.Utxorpc.V1alpha.Submit.Submit
-    Proto.Utxorpc.V1alpha.Submit.Submit_Fields
 
   other-modules:
     Cardano.Rpc.Server.Internal.Orphans
     Paths_cardano_rpc
 
   autogen-modules:
     Paths_cardano_rpc
-    Proto.Cardano.Rpc.Node
-    Proto.Cardano.Rpc.Node_Fields
-    Proto.Utxorpc.V1alpha.Cardano.Cardano
-    Proto.Utxorpc.V1alpha.Cardano.Cardano_Fields
-    Proto.Utxorpc.V1alpha.Query.Query
-    Proto.Utxorpc.V1alpha.Query.Query_Fields
-    Proto.Utxorpc.V1alpha.Submit.Submit
-    Proto.Utxorpc.V1alpha.Submit.Submit_Fields
 
   build-depends:
     aeson,
@@ -98,6 +72,7 @@ library
     cardano-ledger-binary,
     cardano-ledger-conway,
     cardano-ledger-core,
+    cardano-rpc:gen,
     containers,
     contra-tracer,
     data-default,
@@ -107,24 +82,44 @@ library
     grpc-spec,
     proto-lens,
     proto-lens-protobuf-types,
-    proto-lens-runtime,
     rio,
     text,
 
+-- this should be replaced by utxorpc pacakge from hackage
+-- ideally we should upstream whatever is implemented in Proto.Cardano.Rpc.Node
+-- into utxorpc
+library gen
+  import: project-config
+  hs-source-dirs: gen
+  exposed-modules:
+    Proto.Cardano.Rpc.Node
+    Proto.Cardano.Rpc.Node_Fields
+    Proto.Utxorpc.V1alpha.Cardano.Cardano
+    Proto.Utxorpc.V1alpha.Cardano.Cardano_Fields
+    Proto.Utxorpc.V1alpha.Query.Query
+    Proto.Utxorpc.V1alpha.Query.Query_Fields
+    Proto.Utxorpc.V1alpha.Submit.Submit
+    Proto.Utxorpc.V1alpha.Submit.Submit_Fields
+
+  build-depends:
+    proto-lens-protobuf-types,
+    proto-lens-runtime,
+
 test-suite cardano-rpc-test
   import: project-config
   hs-source-dirs: test/cardano-rpc-test
   main-is: cardano-rpc-test.hs
   type: exitcode-stdio-1.0
   build-depends:
+    base,
     cardano-api,
     cardano-api:gen,
     cardano-ledger-api,
     cardano-ledger-conway,
     cardano-ledger-core,
     cardano-rpc,
     containers,
-    hedgehog >=1.1,
+    hedgehog,
     rio,
     tasty,
     tasty-hedgehog,