diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a06a60cb2..ae62a88e5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,8 +31,21 @@ jobs: cache-dependency-path: "**/*.sum" - run: make install-tools - run: make install-fabric-bins + - run: make unit-tests + + utest-postgres: + needs: + - checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + cache-dependency-path: "**/*.sum" + - run: make install-tools - run: make testing-docker-images - - run: make unit-tests-race + - run: make unit-tests-postgres itest-list: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index bfb38d540..cbeab9a13 100755 --- a/Makefile +++ b/Makefile @@ -108,20 +108,27 @@ testing-docker-images: ## Pull images for system testing # include the checks target include $(TOP)/checks.mk +GO_PACKAGES = $$(go list ./... | grep -Ev '/(integration/|mock|fake)'; go list ./integration/nwo) +GO_PACKAGES_SDK = $$(go list ./... | grep '/sdk/dig$$') +GO_TEST_PARAMS ?= -race -cover + .PHONY: unit-tests unit-tests: ## Run unit tests - @export FAB_BINS=$(FAB_BINS); go test -cover $(shell go list ./... | grep -v '/integration/') - cd integration/nwo/; go test -cover ./... - -.PHONY: unit-tests-race -unit-tests-race: ## Run unit tests with race - @export GORACE=history_size=7; export FAB_BINS=$(FAB_BINS); go test -race -cover $(shell go list ./... | grep -v '/integration/') - cd integration/nwo/; export FAB_BINS=$(FAB_BINS); go test -race -cover ./... - -.PHONY: unit-tests-dig -unit-tests-dig: - cd platform/view/sdk/dig; go test -cover ./... - cd platform/fabric/sdk/dig; go test -cover ./... + @echo "Running unit tests..." + export FABRIC_LOGGING_SPEC=error; \ + export FAB_BINS=$(FAB_BINS); \ + go test $(GO_TEST_PARAMS) --skip '(Postgres)' $(GO_PACKAGES) + +.PHONY: unit-tests-postgres +unit-tests-postgres: ## Run unit tests for postgres (requires container images as defined in testing-docker-images) + @echo "Running unit tests..." + export FABRIC_LOGGING_SPEC=error; \ + go test $(GO_TEST_PARAMS) --run '(Postgres)' $(GO_PACKAGES) + +.PHONY: unit-tests-sdk +unit-tests-sdk: ## Run sdk wiring tests + @echo "Running SDK tests..." + go test $(GO_TEST_PARAMS) --run "(TestWiring)" $(GO_PACKAGES_SDK) run-otlp: cd platform/view/services/tracing; docker-compose up -d diff --git a/go.mod b/go.mod index c2692f4b8..cbc07e0a3 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/IBM/idemix/bccsp/types v0.0.0-20240913182345-72941a5f41cd github.com/IBM/mathlib v0.0.3-0.20231011094432-44ee0eb539da github.com/cockroachdb/errors v1.12.0 - github.com/docker/docker v28.0.0+incompatible - github.com/docker/go-connections v0.5.0 + github.com/docker/docker v28.3.3+incompatible + github.com/docker/go-connections v0.6.0 github.com/fsouza/go-dockerclient v1.12.0 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 @@ -101,6 +101,8 @@ require ( github.com/consensys/bavard v0.1.22 // indirect github.com/consensys/gnark-crypto v0.14.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -170,7 +172,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -198,9 +200,11 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect - github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -219,7 +223,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -291,7 +295,7 @@ require ( golang.org/x/crypto v0.40.0 // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.36.0 // indirect golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.34.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect diff --git a/go.sum b/go.sum index dba44127b..a1e4a7e82 100644 --- a/go.sum +++ b/go.sum @@ -624,8 +624,8 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -745,6 +745,10 @@ github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCe github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -769,10 +773,10 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnN github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= -github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -1164,8 +1168,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -1262,12 +1266,16 @@ github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqky github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= -github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -1355,8 +1363,8 @@ github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1985,8 +1993,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2429,8 +2437,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/platform/view/services/storage/driver/sql/common/bench.go b/platform/view/services/storage/driver/sql/common/bench.go index 14db2d761..a2128976a 100644 --- a/platform/view/services/storage/driver/sql/common/bench.go +++ b/platform/view/services/storage/driver/sql/common/bench.go @@ -16,6 +16,11 @@ import ( "github.com/stretchr/testify/assert" ) +const ( + namespace = "ns" + key = "key" +) + var ( returnValue []byte returnErr error diff --git a/platform/view/services/storage/driver/sql/common/helpers.go b/platform/view/services/storage/driver/sql/common/helpers.go index 014182726..b45cb725c 100644 --- a/platform/view/services/storage/driver/sql/common/helpers.go +++ b/platform/view/services/storage/driver/sql/common/helpers.go @@ -9,20 +9,16 @@ package common import ( "context" "database/sql" - "encoding/binary" "fmt" - "strings" "sync" "testing" "time" "unicode/utf8" "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" - driver3 "github.com/hyperledger-labs/fabric-smart-client/platform/common/driver" "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections" "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections/iterators" "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver" - "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/keys" "github.com/stretchr/testify/assert" ) @@ -39,7 +35,7 @@ var Cases = []struct { {"DB2", TTestDB2}, {"RangeQueries1", TTestRangeQueries1}, {"MultiWritesAndRangeQueries", TTestMultiWritesAndRangeQueries}, - {"TTestMultiWrites", TTestMultiWrites}, + {"MultiWrites", TTestMultiWrites}, {"CompositeKeys", TTestCompositeKeys}, } @@ -87,7 +83,14 @@ func TTestDuplicate(t *testing.T, _ *sql.DB, writeDB WriteDB, errorWrapper drive func TTestRangeQueries(t *testing.T, db driver.KeyValueStore) { ns := "namespace" - populateForRangeQueries(t, db, ns) + entries := map[string]driver.UnversionedValue{ + "k1": driver.UnversionedValue("k1_value"), + "k111": driver.UnversionedValue("k111_value"), + "k2": driver.UnversionedValue("k2_value"), + "k3": driver.UnversionedValue("k3_value"), + } + populateDB(t, db, ns, entries) + defer cleanupDB(t, db, ns) itr, err := db.GetStateRangeScanIterator(context.Background(), ns, "", "") assert.NoError(t, err) @@ -153,11 +156,12 @@ func TTestRangeQueries(t *testing.T, db driver.KeyValueStore) { func TTestSimpleReadWrite(t *testing.T, db driver.KeyValueStore) { ns := "ns" key := "key" + defer cleanupDB(t, db, ns) // empty state vv, err := db.GetState(context.Background(), ns, key) assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue{}, vv) + assert.Nil(t, vv) // add data err = db.SetState(context.Background(), ns, key, driver.UnversionedValue("val")) @@ -206,54 +210,45 @@ func TTestSimpleReadWrite(t *testing.T, db driver.KeyValueStore) { // expect state to be empty vv, err = db.GetState(context.Background(), ns, key) assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue{}, vv) + assert.Nil(t, vv) } -func populateDB(t *testing.T, db driver.KeyValueStore, ns, key, keyWithSuffix string) { - err := db.BeginUpdate() - assert.NoError(t, err) - - err = db.SetState(context.Background(), ns, key, driver.UnversionedValue("bar")) - assert.NoError(t, err) - - err = db.SetState(context.Background(), ns, keyWithSuffix, driver.UnversionedValue("bar1")) - assert.NoError(t, err) - - err = db.Commit() - assert.NoError(t, err) - - vv, err := db.GetState(context.Background(), ns, key) - assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue("bar"), vv) +func populateDB(t *testing.T, db driver.KeyValueStore, ns string, entries map[string]driver.UnversionedValue) { + assert.NoError(t, db.BeginUpdate()) + assert.Empty(t, db.SetStates(t.Context(), ns, entries)) + assert.NoError(t, db.Commit()) - vv, err = db.GetState(context.Background(), ns, keyWithSuffix) + // let's check that we only have + itr, err := db.GetStateRangeScanIterator(t.Context(), ns, "", "") assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue("bar1"), vv) - vv, err = db.GetState(context.Background(), ns, "barf") + res, err := collections.ReadAll(itr) assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue{}, vv) + assert.Equal(t, len(entries), len(res)) - vv, err = db.GetState(context.Background(), "barf", "barf") - assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue{}, vv) + for _, r := range res { + v, ok := entries[r.Key] + assert.True(t, ok) + assert.Equal(t, v, r.Raw) + } } -func populateForRangeQueries(t *testing.T, db driver.KeyValueStore, ns string) { - err := db.BeginUpdate() +func cleanupDB(t *testing.T, db driver.KeyValueStore, ns string) { + assert.NoError(t, db.BeginUpdate()) + itr, err := db.GetStateRangeScanIterator(t.Context(), ns, "", "") assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k2", driver.UnversionedValue("k2_value")) - assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k3", driver.UnversionedValue("k3_value")) - assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k1", driver.UnversionedValue("k1_value")) - assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k111", driver.UnversionedValue("k111_value")) + res, err := collections.ReadAll(itr) assert.NoError(t, err) - err = db.Commit() - assert.NoError(t, err) + var keys []string + for _, r := range res { + keys = append(keys, r.Key) + } + + errs := db.DeleteStates(t.Context(), ns, keys...) + assert.Empty(t, errs) + assert.NoError(t, db.Commit()) } func TTestGetNonExistent(t *testing.T, db driver.KeyValueStore) { @@ -262,7 +257,7 @@ func TTestGetNonExistent(t *testing.T, db driver.KeyValueStore) { vv, err := db.GetState(context.Background(), ns, key) assert.NoError(t, err) - assert.Equal(t, driver.UnversionedValue{}, vv) + assert.Nil(t, vv) } func TTestDB1(t *testing.T, db driver.KeyValueStore) { @@ -270,7 +265,11 @@ func TTestDB1(t *testing.T, db driver.KeyValueStore) { key := "foo" keyWithSuffix := key + "/suffix" - populateDB(t, db, ns, key, keyWithSuffix) + entries := map[string]driver.UnversionedValue{ + key: driver.UnversionedValue("bar"), + keyWithSuffix: driver.UnversionedValue("bar1"), + } + populateDB(t, db, ns, entries) err := db.BeginUpdate() assert.NoError(t, err) @@ -290,7 +289,11 @@ func TTestDB2(t *testing.T, db driver.KeyValueStore) { key := "foo" keyWithSuffix := key + "/suffix" - populateDB(t, db, ns, key, keyWithSuffix) + entries := map[string]driver.UnversionedValue{ + key: driver.UnversionedValue("bar"), + keyWithSuffix: driver.UnversionedValue("bar1"), + } + populateDB(t, db, ns, entries) err := db.BeginUpdate() assert.NoError(t, err) @@ -307,21 +310,14 @@ func TTestDB2(t *testing.T, db driver.KeyValueStore) { func TTestRangeQueries1(t *testing.T, db driver.KeyValueStore) { ns := "namespace" - - err := db.BeginUpdate() - assert.NoError(t, err) - - err = db.SetState(context.Background(), ns, "k2", driver.UnversionedValue("k2_value")) - assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k3", driver.UnversionedValue("k3_value")) - assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k1", driver.UnversionedValue("k1_value")) - assert.NoError(t, err) - err = db.SetState(context.Background(), ns, "k111", driver.UnversionedValue("k111_value")) - assert.NoError(t, err) - - err = db.Commit() - assert.NoError(t, err) + entries := map[string]driver.UnversionedValue{ + "k2": driver.UnversionedValue("k2_value"), + "k3": driver.UnversionedValue("k3_value"), + "k1": driver.UnversionedValue("k1_value"), + "k111": driver.UnversionedValue("k111_value"), + } + populateDB(t, db, ns, entries) + defer cleanupDB(t, db, ns) itr, err := db.GetStateRangeScanIterator(context.Background(), ns, "", "") assert.NoError(t, err) @@ -349,33 +345,23 @@ func TTestRangeQueries1(t *testing.T, db driver.KeyValueStore) { func TTestMultiWritesAndRangeQueries(t *testing.T, db driver.KeyValueStore) { ns := "namespace" - assert.NoError(t, db.BeginUpdate()) - - assert.NoError(t, db.SetState(context.Background(), ns, "k2", driver.UnversionedValue("k2_value"))) - assert.NoError(t, db.SetState(context.Background(), ns, "k3", driver.UnversionedValue("k3_value"))) - assert.NoError(t, db.SetState(context.Background(), ns, "k1", driver.UnversionedValue("k1_value"))) - assert.NoError(t, db.SetState(context.Background(), ns, "k111", driver.UnversionedValue("k111_value"))) - - assert.NoError(t, db.Commit()) + entries := map[string]driver.UnversionedValue{ + "k1": driver.UnversionedValue("k1_value"), + "k2": driver.UnversionedValue("k2_value"), + "k3": driver.UnversionedValue("k3_value"), + "k111": driver.UnversionedValue("k111_value"), + } + populateDB(t, db, ns, entries) + defer cleanupDB(t, db, ns) var wg sync.WaitGroup - wg.Add(4) - go func() { - assert.NoError(t, db.SetState(context.Background(), ns, key, []byte("k2_value"))) - wg.Done() - }() - go func() { - assert.NoError(t, db.SetState(context.Background(), ns, key, []byte("k3_value"))) - wg.Done() - }() - go func() { - assert.NoError(t, db.SetState(context.Background(), ns, key, []byte("k1_value"))) - wg.Done() - }() - go func() { - assert.NoError(t, db.SetState(context.Background(), ns, key, []byte("k111_value"))) - wg.Done() - }() + for k, v := range entries { + wg.Add(1) + go func(k string, v driver.UnversionedValue) { + defer wg.Done() + assert.NoError(t, db.SetState(context.Background(), ns, k, v)) + }(k, v) + } wg.Wait() itr, err := db.GetStateRangeScanIterator(context.Background(), ns, "", "") @@ -428,13 +414,15 @@ func TTestMultiWritesAndRangeQueries(t *testing.T, db driver.KeyValueStore) { func TTestMultiWrites(t *testing.T, db driver.KeyValueStore) { ns := "namespace" + key := "test_key" + defer cleanupDB(t, db, ns) + var wg sync.WaitGroup - n := 20 - wg.Add(n) - for i := 0; i < n; i++ { + for i := 0; i < 20; i++ { + wg.Add(1) go func(i int) { + defer wg.Done() assert.NoError(t, db.SetState(context.Background(), ns, key, []byte(fmt.Sprintf("TTestMultiWrites_value_%d", i)))) - wg.Done() }(i) } wg.Wait() @@ -476,6 +464,7 @@ func createCompositeKey(objectType string, attributes []string) (string, error) func TTestCompositeKeys(t *testing.T, db driver.KeyValueStore) { ns := "namespace" keyPrefix := "prefix" + defer cleanupDB(t, db, ns) err := db.BeginUpdate() assert.NoError(t, err) @@ -537,6 +526,8 @@ func TTestCompositeKeys(t *testing.T, db driver.KeyValueStore) { // cannot check if key exists: pq: invalid byte sequence for encoding "UTF8": 0xc2 0x32] func TTestNonUTF8keys(t *testing.T, db driver.KeyValueStore) { ns := "namespace" + key := "test_key" + defer cleanupDB(t, db, ns) // adapted from https://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805 utf8 := map[string][]byte{ @@ -589,18 +580,14 @@ func TTestNonUTF8keys(t *testing.T, db driver.KeyValueStore) { v, err := db.GetState(context.Background(), ns, key) assert.NoError(t, err) - assert.Equal(t, []byte(nil), v) + assert.Nil(t, v) } -var ( - namespace = "test_namespace" - key = "test_key" -) - func TTestUnversionedRange(t *testing.T, db driver.KeyValueStore) { var err error ns := "namespace" + defer cleanupDB(t, db, ns) err = db.BeginUpdate() assert.NoError(t, err) @@ -661,6 +648,7 @@ func TTestUnversionedRange(t *testing.T, db driver.KeyValueStore) { func TTestUnversionedSimple(t *testing.T, db driver.KeyValueStore) { ns := "ns" key := "key" + defer cleanupDB(t, db, ns) v, err := db.GetState(context.Background(), ns, key) assert.NoError(t, err) @@ -719,26 +707,6 @@ func TTestUnversionedSimple(t *testing.T, db driver.KeyValueStore) { assert.Equal(t, []byte(nil), v) } -func BenchmarkConcatenation(b *testing.B) { - var s string - for i := 0; i < b.N; i++ { - s = namespace + keys.NamespaceSeparator + key - } - _ = s -} - -func BenchmarkBuilder(b *testing.B) { - var s string - for i := 0; i < b.N; i++ { - var sb strings.Builder - sb.WriteString(namespace) - sb.WriteString(keys.NamespaceSeparator) - sb.WriteString(key) - s = sb.String() - } - _ = s -} - type notifyEvent struct { Op opType NS string @@ -767,6 +735,7 @@ func TTestUnversionedNotifierSimple(t *testing.T, db driver.UnversionedNotifier) ns := "ns" key := "key" + defer cleanupDB(t, db, ns) v, err := db.GetState(context.Background(), ns, key) assert.NoError(t, err) @@ -861,22 +830,3 @@ func subscribe(db notifier) (chan notifyEvent, error) { time.Sleep(1 * time.Second) // Wait until subscription is complete before inserting values return ch, nil } - -func ToBytes(Block driver3.BlockNum, TxNum driver3.TxNum) []byte { - buf := make([]byte, 8) - binary.BigEndian.PutUint32(buf[:4], uint32(Block)) - binary.BigEndian.PutUint32(buf[4:], uint32(TxNum)) - return buf -} - -func FromBytes(data driver3.RawVersion) (driver3.BlockNum, driver3.TxNum, error) { - if len(data) == 0 { - return 0, 0, nil - } - if len(data) != 8 { - return 0, 0, errors.Errorf("block number must be 8 bytes, but got %d", len(data)) - } - Block := driver3.BlockNum(binary.BigEndian.Uint32(data[:4])) - TxNum := driver3.TxNum(binary.BigEndian.Uint32(data[4:])) - return Block, TxNum, nil -} diff --git a/platform/view/services/storage/driver/sql/common/test_utils.go b/platform/view/services/storage/driver/sql/common/test_utils.go index d0a10c750..b29dd1174 100644 --- a/platform/view/services/storage/driver/sql/common/test_utils.go +++ b/platform/view/services/storage/driver/sql/common/test_utils.go @@ -19,7 +19,18 @@ type provider[V any] func(name string) (V, error) func TestCases(t *testing.T, unversionedProvider provider[driver.KeyValueStore], unversionedNotifierProvider provider[driver.UnversionedNotifier], - baseUnpacker func(p driver.KeyValueStore) *KeyValueStore) { + baseUnpacker func(p driver.KeyValueStore) *KeyValueStore, +) { + for _, c := range Cases { + un, err := unversionedProvider(c.Name) + if err != nil { + t.Fatal(err) + } + t.Run(c.Name, func(xt *testing.T) { + defer utils.IgnoreErrorFunc(un.Close) + c.Fn(xt, un) + }) + } for _, c := range UnversionedCases { un, err := unversionedProvider(c.Name) if err != nil { diff --git a/platform/view/services/storage/driver/sql/common/unversioned.go b/platform/view/services/storage/driver/sql/common/unversioned.go index 1cb50aadf..dc3ed9cda 100644 --- a/platform/view/services/storage/driver/sql/common/unversioned.go +++ b/platform/view/services/storage/driver/sql/common/unversioned.go @@ -54,7 +54,7 @@ func NewKeyValueStore(writeDB WriteDB, readDB *sql.DB, table string, errorWrappe func (db *KeyValueStore) GetStateRangeScanIterator(ctx context.Context, ns driver2.Namespace, startKey, endKey driver2.PKey) (iterators.Iterator[*driver.UnversionedRead], error) { query, params := q.Select().FieldsByName("pkey", "val"). From(q.Table(db.table)). - Where(cond2.And(cond2.Eq("ns", ns), cond2.BetweenStrings("pkey", startKey, endKey))). + Where(cond2.And(cond2.Eq("ns", ns), cond2.BetweenBytes("pkey", []byte(startKey), []byte(endKey)))). OrderBy(q.Asc(common2.FieldName("pkey"))). Format(db.ci) @@ -97,7 +97,11 @@ func (db *KeyValueStore) GetStateSetIterator(ctx context.Context, ns driver2.Nam } func HasKeys(ns driver2.Namespace, keys ...driver2.PKey) cond2.Condition { - return cond2.And(cond2.Eq("ns", ns), cond2.In("pkey", keys...)) + kbytes := make([][]byte, len(keys)) + for i, k := range keys { + kbytes[i] = []byte(k) + } + return cond2.And(cond2.Eq("ns", ns), cond2.In("pkey", kbytes...)) } func (db *KeyValueStore) Close() error { @@ -199,7 +203,7 @@ func (db *KeyValueStore) SetStatesWithTx(ctx context.Context, tx dbTransaction, func (db *KeyValueStore) upsertStatesWithTx(ctx context.Context, tx dbTransaction, ns driver2.Namespace, vals map[driver2.PKey]driver.UnversionedValue) map[driver2.PKey]error { rows := make([]common2.Tuple, 0, len(vals)) for pkey, val := range vals { - rows = append(rows, common2.Tuple{ns, pkey, val}) + rows = append(rows, common2.Tuple{ns, []byte(pkey), val}) } query, params := q.InsertInto(db.table). Fields("ns", "pkey", "val"). @@ -229,7 +233,7 @@ func (db *KeyValueStore) CreateSchema() error { return InitSchema(db.writeDB, fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s ( ns TEXT NOT NULL, - pkey TEXT NOT NULL, + pkey BYTEA NOT NULL, val BYTEA NOT NULL DEFAULT '', PRIMARY KEY (pkey, ns) );`, db.table)) diff --git a/platform/view/services/storage/driver/sql/common/unversioned_test.go b/platform/view/services/storage/driver/sql/common/unversioned_test.go index f318a966f..6bda74b67 100644 --- a/platform/view/services/storage/driver/sql/common/unversioned_test.go +++ b/platform/view/services/storage/driver/sql/common/unversioned_test.go @@ -19,6 +19,12 @@ import ( . "github.com/onsi/gomega" ) +var ( + ns = "namespace" + key = []byte("mykey") + value = []byte("myvalue") +) + type dummyErrorWrapper struct{} func (d *dummyErrorWrapper) WrapError(err error) error { @@ -31,16 +37,12 @@ func TestGetState(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - key := "mykey" - value := []byte("myvalue") - query := regexp.QuoteMeta("SELECT val FROM kv_table WHERE (ns = $1) AND (pkey = $2)") mock.ExpectQuery(query).WithArgs(ns, key). WillReturnRows(mock.NewRows([]string{"val"}).AddRow(value)) store := mockKeyValueStore(db, db) - result, err := store.GetState(context.Background(), ns, key) + result, err := store.GetState(context.Background(), ns, string(key)) Expect(mock.ExpectationsWereMet()).To(Succeed()) Expect(err).ToNot(HaveOccurred()) @@ -53,14 +55,11 @@ func TestGetState_QueryError(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - key := "mykey" - query := regexp.QuoteMeta("SELECT val FROM kv_table WHERE (ns = $1) AND (pkey = $2)") mock.ExpectQuery(query).WithArgs(ns, key).WillReturnError(sql.ErrConnDone) store := mockKeyValueStore(db, db) - _, err = store.GetState(context.Background(), ns, key) + _, err = store.GetState(context.Background(), ns, string(key)) Expect(mock.ExpectationsWereMet()).To(Succeed()) Expect(err).To(HaveOccurred()) @@ -72,15 +71,11 @@ func TestSetState(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - key := "mykey" - val := []byte("value") - query := regexp.QuoteMeta("INSERT INTO kv_table (ns, pkey, val) VALUES ($1, $2, $3)") - mock.ExpectExec(query).WithArgs(ns, key, val).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(query).WithArgs(ns, key, value).WillReturnResult(sqlmock.NewResult(1, 1)) store := mockKeyValueStore(db, db) - err = store.SetState(context.Background(), ns, key, val) + err = store.SetState(context.Background(), ns, string(key), value) Expect(mock.ExpectationsWereMet()).To(Succeed()) Expect(err).ToNot(HaveOccurred()) @@ -92,15 +87,11 @@ func TestSetState_ExecError(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - key := "mykey" - val := []byte("value") - query := regexp.QuoteMeta("INSERT INTO kv_table (ns, pkey, val) VALUES ($1, $2, $3)") - mock.ExpectExec(query).WithArgs(ns, key, val).WillReturnError(sql.ErrConnDone) + mock.ExpectExec(query).WithArgs(ns, key, value).WillReturnError(sql.ErrConnDone) store := mockKeyValueStore(db, db) - err = store.SetState(context.Background(), ns, key, val) + err = store.SetState(context.Background(), ns, string(key), value) Expect(mock.ExpectationsWereMet()).To(Succeed()) Expect(err).To(HaveOccurred()) @@ -112,14 +103,11 @@ func TestDeleteState(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - key := "mykey" - query := regexp.QuoteMeta("DELETE FROM kv_table WHERE (ns = $1) AND (pkey = $2)") mock.ExpectExec(query).WithArgs(ns, key).WillReturnResult(sqlmock.NewResult(1, 1)) store := mockKeyValueStore(db, db) - err = store.DeleteState(context.Background(), ns, key) + err = store.DeleteState(context.Background(), ns, string(key)) Expect(mock.ExpectationsWereMet()).To(Succeed()) Expect(err).ToNot(HaveOccurred()) @@ -131,14 +119,11 @@ func TestDeleteState_ExecError(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - key := "mykey" - query := regexp.QuoteMeta("DELETE FROM kv_table WHERE (ns = $1) AND (pkey = $2)") mock.ExpectExec(query).WithArgs(ns, key).WillReturnError(sql.ErrConnDone) store := mockKeyValueStore(db, db) - err = store.DeleteState(context.Background(), ns, key) + err = store.DeleteState(context.Background(), ns, string(key)) Expect(mock.ExpectationsWereMet()).To(Succeed()) Expect(err).To(HaveOccurred()) @@ -148,9 +133,8 @@ func TestGetStateRangeScanIterator(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" - startKey := "a" - endKey := "z" + startKey := []byte("a") + endKey := []byte("z") query := regexp.QuoteMeta("SELECT pkey, val FROM kv_table WHERE (ns = $1) AND ((pkey >= $2) AND (pkey < $3)) ORDER BY pkey ASC") mock.ExpectQuery(query). @@ -158,7 +142,7 @@ func TestGetStateRangeScanIterator(t *testing.T) { WillReturnRows(sqlmock.NewRows([]string{"pkey", "val"}).AddRow("a", []byte("val1")).AddRow("b", []byte("val2"))) store := mockKeyValueStore(db, db) - iter, err := store.GetStateRangeScanIterator(context.Background(), ns, startKey, endKey) + iter, err := store.GetStateRangeScanIterator(context.Background(), ns, string(startKey), string(endKey)) Expect(err).ToNot(HaveOccurred()) var results []string @@ -179,12 +163,11 @@ func TestGetStateSetIterator(t *testing.T) { db, mock, err := sqlmock.New() Expect(err).ToNot(HaveOccurred()) - ns := "namespace" keys := []string{"a", "b"} query := regexp.QuoteMeta("SELECT pkey, val FROM kv_table WHERE (ns = $1) AND ((pkey) IN (($2), ($3)))") mock.ExpectQuery(query). - WithArgs(ns, "a", "b"). + WithArgs(ns, []byte("a"), []byte("b")). WillReturnRows(sqlmock.NewRows([]string{"pkey", "val"}).AddRow("a", []byte("val1")).AddRow("b", []byte("val2"))) store := mockKeyValueStore(db, db) @@ -212,7 +195,7 @@ func TestExec(t *testing.T) { query := "UPDATE kv_table SET val = $1 WHERE ns = $2 AND pkey = $3" mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta(query)). - WithArgs([]byte("v"), "ns", "key"). + WithArgs([]byte("v"), "ns", []byte("key")). WillReturnResult(sqlmock.NewResult(1, 1)) tx, err := db.Begin() @@ -221,7 +204,7 @@ func TestExec(t *testing.T) { store := mockKeyValueStore(db, db) store.Txn = tx - _, err = store.Exec(context.Background(), query, []byte("v"), "ns", "key") + _, err = store.Exec(context.Background(), query, []byte("v"), "ns", []byte("key")) Expect(err).ToNot(HaveOccurred()) Expect(mock.ExpectationsWereMet()).To(Succeed()) } diff --git a/platform/view/services/storage/driver/sql/postgres/encode.go b/platform/view/services/storage/driver/sql/postgres/encode.go index 3525317e9..c172b0995 100644 --- a/platform/view/services/storage/driver/sql/postgres/encode.go +++ b/platform/view/services/storage/driver/sql/postgres/encode.go @@ -8,47 +8,21 @@ package postgres import ( "encoding/hex" - - "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections" - "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections/iterators" - driver2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver" + "strings" ) func identity(a string) (string, error) { return a, nil } -func decodeUnversionedReadIterator(it iterators.Iterator[*driver2.UnversionedRead], err error) (iterators.Iterator[*driver2.UnversionedRead], error) { - return decodeIterator(it, err, decodeUnversionedRead) -} - -func decodeIterator[R any](it iterators.Iterator[*R], err error, transformer func(v *R) (*R, error)) (iterators.Iterator[*R], error) { - if err != nil { - return nil, err +// decodeBYTEA decodes a postgres response of type BYTEA +func decodeBYTEA(s string) (string, error) { + // we only decode if we have indeed a BYTEA (returned as hex) + if !strings.HasPrefix(s, "\\x") { + return s, nil } - return collections.Map(it, transformer), nil -} -func decode(s string) (string, error) { - b, err := hex.DecodeString(s) + b, err := hex.DecodeString(s[2:]) if err != nil { return "", err } return string(b), err } - -func decodeUnversionedRead(v *driver2.UnversionedRead) (*driver2.UnversionedRead, error) { - if v == nil { - return nil, nil - } - key, err := decode(v.Key) - if err != nil { - return nil, err - } - return &driver2.UnversionedRead{ - Key: key, - Raw: v.Raw, - }, nil -} - -func encode(s string) string { - return hex.EncodeToString([]byte(s)) -} diff --git a/platform/view/services/storage/driver/sql/postgres/encode_test.go b/platform/view/services/storage/driver/sql/postgres/encode_test.go new file mode 100644 index 000000000..59e562b90 --- /dev/null +++ b/platform/view/services/storage/driver/sql/postgres/encode_test.go @@ -0,0 +1,87 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package postgres + +import ( + "fmt" + "testing" + + "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/kvs" + "github.com/stretchr/testify/require" +) + +var someCompositeKey = utils.MustGet(kvs.CreateCompositeKey("prefix", []string{"a", "b", "c"})) + +func TestDecodeBYTEA(t *testing.T) { + tests := []struct { + name string + input string + wantOutput string + expectError bool + }{ + { + name: "no hex returns unchanged", + input: "hello", + wantOutput: "hello", + }, + { + name: "decode valid hex", + input: "\\x68656c6c6f", // "hello" + wantOutput: "hello", + }, + { + name: "invalid hex returns error", + input: "\\xzzzz", + expectError: true, + }, + { + name: "prefix but empty hex", + input: "\\x", + wantOutput: "", // empty decode + }, + { + name: "composite key", + input: fmt.Sprintf("\\x%x", someCompositeKey), + wantOutput: someCompositeKey, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := decodeBYTEA(tc.input) + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.wantOutput, got) + }) + } +} + +func TestIdentity(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"basic ascii", "hello"}, + {"empty string", ""}, + {"unicode", "😀✓漢字"}, + {"whitespace", " spaced\t\n"}, + {"long string", string(make([]byte, 1024))}, + {"composite key", someCompositeKey}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := identity(tc.input) + require.NoError(t, err) + require.Equal(t, tc.input, got) + }) + } +} diff --git a/platform/view/services/storage/driver/sql/postgres/keyvalue.go b/platform/view/services/storage/driver/sql/postgres/keyvalue.go index b25414177..fb62366f0 100644 --- a/platform/view/services/storage/driver/sql/postgres/keyvalue.go +++ b/platform/view/services/storage/driver/sql/postgres/keyvalue.go @@ -7,70 +7,29 @@ SPDX-License-Identifier: Apache-2.0 package postgres import ( - "context" "database/sql" - "github.com/hyperledger-labs/fabric-smart-client/platform/common/driver" - "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections/iterators" - driver2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" common3 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" common4 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/common" - common2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/query/common" ) -type KeyValueStore struct { - *common4.KeyValueStore - - table string - ci common2.CondInterpreter - errorWrapper driver2.SQLErrorWrapper -} - -func (db *KeyValueStore) SetStates(ctx context.Context, ns driver.Namespace, kvs map[driver.PKey]driver.UnversionedValue) map[driver.PKey]error { - encoded := make(map[driver.PKey]driver.UnversionedValue, len(kvs)) - decodeMap := make(map[driver.PKey]driver.PKey, len(kvs)) - for k, v := range kvs { - enc := encode(k) - encoded[enc] = v - decodeMap[enc] = k - } - - errs := db.SetStatesWithTx(ctx, db.Txn, ns, encoded) - decodedErrs := make(map[driver.PKey]error, len(errs)) - for k, err := range errs { - decodedErrs[decodeMap[k]] = err - } - return decodedErrs -} - -func (db *KeyValueStore) SetStateWithTx(ctx context.Context, tx *sql.Tx, ns driver.Namespace, pkey driver.PKey, value driver.UnversionedValue) error { - if errs := db.SetStatesWithTx(ctx, tx, ns, map[driver.PKey]driver.UnversionedValue{encode(pkey): value}); errs != nil { - return errs[encode(pkey)] - } - return nil -} - -func (db *KeyValueStore) GetStateRangeScanIterator(ctx context.Context, ns driver.Namespace, startKey, endKey string) (iterators.Iterator[*driver.UnversionedRead], error) { - return decodeUnversionedReadIterator(db.KeyValueStore.GetStateRangeScanIterator(ctx, ns, encode(startKey), encode(endKey))) -} - -func (db *KeyValueStore) GetStateSetIterator(ctx context.Context, ns driver.Namespace, keys ...driver.PKey) (iterators.Iterator[*driver.UnversionedRead], error) { - encoded := make([]driver.PKey, len(keys)) - for i, k := range keys { - encoded[i] = encode(k) - } - return decodeUnversionedReadIterator(db.KeyValueStore.GetStateSetIterator(ctx, ns, encoded...)) -} - -func NewKeyValueStore(dbs *common3.RWDB, tables common4.TableNames) (*KeyValueStore, error) { +func NewKeyValueStore(dbs *common3.RWDB, tables common4.TableNames) (*common4.KeyValueStore, error) { return newKeyValueStore(dbs.ReadDB, dbs.WriteDB, tables.KVS), nil } type KeyValueStoreNotifier struct { - *KeyValueStore + *common4.KeyValueStore *Notifier } +func (db *KeyValueStoreNotifier) Close() error { + return errors.Join( + db.KeyValueStore.Close(), + db.Notifier.Close(), + ) +} + func (db *KeyValueStoreNotifier) CreateSchema() error { if err := db.KeyValueStore.CreateSchema(); err != nil { return err @@ -78,13 +37,9 @@ func (db *KeyValueStoreNotifier) CreateSchema() error { return db.Notifier.CreateSchema() } -func newKeyValueStore(readDB, writeDB *sql.DB, table string) *KeyValueStore { +func newKeyValueStore(readDB, writeDB *sql.DB, table string) *common4.KeyValueStore { ci := NewConditionInterpreter() errorWrapper := &ErrorMapper{} - return &KeyValueStore{ - KeyValueStore: common4.NewKeyValueStore(readDB, writeDB, table, errorWrapper, ci), - table: table, - ci: ci, - errorWrapper: errorWrapper, - } + + return common4.NewKeyValueStore(readDB, writeDB, table, errorWrapper, ci) } diff --git a/platform/view/services/storage/driver/sql/postgres/notifier.go b/platform/view/services/storage/driver/sql/postgres/notifier.go index 406584a79..453ee69eb 100644 --- a/platform/view/services/storage/driver/sql/postgres/notifier.go +++ b/platform/view/services/storage/driver/sql/postgres/notifier.go @@ -28,13 +28,25 @@ import ( var AllOperations = []driver.Operation{driver.Insert, driver.Update, driver.Delete} +// Notifier implements a simple subscription API to listen for updates on a database table. +// +// Deprecated: Notifier exists to track notification on tokens stored in postgres in the Token SDK. +// The Token SDK is the only user of this, thus, the code may be migrated. Notifier should not be used anymore. type Notifier struct { table string notifyOperations []driver.Operation writeDB *sql.DB listener *pgxlisten.Listener primaryKeys []primaryKey - once sync.Once + + startOnce sync.Once + closeOnce sync.Once + ctx context.Context + cancel context.CancelFunc + + // callback + subscribers []driver.TriggerCallback + mu sync.RWMutex } var operationMap = map[string]driver.Operation{ @@ -52,6 +64,10 @@ func NewSimplePrimaryKey(name driver.ColumnKey) *primaryKey { return &primaryKey{name: name, valueDecoder: identity} } +func NewBytePrimaryKey(name driver.ColumnKey) *primaryKey { + return &primaryKey{name: name, valueDecoder: decodeBYTEA} +} + const ( payloadConcatenator = "&" keySeparator = "_" @@ -59,10 +75,13 @@ const ( ) func NewNotifier(writeDB *sql.DB, table, dataSource string, notifyOperations []driver.Operation, primaryKeys ...primaryKey) *Notifier { - return &Notifier{ + ctx, cancel := context.WithCancel(context.Background()) + + n := &Notifier{ writeDB: writeDB, table: table, notifyOperations: notifyOperations, + primaryKeys: primaryKeys, listener: &pgxlisten.Listener{ Connect: func(ctx context.Context) (*pgx.Conn, error) { return pgx.Connect(ctx, dataSource) }, LogError: func(ctx context.Context, err error) { @@ -70,21 +89,58 @@ func NewNotifier(writeDB *sql.DB, table, dataSource string, notifyOperations []d }, ReconnectDelay: reconnectInterval, }, + ctx: ctx, + cancel: cancel, + } + + // attach handler that calls the subscribers + n.listener.Handle(table, ¬ificationHandler{ + table: table, primaryKeys: primaryKeys, + callback: n.dispatch, + }) + + return n +} + +func (db *Notifier) dispatch(operation driver.Operation, m map[driver.ColumnKey]string) { + db.mu.RLock() + defer db.mu.RUnlock() + for _, callback := range db.subscribers { + callback(operation, m) } } func (db *Notifier) Subscribe(callback driver.TriggerCallback) error { - db.once.Do(func() { + // register the callback + db.mu.Lock() + db.subscribers = append(db.subscribers, callback) + defer db.mu.Unlock() + + // Note that if the db listener is already closed, we still append subscribers here. + // Clearly, this is not very robust. A better implementation would check if the Notifier is still open, otherwise + // ignore the Subscribe call or return an error. + // Since we deprecated this impl, there is no need improve this behavior. + + db.startOnce.Do(func() { logger.Debugf("First subscription for notifier of [%s]. Notifier starts listening...", db.table) go func() { - if err := db.listener.Listen(context.TODO()); err != nil { + if err := db.listener.Listen(db.ctx); err != nil { logger.Errorf("notifier listen for [%s] failed: %s", db.table, err.Error()) } }() }) - db.listener.Handle(db.table, ¬ificationHandler{table: db.table, primaryKeys: db.primaryKeys, callback: callback}) + return nil +} + +func (db *Notifier) Close() error { + db.closeOnce.Do(func() { + db.cancel() // stop listener goroutine + db.mu.Lock() + db.subscribers = nil + db.mu.Unlock() + }) return nil } @@ -134,6 +190,12 @@ func (h *notificationHandler) HandleNotification(ctx context.Context, notificati func (db *Notifier) UnsubscribeAll() error { logger.Debugf("Unsubscribe called") + + // unregister all callbacks + db.mu.Lock() + defer db.mu.Unlock() + clear(db.subscribers) + return nil } diff --git a/platform/view/services/storage/driver/sql/postgres/sql_test.go b/platform/view/services/storage/driver/sql/postgres/sql_test.go index 768007e64..5bd5d44cf 100644 --- a/platform/view/services/storage/driver/sql/postgres/sql_test.go +++ b/platform/view/services/storage/driver/sql/postgres/sql_test.go @@ -7,7 +7,6 @@ SPDX-License-Identifier: Apache-2.0 package postgres import ( - "os" "testing" "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver" @@ -18,12 +17,6 @@ import ( ) func TestPostgres(t *testing.T) { - if os.Getenv("TEST_POSTGRES") != "true" { - t.Skip("set environment variable TEST_POSTGRES to true to include postgres test") - } - if testing.Short() { - t.Skip("skipping postgres test in short mode") - } t.Log("starting postgres") terminate, pgConnStr, err := StartPostgres(t, false) if err != nil { @@ -41,11 +34,11 @@ func TestPostgres(t *testing.T) { return NewPersistenceWithOpts(cp, NewDbProvider(), "", func(dbs *common.RWDB, tables common3.TableNames) (*KeyValueStoreNotifier, error) { return &KeyValueStoreNotifier{ KeyValueStore: newKeyValueStore(dbs.ReadDB, dbs.WriteDB, tables.KVS), - Notifier: NewNotifier(dbs.WriteDB, tables.KVS, pgConnStr, AllOperations, primaryKey{"ns", identity}, primaryKey{"pkey", decode}), + Notifier: NewNotifier(dbs.WriteDB, tables.KVS, pgConnStr, AllOperations, *NewSimplePrimaryKey("ns"), *NewBytePrimaryKey("pkey")), }, nil }) }, func(p driver.KeyValueStore) *common3.KeyValueStore { - return p.(*KeyValueStore).KeyValueStore + return p.(*common3.KeyValueStore) }) } diff --git a/platform/view/services/storage/driver/sql/postgres/test_utils.go b/platform/view/services/storage/driver/sql/postgres/test_utils.go index d3c028787..5b606dd09 100644 --- a/platform/view/services/storage/driver/sql/postgres/test_utils.go +++ b/platform/view/services/storage/driver/sql/postgres/test_utils.go @@ -128,7 +128,7 @@ func StartPostgresWithFmt(configs []*ContainerConfig) (func(), error) { logger := &fmtLogger{} for _, c := range configs { logger.Log("Starting DB ", c.DBName) - if closeFunc, err := startPostgresWithLogger(*c, logger, true); err != nil { + if closeFunc, err := startPostgresWithLogger(*c, logger, false); err != nil { errs = append(errs, err) } else { closeFuncs = append(closeFuncs, closeFunc) @@ -150,17 +150,16 @@ func startPostgresWithLogger(c ContainerConfig, t Logger, printLogs bool) (func( // images d, err := docker.GetInstance() if err != nil { - return nil, fmt.Errorf("can't get docker instance: %s", err) + return nil, fmt.Errorf("can't get docker instance: %w", err) } err = d.CheckImagesExist(c.Image) if err != nil { return nil, fmt.Errorf("image does not exist. Do: docker pull %s", c.Image) } - // start container cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { - return nil, fmt.Errorf("can't start postgres: %s", err) + return nil, fmt.Errorf("can't get docker client: %w", err) } ctx := context.Background() resp, err := cli.ContainerCreate(ctx, &container.Config{ @@ -192,15 +191,15 @@ func startPostgresWithLogger(c ContainerConfig, t Logger, printLogs bool) (func( }, }, nil, nil, c.Container) if err != nil { - return nil, fmt.Errorf("can't start postgres: %s", err) + return nil, fmt.Errorf("can't create postgres container: %w", err) } closeFunc := func() { t.Log("removing postgres container") - _ = cli.ContainerRemove(ctx, resp.ID, container.RemoveOptions{Force: true}) + _ = cli.ContainerRemove(ctx, resp.ID, container.RemoveOptions{RemoveVolumes: true, Force: true}) } if err := cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil { closeFunc() - return nil, fmt.Errorf("can't start postgres: %s", err) + return nil, fmt.Errorf("can't start postgres container: %w", err) } // Forward logs to test logger @@ -213,7 +212,7 @@ func startPostgresWithLogger(c ContainerConfig, t Logger, printLogs bool) (func( Timestamps: false, }) if err != nil { - t.Errorf("can't show logs: %s", err) + t.Errorf("can't show logs: %v", err) } defer utils.CloseMute(reader) @@ -228,7 +227,7 @@ func startPostgresWithLogger(c ContainerConfig, t Logger, printLogs bool) (func( inspect, err := cli.ContainerInspect(ctx, resp.ID) if err != nil { closeFunc() - return nil, fmt.Errorf("can't inspect postgres container: %s", err) + return nil, fmt.Errorf("can't inspect postgres container: %w", err) } if inspect.State.Health == nil { closeFunc() diff --git a/platform/view/services/storage/driver/sql/query/cond/cmp.go b/platform/view/services/storage/driver/sql/query/cond/cmp.go index d1330b9c3..1e50dd7b3 100644 --- a/platform/view/services/storage/driver/sql/query/cond/cmp.go +++ b/platform/view/services/storage/driver/sql/query/cond/cmp.go @@ -74,6 +74,17 @@ func FieldBetweenInts(f common.Serializable, start, end int) Condition { return fieldBetween(f, start, end, func(t int) bool { return t == NoIntLimit }) } +func BetweenBytes(f common.FieldName, start, end []byte) Condition { + var conds []Condition + if len(start) != 0 { + conds = append(conds, CmpVal(f, ">=", start)) + } + if len(end) != 0 { + conds = append(conds, CmpVal(f, "<", end)) + } + return And(conds...) +} + func BetweenStrings(f common.FieldName, start, end string) Condition { return FieldBetweenStrings(f, start, end) } diff --git a/platform/view/services/storage/driver/sql/query/cond/condition_test.go b/platform/view/services/storage/driver/sql/query/cond/condition_test.go index 46e8cc440..61faa056e 100644 --- a/platform/view/services/storage/driver/sql/query/cond/condition_test.go +++ b/platform/view/services/storage/driver/sql/query/cond/condition_test.go @@ -77,6 +77,11 @@ var testMatrix = []testCase{ expectedQuery: "field < NOW()", expectedParams: []common3.Param{}, }, + { + condition: cond2.BetweenBytes("pkey", []byte("start"), []byte("end")), + expectedQuery: "(pkey >= $0) AND (pkey < $1)", + expectedParams: []common3.Param{[]byte("start"), []byte("end")}, + }, } func TestConditions(t *testing.T) { diff --git a/platform/view/services/storage/kvs/kvs_test.go b/platform/view/services/storage/kvs/kvs_test.go index 149af6a09..89d251a23 100644 --- a/platform/view/services/storage/kvs/kvs_test.go +++ b/platform/view/services/storage/kvs/kvs_test.go @@ -10,7 +10,6 @@ import ( "context" "crypto/sha256" "fmt" - "os" "path" "sync" "testing" @@ -177,12 +176,10 @@ func TestSQLiteKVS(t *testing.T) { } func TestPostgresKVS(t *testing.T) { - if os.Getenv("TEST_POSTGRES") != "true" { - t.Skip("set environment variable TEST_POSTGRES to true to include postgres test") - } - if testing.Short() { - t.Skip("skipping postgres test in short mode") - } + // When running this test together with other tests; it may happen that a container instance is still running + // we give this test a slow start ... + time.Sleep(5 * time.Second) + t.Log("starting postgres") terminate, pgConnStr, err := postgres2.StartPostgres(t, false) if err != nil {