diff --git a/_run/minikube/.envrc b/_run/minikube/.envrc deleted file mode 100644 index 1186e1db2..000000000 --- a/_run/minikube/.envrc +++ /dev/null @@ -1,3 +0,0 @@ -source_up .envrc - -export AKASH_HOME=$DEVCACHE_RUN/minikube/.akash diff --git a/_run/minikube/Makefile b/_run/minikube/Makefile deleted file mode 100644 index b39836078..000000000 --- a/_run/minikube/Makefile +++ /dev/null @@ -1,51 +0,0 @@ -include ../common.mk -include ../common-commands.mk -include ../common-minikube.mk - -#SDL_PATH = grafana.yaml - -GATEWAY_HOSTNAME ?= localhost -GATEWAY_HOST ?= $(GATEWAY_HOSTNAME):8443 -GATEWAY_ENDPOINT ?= https://$(GATEWAY_HOST) - -.PHONY: provider-run -provider-run: - $(PROVIDER_SERVICES) run \ - --from "$(PROVIDER_KEY_NAME)" \ - --cluster-k8s \ - --gateway-listen-address "$(GATEWAY_HOST)" \ - --deployment-ingress-static-hosts true \ - --deployment-ingress-domain "$(GATEWAY_HOSTNAME)" \ - --cluster-node-port-quantity 100 \ - --cluster-public-hostname "$(GATEWAY_HOSTNAME)" \ - --bid-price-strategy "randomRange" \ - --deployment-runtime-class "none" - -.PHONY: provider-lease-status -provider-lease-status: - $(PROVIDER_SERVICES) lease-status \ - --dseq "$(DSEQ)" \ - --gseq "$(GSEQ)" \ - --oseq "$(OSEQ)" \ - --from "$(KEY_NAME)" \ - --provider "$(PROVIDER_ADDRESS)" - -.PHONY: provider-service-status -provider-service-status: - $(PROVIDER_SERVICES) lease-status \ - --dseq "$(DSEQ)" \ - --gseq "$(GSEQ)" \ - --oseq "$(OSEQ)" \ - --from "$(KEY_NAME)" \ - --provider "$(PROVIDER_ADDRESS)" - -.PHONY: provider-lease-ping -provider-lease-ping: - curl -sIH "Host: hello.localhost" localhost:$(KIND_HTTP_PORT) - -.PHONY: clean-$(AP_RUN_NAME) -clean-$(AP_RUN_NAME): - -.PHONY: hostname-operator-run -hostname-operator-run: - $(PROVIDER_SERVICES) provider hostname-operator diff --git a/_run/minikube/README.md b/_run/minikube/README.md deleted file mode 100644 index b9b7bebd4..000000000 --- a/_run/minikube/README.md +++ /dev/null @@ -1,284 +0,0 @@ -# Dev Environment: "Minikube" configuration - -The _Minikube_ dev environment builds: - -* A single-node blockchain network -* An Akash Provider Services Daemon (PSD) for bidding and running workloads. -* A Kubernetes cluster managed by **minikube** for the PSD to run workloads on. - -The [instructions](#runbook) below will illustrate how to run a network with a single, local node and execute workloads in [kind](https://kind.sigs.k8s.io/): - -* [Initialize blockchain node and client](#initialize) -* [Run a single-node network](#run-local-network) -* [Query objects on the network](#run-query) -* [Create a provider](#create-a-provider) -* [Run provider services](#run-provider-services) -* [Create a deployment](#create-a-deployment) -* [Bid on an order](#create-a-bid) -* [Terminate a lease](#terminate-lease) - -## Setup - -Four keys and accounts are created. The key names are: - -|Key Name|Use| -|---|---| -|`main`|Primary account (creating deployments, etc...)| -|`provider`|The provider account (bidding on orders, etc...)| -|`validator`|The sole validator for the created network| -|`other`|Misc. account to (receives tokens, etc...)| - -Most `make` commands are configurable and have defaults to make it -such that you don't need to override them for a simple pass-through of -this example. - -|Name|Default|Description| -|---|---|---| -|`KEY_NAME`|`main`|standard key name| -|`PROVIDER_KEY_NAME`|`provider`|name of key to use for provider| -|`DSEQ`|1|deployment sequence| -|`GSEQ`|1|group sequence| -|`OSEQ`|1|order sequence| -|`PRICE`|10uakt|price to bid| - -# Runbook - -The following steps will bring up a network and allow for interacting -with it. - -Running through the entire runbook requires three terminals. -Each command is marked __t1__-__t3__ to indicate a suggested terminal number. - -If at any time you'd like to start over with a fresh chain, simply run: - -__t1 run__ -```sh -make clean minikube-cluster-delete -make init -``` - -## Initialize - -Start and initialize minikube. - -If MINIKUBE_VM_DRIVER variable is empty script is trying to determine available hypervisors -### MacOS: - -### Linux - -```sh -make minikube-cluster-create -``` - -## Build Akash binaries and initialize network - -Initialize keys and accounts: - -### __t1 Step: 2__ -```sh -make init -``` - -## Run local network - -In a separate terminal, the following command will run the `akash` node: - -### __t2 Step: 3__ -```sh -make node-run -``` - -You can check the status of the network with: - -__t1 status__ -```sh -make node-status -``` - -You should see blocks being produced - the block height should be increasing. - -You can now view genesis accounts that were created: - -__t1 status__ -```sh -make query-accounts -``` - -## Create a provider - -Create a provider on the network with the following command: - -### __t1 Step: 4__ -```sh -make provider-create -``` - -View the on-chain representation of the provider with: - -__t1 status__ -```sh -make query-provider -``` - -## Run Provider - -To run Provider as a simple binary connecting to the cluster, in a third terminal, run the command: - -### __t3 Step: 5__ -```sh -make provider-run -``` - -Query the provider service gateway for its status: - -__t1 status__ -```sh -make provider-status -``` - -## Create a deployment - -Create a deployment from the `main` account with: - -### __t1 run Step: 6__ -```sh -make deployment-create -``` - -This particular deployment is created from the sdl file in this directory ([`deployment.yaml`](deployment.yaml)). - -Check that the deployment was created. Take note of the `dseq` - deployment sequence: - -__t1 status__ -```sh -make query-deployments -``` - -After a short time, you should see an order created for this deployment with the following command: - -```sh -make query-orders -``` - -The Provider Services Daemon should see this order and bid on it. - -```sh -make query-bids -``` - -When a bid has been created, you may create a lease: - - -### __t1 run Step: 7__ - -To create a lease, run - -```sh -make lease-create -``` - -You can see the lease with: - -```sh -make query-leases -``` - -You should now see "pending" inventory in the provider status: - -```sh -make provider-status -``` - -## Distribute Manifest - -Now that you have a lease with a provider, you need to send your -workload configuration to that provider by sending it the manifest: - -### __t1 Step: 8__ -```sh -make send-manifest -``` - -You can check the status of your deployment with: - -__t1 status__ -```sh -make provider-lease-status -``` - -You can reach your app with the following (Note: `Host:` header tomfoolery abound) - -__t1 status__ -```sh -make provider-lease-ping -``` - -Get service status - -__t1 service status__ -```sh -make provider-lease-status -``` - -Fetch logs from deployed service (all pods) - -__t1 service logs__ -```sh -make provider-lease-logs -``` - -If you chose to use port 80 when setting up kind, you can browse to your -deployed workload at http://hello.localhost - -## Update Deployment - -Updating active Deployments is a two step process. First edit the `deployment.yaml` with whatever changes are desired. Example; update the `image` field. - 1. Update the Akash Network to inform the Provider that a new Deployment declaration is expected. - * `make deployment-update` - 2. Send the updated manifest to the Provider to run. - * `make send-manifest` - -Between the first and second step, the prior deployment's containers will continue to run until the new manifest file is received, validated, and new container group operational. After health checks on updated group are passing; the prior containers will be terminated. - -#### Limitations - -Akash Groups are translated into Kubernetes Deployments, this means that only a few fields from the Akash SDL are mutable. For example `image`, `command`, `args`, `env` and exposed ports can be modified, but compute resources and placement criteria cannot. - -## Terminate lease - -There are a number of ways that a lease can be terminated. - -#### Provider closes the bid: - -__t1 teardown__ -```sh -make bid-close -``` - -#### Tenant closes the lease - -__t1 teardown__ -```sh -make lease-close -``` - -#### Tenant pauses the group - -__t1 teardown__ -```sh -make group-pause -``` - -#### Tenant closes the group - -__t1 teardown__ -```sh -make group-pause -``` - -#### Tenant closes the deployment - -__t1 teardown__ -```sh -make deployment-close -``` diff --git a/_run/minikube/deployment.yaml b/_run/minikube/deployment.yaml deleted file mode 100644 index eafb51e72..000000000 --- a/_run/minikube/deployment.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- -version: "2.0" - -services: - grafana: - image: grafana/grafana - expose: - - port: 3000 - as: 80 - to: - - global: true - accept: - - webdistest.localhost - params: - storage: - data: - mount: /var/lib/grafana -profiles: - compute: - grafana: - resources: - cpu: - units: 1 - memory: - size: 1Gi - storage: - - size: 512Mi - - name: data - size: 1Gi - attributes: - persistent: true - class: beta2 - placement: - westcoast: - attributes: - region: us-west - pricing: - grafana: - denom: uakt - amount: 1000 -deployment: - grafana: - westcoast: - profile: grafana - count: 1 diff --git a/_run/minikube/provider.yaml b/_run/minikube/provider.yaml deleted file mode 100644 index bb6447cbb..000000000 --- a/_run/minikube/provider.yaml +++ /dev/null @@ -1,12 +0,0 @@ -host: https://localhost:8443 -attributes: - - key: region - value: us-west - - key: capabilities/storage/1/persistent - value: true - - key: capabilities/storage/1/class - value: default - - key: capabilities/storage/2/persistent - value: true - - key: capabilities/storage/2/class - value: beta2 diff --git a/_run/single/.envrc b/_run/single/.envrc deleted file mode 100644 index e3c6d30ef..000000000 --- a/_run/single/.envrc +++ /dev/null @@ -1,3 +0,0 @@ -source_up .envrc - -export AKASH_HOME=$DEVCACHE_RUN/single/.akash diff --git a/_run/single/Makefile b/_run/single/Makefile deleted file mode 100644 index 733ee440c..000000000 --- a/_run/single/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -KUSTOMIZE_INSTALLS ?= \ - akash-node \ - akash-provider \ - akash-operator-hostname \ - akash-operator-inventory - -KUBE_UPLOAD_AKASH_IMAGE := true - -export AKASH_NODE = http://akash.localhost:$(KIND_PORT_BINDINGS) -export AP_NODE = $(AKASH_NODE) - -include ../common.mk -include ../common-commands.mk -include ../common-kube.mk - -PROVIDER_HOSTNAME = akash-provider.localhost -GATEWAY_ENDPOINT ?= https://akash-provider.localhost - -.PHONY: kube-namespace-setup -kube-namespace-setup: - kubectl apply -f "$(KUSTOMIZE_ROOT)/networking" - -.PHONY: provider-lease-ping -provider-lease-ping: - curl -sIH "Host: hello.localhost" localhost:$(KIND_HTTP_PORT) - -PHONY: clean-single -clean-single: - -.PHONY: kube-deployments-rollout -kube-deployments-rollout: kind-deployment-rollout-akash-node - -.PHONY: kube-setup-single -kube-setup-single: akash-node-ready provider-create diff --git a/_run/single/README.md b/_run/single/README.md deleted file mode 100644 index fe911c5c5..000000000 --- a/_run/single/README.md +++ /dev/null @@ -1,332 +0,0 @@ - -# Dev Environment: "Single" configuration - -The _Single_ dev environment builds A single-node blockchain network and -an Akash Provider Services Daemon (PSD) for bidding and running workloads, -all within a kind kubernetes environment. - -The [instructions](#runbook) below will illustrate how to: - -* [Initialize blockchain node and client](#initialize) -* [Run a single-node network](#run-local-network) -* [Query objects on the network](#run-query) -* [Create a provider](#create-a-provider) -* [Run provider services](#run-provider-services) -* [Create a deployment](#create-a-deployment) -* [Bid on an order](#create-a-bid) -* [Terminate a lease](#terminate-lease) - -See [commands](#commands) for a full list of utilities meant -for interacting with the network. - -Run a network with a single, local node and execute workloads in Minikube. - -Running through the entire suite requires three terminals. -Each command is marked __t1__-__t3__ to indicate a suggested terminal number. - -* https://kubectl.docs.kubernetes.io/pages/reference/kustomize.html - -## Setup - -**Developer Deps**: You will need `kubectl` installed - -### Overview - -Four keys and accounts are created. The key names are: - -| Key Name | Use | -|-------------|--------------------------------------------------| -| `main` | Primary account (creating deployments, etc...) | -| `provider` | The provider account (bidding on orders, etc...) | -| `validator` | The sole validator for the created network | -| `other` | Misc. account to (receives tokens, etc...) | - -Most `make` commands are configurable and have defaults to make it -such that you don't need to override them for a simple pass-through of -this example. - -| Name | Default | Description | -|---------------------|------------|---------------------------------| -| `KEY_NAME` | `main` | standard key name | -| `PROVIDER_KEY_NAME` | `provider` | name of key to use for provider | -| `DSEQ` | 1 | deployment sequence | -| `GSEQ` | 1 | group sequence | -| `OSEQ` | 1 | order sequence | -| `PRICE` | 10uakt | price to bid | - -To get DNS routing to work locally, there are two addresses which will probably need to set to configure requests to hit the kind docker container. To route requests back to the local interface, add the following two lines to your `/etc/hosts` for the Akash-Node and Akash-Provider examples to work correctly. - -* `127.0.0.1 akash.localhost` -* `127.0.0.1 akash-provider.localhost` - -Or if it does not conflict with other local rules, use a wildcard for localhost: -* `127.0.0.1 *.localhost` - -## Runbook - -The following steps will bring up a network and allow for interacting -with it. - -Running through the entire runbook requires two terminals. -Each command is marked __t1__-__t2__ to indicate a suggested terminal number. - -If at any time you'd like to start over with a fresh chain, simply run: - -__t1__ -```sh -make clean kind-cluster-clean -``` - -### Initialize Cluster - -Start and initialize kind. There are two options for network manager; standard CNI, or Calico. -Both are configured with Makefile targets as specified below. Using Calico enables testing of -Network Policies. - -**note**: If anything else is listening on port 80 (any other web server), this will fail. - -Pick one of the following commands: -__t1__ -```sh -# Standard Networking -make kind-cluster-setup - -# Calico Network Manger -KIND_CONFIG=calico make kind-cluster-setup -``` - -Check all pods in kube-system and ingress-nginx namespaces are in Running state. -If some pods are in Pending stay give it a little wait and check again -```shell -kubectl --context kind-single -n ingress-nginx -n kube-system get pods -``` - -### (Optional) Upload a local docker image - -If you specified a custom image in the earlier step you need to upload that image into the Kubernetes -cluster created by the `kind` command. This uploads an image from your local docker into the Kubernetes cluster. - -__t1__ -```sh -DOCKER_IMAGE=ovrclk/akash:mycustomtag make kind-upload-images -``` - -### Build Akash binaries and initialize network - -__t1__ -```sh -make init -``` - -### Initialize kustomize - -```sh -make kustomize-init -``` - -### Run local network - -```sh -make kustomize-install-node -``` - -You can check the status of the network with: - -__t1__ -```sh -make node-status -``` - -You should see blocks being produced - the block height should be increasing. - -You can now view genesis accounts that were created: - -**If this command fails**, consider adding `127.0.0.1 akash.localhost` to your `/etc/hosts` for DNS. - -__t1__ -```sh -make query-accounts -``` - -### Create a provider - -Create a provider on the network with the following command: - -__t1__ -```sh -make provider-create -``` - -View the on-chain representation of the provider with: - -__t1__ -```sh -make query-provider -``` - -### Run provider services - -In a separate terminal, run the following command - -__t2__ -```sh -make kustomize-install-provider -``` - -Query the provider service gateway for its status: - -__t1__ -```sh -make provider-status -``` - -### Create a deployment - -Create a deployment from the `main` account with: - -__t1__ -```sh -make deployment-create -``` - -This particular deployment is created from the sdl file in this directory ([`deployment.yaml`](deployment.yaml)). - -Check that the deployment was created. Take note of the `dseq` - deployment sequence: - -__t1__ -```sh -make query-deployments -``` - -After a short time, you should see an order created for this deployment with the following command: - -__t1__ -```sh -make query-orders -``` - -The Provider Services Daemon should see this order and bid on it. - -__t1__ -```sh -make query-bids -``` - -You should now see "pending" inventory inventory in the provider status: - -__t1__ -```sh -make provider-status -``` - -### Create a lease - -Create a lease for the bid from the provider: - -__t1__ -```sh -make lease-create -``` - -You should be able to see the lease with - -__t1__ -```sh -make query-leases -``` - -### Distribute Manifest - -Now that you have a lease with a provider, you need to send your -workload configuration to that provider by sending it the manifest: - -__t1__ -```sh -make send-manifest -``` - -You can check the status of your deployment with: - -__t1__ -```sh -make provider-lease-status -``` - -You can reach your app with the following (Note: `Host:` header tomfoolery abound) -__t1__ -```sh -make provider-lease-ping -``` - -If you chose to use port 80 when setting up kind, you can browse to your -deployed workload at http://hello.localhost - -### Withdraw from the lease - -Withdraw some funds from the lease - -__t1__ -```sh -make lease-withdraw -``` - -You should be able to see the escrow payment change in - -__t1__ -```sh -make query-deployment -``` - -and - -__t1__ -```sh -make query-accounts -``` - -## Update Provider - -If the KinD configuration uses Docker's random port assignment then the on-chain Provider data will need to be updated for `send-manfiest` to be able to correctly route the manifest POST request. - -For example you might need to update the `provider.yaml`'s first line to include the port number. eg: `host: http://akash-provider.localhost:41109` - - -## Update Deployment - -Updating active Deployments is a two step process. First edit the `deployment.yaml` with whatever changes are desired. Example; update the `image` field. - - 1. Update the Akash Network to inform the Provider that a new Deployment declaration is expected. - * `make deployment-update` - 2. Send the updated manifest to the Provider to run. - * `make send-manifest` - -Between the first and second step, the prior deployment's containers will continue to run until the new manifest file is received, validated, and new container group operational. After health checks on updated group are passing; the prior containers will be terminated. - -#### Limitations - -Akash Groups are translated into Kubernetes Deployments, this means that only a few fields from the Akash SDL are mutable. For example `image`, `command`, `args`, `env` and exposed ports can be modified, but compute resources and placement criteria cannot. - -### Terminate lease - -There are a number of ways that a lease can be terminated. - -#### Provider closes the bid: - -__t1__ -```sh -make lease-close -``` - -#### Tenant closes the order - -__t1__ -```sh -make order-close -``` - -#### Tenant closes the deployment - -__t1__ -```sh -make deployment-close -``` diff --git a/_run/single/deployment.yaml b/_run/single/deployment.yaml deleted file mode 100644 index 0fd61c35e..000000000 --- a/_run/single/deployment.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -version: "2.0" - -services: - web: - image: quay.io/ovrclk/demo-app - expose: - - port: 80 - as: 80 - accept: - - hello.localhost - to: - - global: true - bew: - image: quay.io/ovrclk/demo-app - expose: - - port: 80 - as: 80 - accept: - - hello1.localhost - to: - - global: true - -profiles: - compute: - web: - resources: - cpu: - units: 0.1 - memory: - size: 16Mi - storage: - size: 128Mi - placement: - westcoast: - attributes: - region: us-west - pricing: - web: - denom: uakt - amount: 1000 - -deployment: - web: - westcoast: - profile: web - count: 1 - bew: - westcoast: - profile: web - count: 1 \ No newline at end of file diff --git a/_run/single/deployment2.yaml b/_run/single/deployment2.yaml deleted file mode 100644 index 88bc875d9..000000000 --- a/_run/single/deployment2.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -version: "2.0" - -services: - web: - image: chentex/random-logger - expose: - - port: 80 - as: 80 - accept: - - hello.localhost - to: - - global: true - bew: - image: chentex/random-logger - expose: - - port: 80 - as: 80 - accept: - - hello1.localhost - to: - - global: true - -profiles: - compute: - web: - resources: - cpu: - units: 0.1 - memory: - size: 16Mi - storage: - size: 128Mi - placement: - westcoast: - attributes: - region: us-west - pricing: - web: - denom: uakt - amount: 1000 - -deployment: - web: - westcoast: - profile: web - count: 1 - bew: - westcoast: - profile: web - count: 1 \ No newline at end of file diff --git a/_run/single/kind-config.yaml b/_run/single/kind-config.yaml deleted file mode 100644 index d4c70073c..000000000 --- a/_run/single/kind-config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - extraPortMappings: - - containerPort: 80 - hostPort: 80 - protocol: TCP - - containerPort: 443 - hostPort: 443 - protocol: TCP diff --git a/_run/single/provider.yaml b/_run/single/provider.yaml deleted file mode 100644 index abef17f74..000000000 --- a/_run/single/provider.yaml +++ /dev/null @@ -1,4 +0,0 @@ -host: https://akash-provider.localhost -attributes: - - key: region - value: us-west diff --git a/integration/deployment_test.go b/integration/deployment_test.go new file mode 100644 index 000000000..8b2a7250e --- /dev/null +++ b/integration/deployment_test.go @@ -0,0 +1,214 @@ +//go:build e2e + +package integration + +import ( + "context" + "fmt" + "net" + "net/url" + "path/filepath" + "strings" + "time" + + "golang.org/x/sync/errgroup" + + dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" + mtypes "github.com/akash-network/akash-api/go/node/market/v1beta4" + clitestutil "github.com/akash-network/node/testutil/cli" + "github.com/akash-network/node/testutil/network" + deploycli "github.com/akash-network/node/x/deployment/client/cli" + mcli "github.com/akash-network/node/x/market/client/cli" + cutil "github.com/akash-network/provider/cluster/util" + ptestutil "github.com/akash-network/provider/testutil/provider" + "github.com/akash-network/provider/tools/fromctx" + "github.com/cosmos/cosmos-sdk/client/flags" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +type E2EDeploymentCreate struct { + IntegrationTestSuite +} + +func (s *E2EDeploymentCreate) TestE2EDeploymentCreateProviderShutdown() { + // create a deployment + deploymentPath, err := filepath.Abs("../testdata/deployment/deployment-v2.yaml") + s.Require().NoError(err) + + deploymentID := dtypes.DeploymentID{ + Owner: s.keyTenant.GetAddress().String(), + DSeq: uint64(102), + } + + // Create Deployments + res, err := deploycli.TxCreateDeploymentExec( + s.validator.ClientCtx, + s.keyTenant.GetAddress(), + deploymentPath, + cliGlobalFlags(fmt.Sprintf("--dseq=%v", deploymentID.DSeq))..., + ) + s.Require().NoError(err) + s.Require().NoError(s.waitForBlocksCommitted(3)) + clitestutil.ValidateTxSuccessful(s.T(), s.validator.ClientCtx, res.Bytes()) + + bidID := mtypes.MakeBidID( + mtypes.MakeOrderID(dtypes.MakeGroupID(deploymentID, 1), 1), + s.keyProvider.GetAddress(), + ) + // check bid + _, err = mcli.QueryBidExec(s.validator.ClientCtx, bidID) + s.Require().NoError(err) + + // create lease + _, err = mcli.TxCreateLeaseExec( + s.validator.ClientCtx, + bidID, + s.keyTenant.GetAddress(), + cliGlobalFlags()..., + ) + s.Require().NoError(err) + s.Require().NoError(s.waitForBlocksCommitted(2)) + clitestutil.ValidateTxSuccessful(s.T(), s.validator.ClientCtx, res.Bytes()) + + // Assert provider made bid and created lease; test query leases --------- + resp, err := mcli.QueryLeasesExec(s.validator.ClientCtx.WithOutputFormat("json")) + s.Require().NoError(err) + + leaseRes := &mtypes.QueryLeasesResponse{} + err = s.validator.ClientCtx.Codec.UnmarshalJSON(resp.Bytes(), leaseRes) + s.Require().NoError(err) + + s.Require().Len(leaseRes.Leases, 1) + + lease := newestLease(leaseRes.Leases) + lid := lease.LeaseID + s.Require().Equal(s.keyProvider.GetAddress().String(), lid.Provider) + + // Send Manifest to Provider + _, err = ptestutil.TestSendManifest( + s.validator.ClientCtx.WithOutputFormat("json"), + lid.BidID(), + deploymentPath, + fmt.Sprintf("--%s=%s", flags.FlagFrom, s.keyTenant.GetAddress().String()), + fmt.Sprintf("--%s=%s", flags.FlagHome, s.validator.ClientCtx.HomeDir), + ) + + s.Require().NoError(err) + s.Require().NoError(s.waitForBlocksCommitted(2)) + + appURL := fmt.Sprintf("http://%s:%s/", s.appHost, s.appPort) + queryAppWithHostname(s.T(), appURL, 50, "test.localhost") + + // Get initial pod information + kubeClient := fromctx.MustKubeClientFromCtx(s.ctx) + namespace := cutil.LeaseIDToNamespace(lid) + observer := NewObserver(namespace) + + err = observer.Observe(kubeClient) + s.Require().NoError(err) + + s.T().Logf("canceling provider context to force a restart") + s.ctxCancel() + + ctx, cancel := context.WithCancel(context.Background()) + s.ctxCancel = cancel + + group, ctx := errgroup.WithContext(ctx) + ctx = context.WithValue(ctx, fromctx.CtxKeyErrGroup, group) + + s.ctx = ctx + s.group = group + + // Restart the provider by stopping and starting it again + cliHome := strings.Replace(s.validator.ClientCtx.HomeDir, "simd", "simcli", 1) + + numPorts := 3 + if s.ipMarketplace { + numPorts++ + } + + ports, err := network.GetFreePorts(numPorts) + s.Require().NoError(err) + + // address for provider to listen on + provHost := fmt.Sprintf("localhost:%d", ports[0]) + provURL := url.URL{ + Host: provHost, + Scheme: "https", + } + + hostnameOperatorPort := ports[2] + hostnameOperatorHost := fmt.Sprintf("localhost:%d", hostnameOperatorPort) + + var ipOperatorHost string + var ipOperatorPort int + if s.ipMarketplace { + ipOperatorPort = ports[3] + ipOperatorHost = fmt.Sprintf("localhost:%d", ipOperatorPort) + } + + extraArgs := []string{ + fmt.Sprintf("--%s=%s", flags.FlagFees, sdk.NewCoins(sdk.NewCoin(s.cfg.BondDenom, sdk.NewInt(20))).String()), + "--deployment-runtime-class=none", // do not use gvisor in test + fmt.Sprintf("--hostname-operator-endpoint=%s", hostnameOperatorHost), + } + + if s.ipMarketplace { + extraArgs = append(extraArgs, fmt.Sprintf("--ip-operator-endpoint=%s", ipOperatorHost)) + extraArgs = append(extraArgs, "--ip-operator") + } + + s.group.Go(func() error { + _, err := ptestutil.RunLocalProvider(ctx, + s.validator.ClientCtx, + s.validator.ClientCtx.ChainID, + s.validator.RPCAddress, + cliHome, + s.keyProvider.GetName(), + provURL.Host, + extraArgs..., + ) + + if err != nil { + s.T().Logf("provider exit %v", err) + } + + return err + }) + + dialer := net.Dialer{ + Timeout: time.Second * 3, + } + + // Wait for the provider gateway to be up and running + s.T().Log("waiting for provider gateway") + waitForTCPSocket(s.ctx, dialer, provHost, s.T()) + + // --- Start hostname operator + s.group.Go(func() error { + s.T().Logf("starting hostname operator for test on %s", hostnameOperatorHost) + _, err := ptestutil.RunLocalHostnameOperator(s.ctx, s.validator.ClientCtx, hostnameOperatorPort) + s.Assert().NoError(err) + return err + }) + + s.T().Log("waiting for hostname operator") + waitForTCPSocket(s.ctx, dialer, hostnameOperatorHost, s.T()) + + if s.ipMarketplace { + s.group.Go(func() error { + s.T().Logf("starting ip operator for test on %v", ipOperatorHost) + _, err := ptestutil.RunLocalIPOperator(s.ctx, s.validator.ClientCtx, ipOperatorPort, s.keyProvider.GetAddress()) + s.Assert().NoError(err) + return err + }) + + s.T().Log("waiting for IP operator") + waitForTCPSocket(s.ctx, dialer, ipOperatorHost, s.T()) + } + + s.Require().NoError(s.network.WaitForNextBlock()) + + err = observer.VerifyNoChangeOccurred(kubeClient) + s.Require().NoError(err) +} diff --git a/integration/deployment_update_test.go b/integration/deployment_update_test.go index 03bb0b892..2c0fb50d4 100644 --- a/integration/deployment_update_test.go +++ b/integration/deployment_update_test.go @@ -20,6 +20,9 @@ import ( mcli "github.com/akash-network/node/x/market/client/cli" ptestutil "github.com/akash-network/provider/testutil/provider" + + "github.com/akash-network/provider/tools/fromctx" + cutil "github.com/akash-network/provider/cluster/util" ) type E2EDeploymentUpdate struct { @@ -109,6 +112,14 @@ func (s *E2EDeploymentUpdate) TestE2EDeploymentUpdate() { s.Require().NoError(s.waitForBlocksCommitted(2)) clitestutil.ValidateTxSuccessful(s.T(), s.validator.ClientCtx, res.Bytes()) + // Get initial pod information + kubeClient := fromctx.MustKubeClientFromCtx(s.ctx) + namespace := cutil.LeaseIDToNamespace(lid) + observer := NewObserver(namespace) + + err = observer.Observe(kubeClient) + s.Require().NoError(err) + // Send Updated Manifest to Provider _, err = ptestutil.TestSendManifest( s.validator.ClientCtx.WithOutputFormat("json"), @@ -120,6 +131,9 @@ func (s *E2EDeploymentUpdate) TestE2EDeploymentUpdate() { s.Require().NoError(err) s.Require().NoError(s.waitForBlocksCommitted(2)) queryAppWithHostname(s.T(), appURL, 50, "testupdateb.localhost") + + err = observer.VerifyNewPodCreate(kubeClient) + s.Require().NoError(err) } func (s *E2EDeploymentUpdate) TestE2ELeaseShell() { @@ -187,6 +201,14 @@ func (s *E2EDeploymentUpdate) TestE2ELeaseShell() { s.Require().NoError(err) s.Require().NoError(s.waitForBlocksCommitted(2)) + // Get initial pod information + kubeClient := fromctx.MustKubeClientFromCtx(s.ctx) + namespace := cutil.LeaseIDToNamespace(lID) + observer := NewObserver(namespace) + + err = observer.Observe(kubeClient) + s.Require().NoError(err) + extraArgs := []string{ fmt.Sprintf("--%s=%s", flags.FlagFrom, s.keyTenant.GetAddress().String()), fmt.Sprintf("--%s=%s", flags.FlagHome, s.validator.ClientCtx.HomeDir), @@ -257,4 +279,7 @@ func (s *E2EDeploymentUpdate) TestE2ELeaseShell() { lID, 99, false, false, "notaservice", "/bin/echo", "/foo") require.Error(s.T(), err) require.Regexp(s.T(), ".*no service for that lease.*", err.Error()) + + err = observer.VerifyNoChangeOccurred(kubeClient) + s.Require().NoError(err) } diff --git a/integration/e2e_test.go b/integration/e2e_test.go index c21d34224..69b250b81 100644 --- a/integration/e2e_test.go +++ b/integration/e2e_test.go @@ -226,7 +226,7 @@ func (s *IntegrationTestSuite) SetupSuite() { require.NoError(s.T(), err) // create Provider blockchain declaration - _, err = cli.TxCreateProviderExec( + res, err = cli.TxCreateProviderExec( s.validator.ClientCtx, s.keyProvider.GetAddress(), fmt.Sprintf("%s/%s", s.network.BaseDir, fstat.Name()), @@ -561,6 +561,7 @@ func TestIntegrationTestSuite(t *testing.T) { suite.Run(t, new(E2EContainerToContainer)) suite.Run(t, new(E2EAppNodePort)) suite.Run(t, new(E2EDeploymentUpdate)) + suite.Run(t, new(E2EDeploymentCreate)) suite.Run(t, new(E2EApp)) suite.Run(t, new(E2EPersistentStorageDefault)) suite.Run(t, new(E2EPersistentStorageBeta2)) diff --git a/integration/test_helpers.go b/integration/test_helpers.go index 2ded90c48..e9bc2f0c0 100644 --- a/integration/test_helpers.go +++ b/integration/test_helpers.go @@ -4,6 +4,9 @@ package integration import ( "bytes" + "context" + "errors" + "fmt" "io" "net" "net/http" @@ -12,6 +15,9 @@ import ( "time" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) const ( @@ -174,3 +180,116 @@ func appEnv(t *testing.T) (string, string) { require.NotEmpty(t, appPort) return host, appPort } + +type Observer struct { + initialUIDs map[string]struct{} + initialRestarts map[string]int32 + namespace string +} + +type ErrUnexpectedStateChange struct { + reason string +} + +func (e ErrUnexpectedStateChange) Error() string { + return fmt.Sprintf("unexpected state change: %s", e.reason) +} + +func NewObserver(namespace string) *Observer { + return &Observer{ + initialUIDs: make(map[string]struct{}), + initialRestarts: make(map[string]int32), + namespace: namespace, + } +} + +func (o *Observer) Observe(kubeClient kubernetes.Interface) error { + initialPods, err := kubeClient.CoreV1().Pods(o.namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "akash.network/manifest-service=web", + }) + if err != nil { + return err + } + + for _, pod := range initialPods.Items { + o.initialUIDs[string(pod.UID)] = struct{}{} + for _, containerStatus := range pod.Status.ContainerStatuses { + o.initialRestarts[pod.Name] = containerStatus.RestartCount + } + } + + return nil +} + +func (o *Observer) podRestartedOrNewPodCreated(pod corev1.Pod) bool { + if o.newPodCreated(pod) { + return true + } + + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.RestartCount > o.initialRestarts[pod.Name] { + return true + } + } + return false +} + +func (o *Observer) newPodCreated(pod corev1.Pod) bool { + _, exists := o.initialUIDs[string(pod.UID)] + return !exists +} + +func (o *Observer) VerifyNewPodCreate(kubeClient kubernetes.Interface) error { + timeout := time.After(30 * time.Second) + tick := time.Tick(5 * time.Second) + for { + select { + case <-timeout: + return errors.New("timeout waiting for deployment update") + case <-tick: + updatedPods, err := kubeClient.CoreV1().Pods(o.namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "akash.network/manifest-service=web", + }) + if err != nil { + return err + } + + newPodCreated := false + for _, pod := range updatedPods.Items { + // Check if this is a new pod + newPodCreated = o.newPodCreated(pod) + if newPodCreated { + break + } + } + + if newPodCreated { + return nil + } + } + } +} + +func (o *Observer) VerifyNoChangeOccurred(kubeClient kubernetes.Interface) error { + timeout := time.After(30 * time.Second) + tick := time.Tick(5 * time.Second) + for { + select { + case <-timeout: + return nil + case <-tick: + updatedPods, err := kubeClient.CoreV1().Pods(o.namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "akash.network/manifest-service=web", + }) + if err != nil { + return err + } + + for _, pod := range updatedPods.Items { + if o.podRestartedOrNewPodCreated(pod) { + return ErrUnexpectedStateChange{"container restarted or new pod created"} + } + } + } + } +}