From 78f1f811064cffc50bd1212a6d6085de0f9b3ade Mon Sep 17 00:00:00 2001 From: Abhishek Kekane Date: Tue, 12 Nov 2024 05:49:23 +0000 Subject: [PATCH 1/3] Fix performance glitch while sorting image locations Some of the available glance stores like file, cinder etc has capability to reuse already initiated driver (DRIVER_REUSABLE = 0b01000000). In Caracal we have added a feature to sort image locations based on store weight. As RBD driver of glance does not have this reuse capability, during image list API call it initializes the RBD driver for each of the available image which is causing noticable delay in list call. To avoid this, using new interface added in glance_store which will directly get the weight of the store from memory and return it back to user. Depends-On: https://review.opendev.org/c/openstack/glance_store/+/940531 Closes-Bug: #2086675 Change-Id: I662ba19697e03917ca999920ea7be93a0b2a8296 (cherry picked from commit ce537e073b2e7efb332d719bb45bf20c403c140a) --- glance/common/utils.py | 3 +-- glance/tests/unit/common/test_utils.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/glance/common/utils.py b/glance/common/utils.py index feb2c81565..860b9fb600 100644 --- a/glance/common/utils.py +++ b/glance/common/utils.py @@ -724,13 +724,12 @@ def get_store_weight(location): if not store_id: return 0 try: - store = glance_store.get_store_from_store_identifier(store_id) + return glance_store.get_store_weight(store_id) except glance_store.exceptions.UnknownScheme: msg = (_LW("Unable to find store '%s', returning " "default weight '0'") % store_id) LOG.warning(msg) return 0 - return store.weight if store is not None else 0 sorted_locations = sorted(locations, key=get_store_weight, reverse=True) LOG.debug(('Sorted locations: %s'), sorted_locations) diff --git a/glance/tests/unit/common/test_utils.py b/glance/tests/unit/common/test_utils.py index 817ea05bb4..f77fa502c8 100644 --- a/glance/tests/unit/common/test_utils.py +++ b/glance/tests/unit/common/test_utils.py @@ -189,7 +189,7 @@ def test_sort_image_locations_multistore_disabled(self): 'url': 'rbd://cccccccc/images/id', 'metadata': {'store': 'rbd3'} }] - mp = "glance.common.utils.glance_store.get_store_from_store_identifier" + mp = "glance.common.utils.glance_store.get_store_weight" with mock.patch(mp) as mock_get_store: utils.sort_image_locations(locations) @@ -215,10 +215,9 @@ def test_sort_image_locations(self): 'url': 'rbd://cccccccc/images/id', 'metadata': {'store': 'rbd3'} }] - mp = "glance.common.utils.glance_store.get_store_from_store_identifier" + mp = "glance.common.utils.glance_store.get_store_weight" with mock.patch(mp) as mock_get_store: - mock_store = mock_get_store.return_value - mock_store.weight = 100 + mock_get_store.return_value = 100 utils.sort_image_locations(locations) # Since 3 stores are configured, internal method will be called 3 times @@ -243,7 +242,7 @@ def test_sort_image_locations_without_metadata(self): 'url': 'rbd://cccccccc/images/id', 'metadata': {} }] - mp = "glance.common.utils.glance_store.get_store_from_store_identifier" + mp = "glance.common.utils.glance_store.get_store_weight" with mock.patch(mp) as mock_get_store: utils.sort_image_locations(locations) @@ -270,10 +269,9 @@ def test_sort_image_locations_with_partial_metadata(self): 'url': 'rbd://cccccccc/images/id', 'metadata': {} }] - mp = "glance.common.utils.glance_store.get_store_from_store_identifier" + mp = "glance.common.utils.glance_store.get_store_weight" with mock.patch(mp) as mock_get_store: - mock_store = mock_get_store.return_value - mock_store.weight = 100 + mock_get_store.return_value = 100 utils.sort_image_locations(locations) # Since 3 stores are configured, but only one location has @@ -300,7 +298,7 @@ def test_sort_image_locations_unknownscheme(self): 'url': 'rbd://cccccccc/images/id', 'metadata': {'store': 'rbd3'} }] - mp = "glance.common.utils.glance_store.get_store_from_store_identifier" + mp = "glance.common.utils.glance_store.get_store_weight" with mock.patch(mp) as mock_get_store: mock_get_store.side_effect = store.UnknownScheme() sorted_locations = utils.sort_image_locations(locations) From 99dac1bb4c7cb7ea107d21f9fc35b22e232b6e06 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 11 Feb 2025 10:22:15 +0000 Subject: [PATCH 2/3] Use stackhpc fork of glance-store --- requirements.txt | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 98552adde4..00cbfc75e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ retrying!=1.3.0,>=1.2.3 # Apache-2.0 osprofiler>=1.4.0 # Apache-2.0 # Glance Store -glance-store>=2.3.0 # Apache-2.0 +glance-store @ git+https://github.com/stackhpc/glance_store@stackhpc/4.7.0.3 # Apache-2.0 debtcollector>=1.19.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index bfa45059af..3edb12b6f3 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ setenv = OS_TEST_DBAPI_ADMIN_CONNECTION=sqlite:////tmp/placeholder-never-created-nor-used.db # TODO(stephenfin): Remove once we bump our upper-constraint to SQLAlchemy 2.0 SQLALCHEMY_WARN_20=1 + TOX_CONSTRAINTS_FILE=https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/2024.1/upper-constraints.txt usedevelop = True install_command = python -m pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2024.1} {opts} {packages} deps = -r{toxinidir}/test-requirements.txt From 34c8ec145f9383a9759555ba6471bcd751cfeb39 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 11 Feb 2025 12:46:01 +0000 Subject: [PATCH 3/3] Add test profile to qemu packages --- bindep.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bindep.txt b/bindep.txt index 0cb290c2cf..53fa6edff6 100644 --- a/bindep.txt +++ b/bindep.txt @@ -24,10 +24,10 @@ postgresql-server [platform:rpm] postgresql-server-devel [platform:suse] # Provides pg_config libpython3-dev [platform:dpkg] python3-devel [platform:rpm] -qemu [platform:dpkg devstack build-image-dib] -qemu-utils [platform:dpkg devstack build-image-dib] -qemu-img [platform:redhat] -qemu-tools [platform:suse] # Provides qemu-img +qemu [platform:dpkg devstack build-image-dib test] +qemu-utils [platform:dpkg devstack build-image-dib test] +qemu-img [platform:redhat test] +qemu-tools [platform:suse test] # Provides qemu-img libpq-dev [platform:dpkg] libpcre3-dev [platform:dpkg] pcre-devel [platform:redhat]