From a92c63413b8a0a8c4f88af2ba9c75122cfb4bcc6 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 12 Apr 2023 18:50:44 +0300 Subject: [PATCH 1/6] Update all packages with pip-upgrade --- dev-requirements.txt | 28 ++++++++++++------------- gprofiler/gprofiler_types.py | 3 +-- gprofiler/metadata/py_module_version.py | 4 ++-- gprofiler/metadata/system_metadata.py | 3 +-- gprofiler/system_metrics.py | 3 +-- granulate-utils | 2 +- requirements.txt | 20 +++++++++--------- tests/conftest.py | 1 - tests/test_app_metadata.py | 4 +++- tests/test_java.py | 1 - 10 files changed, 33 insertions(+), 36 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index e801c4c77..26e699ac9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,17 +1,17 @@ -pytest==7.0.1 -flake8==5.0.4 -black==22.8.0 -mypy==0.971 -isort==5.10.1 +pytest==7.3.0 +flake8==6.0.0 +black==23.3.0 +mypy==1.2.0 +isort==5.12.0 PyYAML==6.0 -docker==5.0.0 +docker==6.0.1 six==1.16.0 -types-requests==0.1.9 -types-dataclasses==0.1.5 -types-psutil==5.8.19 -types-PyYAML==6.0.3 +types-requests==2.28.11.17 +types-dataclasses==0.6.6 +types-psutil==5.9.5.11 +types-PyYAML==6.0.12.9 types-pkg-resources==0.1.3 -types-protobuf==3.19.22 -types-toml==0.10.8 -types-retry==0.9.9 -types-beautifulsoup4==4.11.1 \ No newline at end of file +types-protobuf==4.22.0.2 +types-toml==0.10.8.6 +types-retry==0.9.9.3 +types-beautifulsoup4==4.12.2 \ No newline at end of file diff --git a/gprofiler/gprofiler_types.py b/gprofiler/gprofiler_types.py index 11144becf..1f2a26183 100644 --- a/gprofiler/gprofiler_types.py +++ b/gprofiler/gprofiler_types.py @@ -35,7 +35,6 @@ class ProfileData: class ProfilingErrorStack(StackToSampleCount): - PROFILING_ERROR_STACK_PATTERN = re.compile(r".*;\[Profiling .+: .+\]") def __init__(self, what: str, reason: str, comm: str): @@ -55,7 +54,7 @@ def attach_error_to_stacks( ) -> StackToSampleCount: _, error_frame = next(iter(error_stack)).split(";", maxsplit=1) dest_stacks: StackToSampleCount = StackToSampleCount() - for (frame, count) in source_stacks.items(): + for frame, count in source_stacks.items(): comm, stack = frame.split(";", maxsplit=1) annotated = f"{comm};{error_frame};{stack}" dest_stacks[annotated] = count diff --git a/gprofiler/metadata/py_module_version.py b/gprofiler/metadata/py_module_version.py index 6eb663927..a9bb5531f 100644 --- a/gprofiler/metadata/py_module_version.py +++ b/gprofiler/metadata/py_module_version.py @@ -46,7 +46,7 @@ def _get_packages_dir(file_path: str) -> Optional[str]: def _get_metadata(dist: pkg_resources.Distribution) -> Dict[str, str]: """Based on pip._internal.utils.get_metadata""" metadata_name = "METADATA" - if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(metadata_name): # type: ignore + if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(metadata_name): metadata = dist.get_metadata(metadata_name) elif dist.has_metadata("PKG-INFO"): metadata_name = "PKG-INFO" @@ -109,7 +109,7 @@ def _files_from_legacy(dist: pkg_resources.Distribution) -> Optional[Iterator[st return None paths = (p for p in text.splitlines(keepends=False) if p) root = dist.location - info = dist.egg_info # type: ignore + info = dist.egg_info if root is None or info is None: return paths try: diff --git a/gprofiler/metadata/system_metadata.py b/gprofiler/metadata/system_metadata.py index 285ea3859..e78b7ea78 100644 --- a/gprofiler/metadata/system_metadata.py +++ b/gprofiler/metadata/system_metadata.py @@ -259,8 +259,7 @@ def get_static_system_info() -> SystemInfo: processors=cpu_count, cpu_model_name=cpu_model_name, cpu_flags=cpu_flags, - memory_capacity_mb=round(psutil.virtual_memory().total / 1024 / 1024), # type: ignore # virtual_memory doesn't - # have a return type is types-psutil + memory_capacity_mb=round(psutil.virtual_memory().total / 1024 / 1024), hostname=hostname, system=platform.system(), os_name=os_name, diff --git a/gprofiler/system_metrics.py b/gprofiler/system_metrics.py index 8508d42e8..641d2de09 100644 --- a/gprofiler/system_metrics.py +++ b/gprofiler/system_metrics.py @@ -70,8 +70,7 @@ def stop(self) -> None: def _continuously_poll_memory(self, polling_rate_seconds: int) -> None: while not self._stop_event.is_set(): - current_ram_percent = psutil.virtual_memory().percent # type: ignore # virtual_memory doesn't have a - # return type is types-psutil + current_ram_percent = psutil.virtual_memory().percent self._mem_percentages.append(current_ram_percent) self._stop_event.wait(timeout=polling_rate_seconds) diff --git a/granulate-utils b/granulate-utils index 38d7add30..2cb2f6fd8 160000 --- a/granulate-utils +++ b/granulate-utils @@ -1 +1 @@ -Subproject commit 38d7add30a082eab46ff6b42b428c54337c8a4e3 +Subproject commit 2cb2f6fd807c6135de7d87295e3f1a6d18d786c6 diff --git a/requirements.txt b/requirements.txt index a3a824d85..fd4b7056a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -importlib-resources==5.1.0 -psutil==5.8.0 -requests==2.27.1 -ConfigArgParse==1.3 -distro==1.7.0 -setuptools==65.5.1 # For pkg_resources +importlib-resources==5.12.0 +psutil==5.9.4 +requests==2.28.2 +ConfigArgParse==1.5.3 +distro==1.8.0 +setuptools==67.6.1 # For pkg_resources six==1.16.0 -packaging==21.2 -pyelftools==0.28 +packaging==23.0 +pyelftools==0.29 curlify==2.2.1 retry==0.9.2 -websocket-client==1.3.1 +websocket-client==1.5.1 netifaces==0.11.0; sys.platform == "win32" WMI==1.5.1; sys.platform == "win32" ./granulate-utils/ humanfriendly==10.0 -beautifulsoup4==4.11.1 +beautifulsoup4==4.12.2 diff --git a/tests/conftest.py b/tests/conftest.py index 557e4e481..1e72610bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -355,7 +355,6 @@ def application_docker_image( runtime: str, application_image_tag: str, ) -> Iterable[Image]: - if is_aarch64(): if runtime == "nodejs": if application_image_tag == "12-glibc": diff --git a/tests/test_app_metadata.py b/tests/test_app_metadata.py index 2b8119eee..799c7a112 100644 --- a/tests/test_app_metadata.py +++ b/tests/test_app_metadata.py @@ -205,7 +205,9 @@ def test_app_metadata( assert application_docker_container.name in metadata["containers"] # find its app metadata index - find a stack line from the app of this container - stack = next(filter(lambda l: application_docker_container.name in l and application_executable in l, lines[1:])) + stack = next( + filter(lambda line: application_docker_container.name in line and application_executable in line, lines[1:]) + ) # stack begins with index idx = int(stack.split(";")[0]) diff --git a/tests/test_java.py b/tests/test_java.py index fd0cfdd85..1c59a1fc2 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -384,7 +384,6 @@ def test_sanity_other_jvms( search_for: str, profiler_state: ProfilerState, ) -> None: - if is_aarch64(): pytest.xfail( "Different JVMs are not supported on aarch64, see https://github.com/Granulate/gprofiler/issues/717" From 1561bb44df2f130a1b91faf9b41adb39c85892f1 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 12 Apr 2023 18:53:58 +0300 Subject: [PATCH 2/6] update --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 26e699ac9..3c88ed465 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,4 +14,4 @@ types-pkg-resources==0.1.3 types-protobuf==4.22.0.2 types-toml==0.10.8.6 types-retry==0.9.9.3 -types-beautifulsoup4==4.12.2 \ No newline at end of file +types-beautifulsoup4==4.12.0.3 From 8d5f9a655728eff4bd1ab910a8d52d97ae827440 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 12 Apr 2023 18:55:39 +0300 Subject: [PATCH 3/6] fix --- granulate-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/granulate-utils b/granulate-utils index 2cb2f6fd8..09469a56c 160000 --- a/granulate-utils +++ b/granulate-utils @@ -1 +1 @@ -Subproject commit 2cb2f6fd807c6135de7d87295e3f1a6d18d786c6 +Subproject commit 09469a56c1255fcb76950eb3e2e8eda6285db2c2 From 3d8433e043cfbe58df2063cc17105e19a9e6c2bc Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 12 Apr 2023 19:06:11 +0300 Subject: [PATCH 4/6] try --- gprofiler/metadata/py_module_version.py | 10 +++++----- tests/conftest.py | 3 +-- tests/containers/python/lister.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/gprofiler/metadata/py_module_version.py b/gprofiler/metadata/py_module_version.py index a9bb5531f..7b457819d 100644 --- a/gprofiler/metadata/py_module_version.py +++ b/gprofiler/metadata/py_module_version.py @@ -46,7 +46,7 @@ def _get_packages_dir(file_path: str) -> Optional[str]: def _get_metadata(dist: pkg_resources.Distribution) -> Dict[str, str]: """Based on pip._internal.utils.get_metadata""" metadata_name = "METADATA" - if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(metadata_name): + if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(metadata_name): # type: ignore metadata = dist.get_metadata(metadata_name) elif dist.has_metadata("PKG-INFO"): metadata_name = "PKG-INFO" @@ -109,7 +109,7 @@ def _files_from_legacy(dist: pkg_resources.Distribution) -> Optional[Iterator[st return None paths = (p for p in text.splitlines(keepends=False) if p) root = dist.location - info = dist.egg_info + info = dist.egg_info # type: ignore if root is None or info is None: return paths try: @@ -219,8 +219,8 @@ def _populate_packages_versions(packages_versions: Dict[str, Optional[Tuple[str, # This function resolves symlinks and makes paths absolute for comparison purposes which isn't required # for our usage. if hasattr(pkg_resources, "_normalize_cached"): - original__normalize_cache = pkg_resources._normalize_cached # type: ignore - pkg_resources._normalize_cached = lambda path: path # type: ignore + original__normalize_cache = pkg_resources._normalize_cached + pkg_resources._normalize_cached = lambda path: path else: global _warned_no__normalized_cached if not _warned_no__normalized_cached: @@ -250,7 +250,7 @@ def _populate_packages_versions(packages_versions: Dict[str, Optional[Tuple[str, packages_versions[module_path] = package_info finally: # Don't forget to restore the original implementation in case someone else uses this function - pkg_resources._normalize_cached = original__normalize_cache # type: ignore + pkg_resources._normalize_cached = original__normalize_cache _exceptions_logged = 0 diff --git a/tests/conftest.py b/tests/conftest.py index 685d16bac..6632aa129 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -74,8 +74,7 @@ def chdir(path: Path) -> Iterator[None]: @fixture(params=[False, True]) def in_container(request: FixtureRequest) -> bool: - return cast(bool, request.param) # type: ignore # SubRequest isn't exported yet, - # https://github.com/pytest-dev/pytest/issues/7469 + return cast(bool, request.param) @fixture diff --git a/tests/containers/python/lister.py b/tests/containers/python/lister.py index 0a0f2c7a4..a6606864c 100755 --- a/tests/containers/python/lister.py +++ b/tests/containers/python/lister.py @@ -30,7 +30,7 @@ def parser(): while True: # Have some package stacks. # Notice the name of the package name (PyYAML) is different from the name of the module (yaml) - yaml.parse("") # type: ignore + yaml.parse("") if __name__ == "__main__": From 8affc60a99a154936b37d60bdb1adcb9bca60489 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 15 Apr 2023 21:02:44 +0300 Subject: [PATCH 5/6] add preload --- gprofiler/metadata/system_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gprofiler/metadata/system_metadata.py b/gprofiler/metadata/system_metadata.py index e78b7ea78..5e6a9996e 100644 --- a/gprofiler/metadata/system_metadata.py +++ b/gprofiler/metadata/system_metadata.py @@ -1,4 +1,5 @@ import array +import codecs import errno import ipaddress import os @@ -324,6 +325,10 @@ def _initialize_system_info() -> Any: mac_address = UNKNOWN_VALUE local_ip = UNKNOWN_VALUE + # ensure the "ascii" codec module is loaded - it used used by "distro" which is run in a separate + # mount namespace, unable to load the library if loaded lazily. + codecs.lookup("ascii") + # move to host mount NS for distro & ldd. # now, distro will read the files on host. # also move to host UTS NS for the hostname. From e61c301a3d1aa43d6ca1931718ddae2beb8663c6 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Mon, 17 Apr 2023 13:14:38 +0300 Subject: [PATCH 6/6] bla --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0bc04d863..3c88ed465 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,5 @@ pytest==7.3.0 +flake8==6.0.0 black==23.3.0 mypy==1.2.0 isort==5.12.0