Fix smoke-test build and fix linter issues

Author: treff7es

datahub-actions/setup.py

@@ -120,12 +120,7 @@ def get_long_description():
     *base_requirements,
     *framework_common,
     *mypy_stubs,
-    "black>=22.1.0",
     "coverage>=5.1",
-    "flake8>=3.8.3",
-    "flake8-tidy-imports>=4.3.0",
-    "isort>=5.7.0",
-    "mypy==1.0.0",
     "pytest>=6.2.2",
     "pytest-cov>=2.8.1",
     "pytest-dependency>=0.5.1",

datahub-actions/smoke-test/build.gradle

@@ -19,23 +19,20 @@ task installDev(type: Exec) {
 }
 
 task pythonLint(type: Exec, dependsOn: installDev) {
-    commandLine 'bash', '-c',
-        "source ${venv_name}/bin/activate && set -x && " +
-        "black --check --diff tests/ && " +
-        "isort --check --diff tests/ && " +
-        "ruff --statistics tests/ && " +
-        "mypy tests/"
+  commandLine 'bash', '-c',
+      "source ${venv_name}/bin/activate && set -x && " +
+          "ruff check tests/ && " +
+          "ruff format --check tests/ && " +
+          "mypy --show-traceback --show-error-codes tests/"
 }
+
 task pythonLintFix(type: Exec, dependsOn: installDev) {
-    commandLine 'bash', '-c',
-        "source ${venv_name}/bin/activate && set -x && " +
-        "black tests/ && " +
-        "isort tests/ && " +
-        "ruff --fix tests/ && " +
-        "mypy tests/"
+  commandLine 'bash', '-c',
+      "source ${venv_name}/bin/activate && set -x && " +
+          "ruff check --fix tests/ && " +
+          "ruff format tests/ "
 }
 
-
 task lint {
     dependsOn pythonLint
 }

datahub-actions/smoke-test/pyproject.toml

@@ -1,46 +1,60 @@
 [build-system]
-requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
+requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
 
-[project]
-name = "smoke-test"
-version = "0.0.0"
-description = ""
-authors = [
-	{ name="Acryl Data", email="[email protected]" },
+[tool.ruff]
+line-length = 88
+target-version = "py38"
+exclude = [
+	".git",
+	"venv",
+	".tox",
+	"__pycache__",
 ]
-requires-python = ">=3.9"
-
 
-[tool.black]
-extend-exclude = '''
-# A regex preceded with ^/ will apply only to files and directories
-# in the root of the project.
-tmp
-venv
-'''
-include = '\.pyi?$'
-target-version = ['py310']
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
 
-[tool.isort]
-profile = 'black'
+[tool.ruff.lint.isort]
+combine-as-imports = true
+known-first-party = ["datahub"]
+extra-standard-library = ["__future__"]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+force-sort-within-sections = false
+force-wrap-aliases = false
+split-on-trailing-comma = false
+order-by-type = true
+relative-imports-order = "closest-to-furthest"
+force-single-line = false
+single-line-exclusions = ["typing"]
+length-sort = false
+from-first = false
+required-imports = []
+classes = ["typing"]
 
-[tool.ruff]
+[tool.ruff.lint]
+extend-select = [
+	"B",    # flake8-bugbear
+	"C90",  # mccabe complexity
+	"E",    # pycodestyle errors
+	"F",    # pyflakes
+	"G010", # logging.warn -> logging.warning
+	"I",    # isort
+	"TID",  # flake8-tidy-imports
+	"RUF100",  # unused-noqa
+]
 ignore = [
-	'E501', # Ignore line length, since black handles that.
-	'D203', # Ignore 1 blank line required before class docstring.
+	"E501",  # Line length violations (handled by formatter)
 ]
 
-[tool.mypy]
-exclude = "^(venv/|build/|dist/)"
-ignore_missing_imports = true
-namespace_packages = false
-check_untyped_defs = true
-disallow_untyped_decorators = true
-warn_unused_configs = true
-# eventually we'd like to enable these
-disallow_incomplete_defs = false
-disallow_untyped_defs = false
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
 
-[tool.pyright]
-extraPaths = ['tests']
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]

datahub-actions/smoke-test/requirements.txt

@@ -10,15 +10,16 @@ networkx
 # libaries for linting below this
 black==23.7.0
 isort==5.12.0
-mypy==1.5.1
-ruff==0.0.287
+mypy==1.10.1
+ruff==0.9.7
 # stub version are copied from metadata-ingestion/setup.py and that should be the source of truth
 types-requests>=2.28.11.6,<=2.31.0.3
 types-PyYAML
+joblib-stubs
 # https://github.com/docker/docker-py/issues/3256
 requests<=2.31.0
 # Missing numpy requirement in 8.0.0
 deepdiff!=8.0.0
 acryl-datahub
 jinja2
--e ../datahub-actions
+-e ../

datahub-actions/smoke-test/tests/actions/doc_propagation/test_propagation.py

@@ -7,9 +7,12 @@
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Tuple
 
-import datahub.metadata.schema_classes as models
 import pytest
 import tenacity
+from jinja2 import Template
+from pydantic import BaseModel
+
+import datahub.metadata.schema_classes as models
 from datahub.api.entities.dataset.dataset import Dataset
 from datahub.emitter.mce_builder import make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
@@ -18,9 +21,6 @@
 from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
 from datahub.ingestion.sink.file import FileSink, FileSinkConfig
 from datahub.utilities.urns.urn import Urn
-from jinja2 import Template
-from pydantic import BaseModel
-
 from tests.utils import (
     delete_urns_from_file,
     get_gms_url,

datahub-actions/smoke-test/tests/utils.py

@@ -6,12 +6,12 @@
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Tuple
 
-from datahub.cli import cli_utils, env_utils
-from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
-from datahub.ingestion.run.pipeline import Pipeline
 from joblib import Parallel, delayed
 
 import requests_wrapper as requests
+from datahub.cli import cli_utils, env_utils
+from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
+from datahub.ingestion.run.pipeline import Pipeline
 from tests.consistency_utils import wait_for_writes_to_sync
 
 TIME: int = 1581407189000
@@ -97,7 +97,7 @@ def check_endpoint(url):
         else:
             return f"{url}: is Not reachable, status_code: {get.status_code}"
     except requests.exceptions.RequestException as e:
-        raise SystemExit(f"{url}: is Not reachable \nErr: {e}")
+        raise SystemExit(f"{url}: is Not reachable \nErr: {e}") from e
 
 
 def ingest_file_via_rest(filename: str) -> Pipeline:

datahub-actions/src/datahub_actions/plugin/action/propagation/docs/propagation_action.py

@@ -852,4 +852,4 @@ def _propagate_to_siblings(
                     yield maybe_mcp
 
     def close(self) -> None:
-        return super().close()
+        return

datahub-actions/src/datahub_actions/plugin/action/snowflake/snowflake_util.py

@@ -127,4 +127,4 @@ def run_query(self, database: str, schema: str, query: str) -> None:
             )
 
     def close(self) -> None:
-        return super().close()
+        return

datahub-actions/src/datahub_actions/plugin/action/snowflake/tag_propagator.py

@@ -61,7 +61,7 @@ def __init__(self, config: SnowflakeTagPropagatorConfig, ctx: PipelineContext):
 
     def close(self) -> None:
         self.snowflake_tag_helper.close()
-        return super().close()
+        return
 
     @classmethod
     def create(cls, config_dict: dict, ctx: PipelineContext) -> "Action":

datahub-actions/src/datahub_actions/plugin/action/tag/tag_propagation_action.py

@@ -158,4 +158,4 @@ def act(self, event: EventEnvelope) -> None:
                 logger.debug(f"Not propagating {tag_propagation_directive.tag}")
 
     def close(self) -> None:
-        return super().close()
+        return