diff --git a/.github/workflows/pytest-auth.yml b/.github/workflows/pytest-auth.yml index 3b8c10c250..feaeca61bf 100644 --- a/.github/workflows/pytest-auth.yml +++ b/.github/workflows/pytest-auth.yml @@ -54,7 +54,7 @@ jobs: # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -69,7 +69,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-backup_db.yml b/.github/workflows/pytest-dbutils-backup_db.yml index beecf28097..bc677c6b8d 100644 --- a/.github/workflows/pytest-dbutils-backup_db.yml +++ b/.github/workflows/pytest-dbutils-backup_db.yml @@ -55,7 +55,7 @@ jobs: # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -75,7 +75,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-influxdbfuncts.yml b/.github/workflows/pytest-dbutils-influxdbfuncts.yml index ff3dcc5552..47b335e9e8 100644 --- a/.github/workflows/pytest-dbutils-influxdbfuncts.yml +++ b/.github/workflows/pytest-dbutils-influxdbfuncts.yml @@ -44,7 +44,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -64,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-mysqlfuncts.yml b/.github/workflows/pytest-dbutils-mysqlfuncts.yml index 37e3068ffc..6454fa4a86 100644 --- a/.github/workflows/pytest-dbutils-mysqlfuncts.yml +++ b/.github/workflows/pytest-dbutils-mysqlfuncts.yml @@ -44,7 +44,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -64,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-postgresqlfuncts.yml b/.github/workflows/pytest-dbutils-postgresqlfuncts.yml index 0398771cdb..87499a2384 100644 --- a/.github/workflows/pytest-dbutils-postgresqlfuncts.yml +++ b/.github/workflows/pytest-dbutils-postgresqlfuncts.yml @@ -44,7 +44,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -64,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-sqlitefuncts.yml b/.github/workflows/pytest-dbutils-sqlitefuncts.yml index 3848e6efa8..e5bb742d7d 100644 --- a/.github/workflows/pytest-dbutils-sqlitefuncts.yml +++ b/.github/workflows/pytest-dbutils-sqlitefuncts.yml @@ -43,7 +43,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -63,7 +63,7 @@ jobs: test_output_suffix: ${{ env.OUTPUT_SUFFIX }} - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-timescaldbfuncts.yml b/.github/workflows/pytest-dbutils-timescaldbfuncts.yml index 1aeb461487..7639d195c7 100644 --- a/.github/workflows/pytest-dbutils-timescaldbfuncts.yml +++ b/.github/workflows/pytest-dbutils-timescaldbfuncts.yml @@ -43,7 +43,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -63,7 +63,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-miscellaneous-tests.yml b/.github/workflows/pytest-miscellaneous-tests.yml index 9fc023b5a9..80dc6d8d27 100644 --- a/.github/workflows/pytest-miscellaneous-tests.yml +++ b/.github/workflows/pytest-miscellaneous-tests.yml @@ -51,7 +51,7 @@ jobs: # Setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -98,7 +98,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-testutils.yml b/.github/workflows/pytest-testutils.yml index 39cebe91aa..fa1a06887b 100644 --- a/.github/workflows/pytest-testutils.yml +++ b/.github/workflows/pytest-testutils.yml @@ -42,7 +42,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -57,7 +57,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-vctl.yml b/.github/workflows/pytest-vctl.yml index d0d5d868fb..b541753236 100644 --- a/.github/workflows/pytest-vctl.yml +++ b/.github/workflows/pytest-vctl.yml @@ -53,7 +53,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -68,7 +68,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-web.yml b/.github/workflows/pytest-web.yml index 482fbe3160..fe0496f35a 100644 --- a/.github/workflows/pytest-web.yml +++ b/.github/workflows/pytest-web.yml @@ -51,13 +51,13 @@ jobs: # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v7 + uses: volttron/volttron-build-action@v6 with: python_version: ${{ matrix.python-version }} os: ${{ matrix.os }} @@ -66,7 +66,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/README.md b/README.md index 6999280281..bea8985a36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ ![image](docs/source/files/VOLLTRON_Logo_Black_Horizontal_with_Tagline.png) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/fcf58045b4804edf8f4d3ecde3016f76)](https://app.codacy.com/gh/VOLTTRON/volttron?utm_source=github.com&utm_medium=referral&utm_content=VOLTTRON/volttron&utm_campaign=Badge_Grade_Settings) +# VOLTTRON + +This repository is for the current production VOLTTRON. We are working on VOLTTRON 10 (modular) which is available under +github at https://github.com/eclipse-volttron/. The modular version of VOLTTRON will help ease deployment and support +flexible deployment where in only required agents/applications can be installed, thereby simplifying setup and upgrade +steps for the end user. The VOLTTRON team are currently working on porting agents from monolithic VOLTTRON to the +modular version of VOLTTRON. To know more about modular VOLTTRON, please visit our new documentation site available +at https://eclipse-volttron.readthedocs.io/en/latest/. We would love for you to try it out and give us early +feedback. Also, until our work on modular VOLTTRON is completed, please continue cloning and using this +repository for your production systems. VOLTTRON™ is an open source platform for distributed sensing and control. The platform provides services for collecting and storing data from buildings and diff --git a/bootstrap.py b/bootstrap.py index 6abc0ce359..8121b03c4c 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -123,7 +123,7 @@ def update(operation, verbose=None, offline=False, optional_requirements=[], rab # option_requirements contains wheel as first entry # Build option_requirements separately to pass install options - build_option = '--build-option' if wheeling else '--install-option' + build_option = '--build-option' if wheeling else '--config-settings' for requirement, options in option_requirements: args = [] @@ -227,7 +227,7 @@ def main(argv=sys.argv): sys.exit(77) # Python3 for life! - if sys.version_info.major < 3 or sys.version_info.minor < 6: + if sys.version_info.major < 3 or sys.version_info.minor < 8: sys.stderr.write('error: Python >= 3.8 is required\n') sys.exit(1) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py b/deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py similarity index 100% rename from services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py rename to deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py diff --git a/deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py diff --git a/deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py diff --git a/deprecated/OldDnp3/OldDnp3Driver/dnp3-driver.rst b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/dnp3-driver.rst similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driver/dnp3-driver.rst rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/dnp3-driver.rst diff --git a/deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv diff --git a/deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config diff --git a/docs/source/conf.py b/docs/source/conf.py index 80eaeb23f6..91ebf436cf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,12 +16,14 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) -from glob import glob -from mock import Mock as MagicMock import os import subprocess import sys +from glob import glob + import yaml +from mock import Mock as MagicMock + # Copied from volttron.platform.agent.util because it is not required # that volttron be installed to utilize this script. @@ -69,14 +71,17 @@ def execute_command(cmds, class Mock(MagicMock): + @classmethod def __getattr__(cls, name): - return Mock() + return Mock() -autodoc_mock_imports = ['loadshape', 'numpy', 'sympy', 'xlrd', 'stomp', 'oadr2', 'pyodbc', 'lxml', 'pytest', - 'pint', 'pandas', 'suds', 'paho', 'pymongo', 'bson', 'subprocess32', 'heaters', 'meters', - 'hvac', 'blinds', 'vehicles'] +autodoc_mock_imports = [ + 'loadshape', 'numpy', 'sympy', 'xlrd', 'stomp', 'oadr2', 'pyodbc', 'lxml', + 'pytest', 'pint', 'pandas', 'suds', 'paho', 'pymongo', 'bson', + 'subprocess32', 'heaters', 'meters', 'hvac', 'blinds', 'vehicles' +] # -- Project information ----------------------------------------------------- @@ -85,10 +90,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '9.0' +version = '9.0.3' # The full version, including alpha/beta/rc tags -release = '9.0-rc' - +release = '9.0.3' # -- General configuration --------------------------------------------------- @@ -151,7 +155,6 @@ def __getattr__(cls, name): # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -180,13 +183,11 @@ def __getattr__(cls, name): # # html_sidebars = {} - # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'VOLTTRONdoc' - # -- Options for LaTeX output ------------------------------------------------ latex_elements = { @@ -215,16 +216,11 @@ def __getattr__(cls, name): 'The VOLTTRON Community', 'manual'), ] - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (main_doc, 'volttron', 'VOLTTRON Documentation', - [author], 1) -] - +man_pages = [(main_doc, 'volttron', 'VOLTTRON Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -232,20 +228,22 @@ def __getattr__(cls, name): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (main_doc, 'VOLTTRON', 'VOLTTRON Documentation', - author, 'VOLTTRON', 'One line description of project.', - 'Miscellaneous'), + (main_doc, 'VOLTTRON', 'VOLTTRON Documentation', author, 'VOLTTRON', + 'One line description of project.', 'Miscellaneous'), ] - # -- Extension configuration ------------------------------------------------- # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3.6': None, - 'volttron-ansible': ('https://volttron.readthedocs.io/projects/volttron-ansible/en/main/', - None)} +intersphinx_mapping = { + 'https://docs.python.org/3.6': + None, + 'volttron-ansible': + ('https://volttron.readthedocs.io/projects/volttron-ansible/en/main/', + None) +} # -- Options for todo extension ---------------------------------------------- @@ -290,11 +288,15 @@ def generate_apidoc(app): # generate api-docs for each api docs directory for docs_subdir in config.keys(): docs_subdir_path = os.path.join(apidocs_base_dir, docs_subdir) - agent_dirs = glob(os.path.join(volttron_root, config[docs_subdir]["path"], "*/")) + agent_dirs = glob( + os.path.join(volttron_root, config[docs_subdir]["path"], "*/")) file_excludes = [] if config[docs_subdir].get("file_excludes"): - for exclude_pattern in config[docs_subdir].get("file_excludes", []): - file_excludes.append(os.path.join(volttron_root, config[docs_subdir]["path"], exclude_pattern)) + for exclude_pattern in config[docs_subdir].get( + "file_excludes", []): + file_excludes.append( + os.path.join(volttron_root, config[docs_subdir]["path"], + exclude_pattern)) print("after file excludes. calling apidoc") agent_excludes = \ config[docs_subdir].get("agent_excludes") if config[docs_subdir].get("agent_excludes", []) else [] @@ -311,19 +313,24 @@ def run_apidoc(docs_dir, agent_dirs, agent_excludes, exclude_pattern): :param agent_excludes: agent directories to be skipped :param exclude_pattern: file name patterns to be excluded. This passed on to sphinx-apidoc command for exclude """ - print(f"In run apidoc params {docs_dir}, {agent_dirs}, {agent_excludes}, {exclude_pattern}") + print( + f"In run apidoc params {docs_dir}, {agent_dirs}, {agent_excludes}, {exclude_pattern}" + ) for agent_src_dir in agent_dirs: agent_src_dir = os.path.abspath(agent_src_dir) - agent_src_dir = agent_src_dir[:-1] if agent_src_dir.endswith("/") else agent_src_dir + agent_src_dir = agent_src_dir[:-1] if agent_src_dir.endswith( + "/") else agent_src_dir name = os.path.basename(agent_src_dir) agent_doc_dir = os.path.join(docs_dir, name) if name not in agent_excludes: sys.path.insert(0, agent_src_dir) - cmd = ["sphinx-apidoc", '-e', '-a', '-M', '-d 4', - '-t', os.path.join(script_dir, 'apidocs-templates'), - '--force', '-o', agent_doc_dir, agent_src_dir, - os.path.join(agent_src_dir, "setup.py"), os.path.join(agent_src_dir, "conftest.py") - ] + cmd = [ + "sphinx-apidoc", '-e', '-a', '-M', '-d 4', '-t', + os.path.join(script_dir, 'apidocs-templates'), '--force', '-o', + agent_doc_dir, agent_src_dir, + os.path.join(agent_src_dir, "setup.py"), + os.path.join(agent_src_dir, "conftest.py") + ] cmd.extend(exclude_pattern) subprocess.check_call(cmd) @@ -363,5 +370,6 @@ def clean_api_rst(app, exception): global apidocs_base_dir import shutil if os.path.exists(apidocs_base_dir): - print("Cleanup: Removing apidocs directory {}".format(apidocs_base_dir)) + print( + "Cleanup: Removing apidocs directory {}".format(apidocs_base_dir)) shutil.rmtree(apidocs_base_dir) diff --git a/docs/source/developing-volttron/community.rst b/docs/source/developing-volttron/community.rst index 05f32dde04..24718c7d1c 100644 --- a/docs/source/developing-volttron/community.rst +++ b/docs/source/developing-volttron/community.rst @@ -12,18 +12,10 @@ Contributing back to the project, which is encouraged but not required, enhances To learn more, check out :ref:`Contributing ` and :ref:`Documentation `. -Slack Channel -============= - -volttron-community.slack.com is where the |VOLTTRON| community at large can ask questions and meet with others -using |VOLTTRON|. To be added to Slack please email the VOLTTRON team at -`volttron@pnnl.gov `__. - - Mailing List ============ -Join the mailing list by emailing `volttron@pnnl.gov `__. +Join the mailing list at Eclipse: https://projects.eclipse.org/projects/iot.volttron/contact Stack Overflow @@ -39,8 +31,8 @@ Office Hours PNNL hosts office hours every other week on Fridays at 11 AM (PST). These meetings are designed to be very informal where VOLTTRON developers can answer specific questions about the inner workings of VOLTTRON. These meetings are also available for topical discussions of different aspects of the VOLTTRON platform. Currently the office hours are -available through a Zoom meeting. To be invited to the link meeting, contact the volttron team via email: -``__ +available through a Zoom meeting. All members of the mailing list will receive invites to the meetings. Join the +mailing list https://projects.eclipse.org/projects/iot.volttron/contact. Meetings are recorded and can be reviewed `here `__. diff --git a/docs/source/platform-features/web-api/introduction.rst b/docs/source/platform-features/web-api/introduction.rst index 0713d0e895..de138c96d9 100644 --- a/docs/source/platform-features/web-api/introduction.rst +++ b/docs/source/platform-features/web-api/introduction.rst @@ -105,8 +105,14 @@ set to ``https://localhost:8443`` the following HTTP request (with a proper GET https://localhost:8443/vui/ -Access to the API may be disabled by removing "vui" from the list of groups in ``$VOLTTRON_HOME/web-users.json`` for any user which should not have access -to the API. +Access to the API may be disabled by removing "vui" from the list of groups in ``$VOLTTRON_HOME/web-users.json`` for +any user which should not have access to the API. + +.. Note:: + Certain features of the API require that appropriate agents be installed to handle requests. In particular, + devices endpoints require at least the Platform Driver Agent and the Actuator Agent to both be installed. + For use with BACnet devices, the BACnet Proxy Agent may also be required. Missing agents are likely to result + in timeout errors from the API as the request has been sent to an agent which is not there to respond. Path Structure --------------- diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 06556a1356..0000000000 --- a/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[tool.yapfignore] -ignore_patterns = [ - ".env/**", - ".pytest_cache/**", - "dist/**", - "docs/**", -] - -[tool.yapf] -based_on_style = "pep8" -spaces_before_comment = 4 -column_limit = 99 -split_before_logical_operator = true \ No newline at end of file diff --git a/requirements.py b/requirements.py index 80fa44c40e..727f98bd31 100644 --- a/requirements.py +++ b/requirements.py @@ -30,10 +30,10 @@ # wheel version 0.31 has removed metadata.json file # https://github.com/pypa/wheel/issues/195 # so sticking to 0.30 for now. Could upgrade to wheel 0.31 with code changes -option_requirements = [('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] +option_requirements = [('pip==24.0', []), ('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] -install_requires = ['gevent==21.12.0', +install_requires = ['gevent==23.9.1', 'grequests==0.6.0', 'requests==2.23.0', 'idna<3,>=2.5', @@ -42,11 +42,12 @@ 'python-dateutil==2.8.2', 'pytz==2022.1', 'PyYAML==6.0', - 'setuptools>=40.0.0', + 'setuptools>=40.0.0,<=70.0.0', # tzlocal 3.0 breaks without the backports.tzinfo package on python < 3.9 https://pypi.org/project/tzlocal/3.0/ 'tzlocal==2.1', #'pyOpenSSL==19.0.0', 'cryptography==37.0.4', + 'watchdog<5.0', 'watchdog-gevent==0.1.1', 'deprecated==1.2.14'] diff --git a/services/core/ActuatorAgent/actuator/agent.py b/services/core/ActuatorAgent/actuator/agent.py index 369489ed75..82f666c9ca 100644 --- a/services/core/ActuatorAgent/actuator/agent.py +++ b/services/core/ActuatorAgent/actuator/agent.py @@ -1382,11 +1382,8 @@ def _request_new_schedule(self, sender, task_id, priority, requests, publish_res 'data': {'agentID': sender, 'taskID': task_id}}) - # If we are successful we do something else with the real result data - data = result.data if not result.success else {} - results = {'result': success, - 'data': data, + 'data': result.data, 'info': result.info_string} if publish_result: diff --git a/services/core/ActuatorAgent/actuator/scheduler.py b/services/core/ActuatorAgent/actuator/scheduler.py index c1190e5491..b2fcc4ae52 100644 --- a/services/core/ActuatorAgent/actuator/scheduler.py +++ b/services/core/ActuatorAgent/actuator/scheduler.py @@ -21,14 +21,14 @@ # # ===----------------------------------------------------------------------=== # }}} - - import bisect import logging -from pickle import dumps, loads + +from base64 import b64encode from collections import defaultdict, namedtuple from copy import deepcopy from datetime import timedelta +from pickle import dumps, loads from volttron.platform.agent import utils @@ -340,7 +340,7 @@ def save_state(self, now): try: self._cleanup(now) - self.save_state_callback(dumps(self.tasks)) + self.save_state_callback(b64encode(dumps(self.tasks)).decode("utf-8")) except Exception: _log.error('Failed to save scheduler state!') @@ -411,7 +411,10 @@ def request_slots(self, agent_id, id_, requests, priority, now=None): self.save_state(now) - return RequestResult(True, preempted_tasks, '') + if preempted_tasks: + return RequestResult(True, list(preempted_tasks), 'TASK_WERE_PREEMPTED') + else: + return RequestResult(True, {}, '') def cancel_task(self, agent_id, task_id, now): if task_id not in self.tasks: diff --git a/services/core/IEEE_2030_5/requirements_demo.txt b/services/core/IEEE_2030_5/requirements_demo.txt index cf84bba01d..0fdf400704 100644 --- a/services/core/IEEE_2030_5/requirements_demo.txt +++ b/services/core/IEEE_2030_5/requirements_demo.txt @@ -2,3 +2,4 @@ nicegui requests xsdata>=23.8 blinker +pandas diff --git a/services/core/PlatformDriverAgent/platform_driver/agent.py b/services/core/PlatformDriverAgent/platform_driver/agent.py index 967e2c81f1..5c3d097433 100644 --- a/services/core/PlatformDriverAgent/platform_driver/agent.py +++ b/services/core/PlatformDriverAgent/platform_driver/agent.py @@ -487,7 +487,10 @@ def heart_beat(self): """ _log.debug("sending heartbeat") for device in self.instances.values(): - device.heart_beat() + try: + device.heart_beat() + except (Exception, gevent.Timeout) as e: + _log.warning(f'Failed to set heart_beat point on device: {device.device_name} -- {e}.') @RPC.export def revert_point(self, path, point_name, **kwargs): diff --git a/services/core/PlatformDriverAgent/platform_driver/driver.py b/services/core/PlatformDriverAgent/platform_driver/driver.py index b86688f465..8fe1e37e30 100644 --- a/services/core/PlatformDriverAgent/platform_driver/driver.py +++ b/services/core/PlatformDriverAgent/platform_driver/driver.py @@ -326,29 +326,47 @@ def get_paths_for_point(self, point): return depth_first, breadth_first def get_point(self, point_name, **kwargs): - return self.interface.get_point(point_name, **kwargs) + try: + return self.interface.get_point(point_name, **kwargs) + except AttributeError as e: + _log.warning(e) + def set_point(self, point_name, value, **kwargs): - return self.interface.set_point(point_name, value, **kwargs) + try: + return self.interface.set_point(point_name, value, **kwargs) + except AttributeError as e: + _log.warning(e) def scrape_all(self): - return self.interface.scrape_all() + try: + return self.interface.scrape_all() + except AttributeError as e: + _log.warning(e) def get_multiple_points(self, point_names, **kwargs): - return self.interface.get_multiple_points(self.device_name, - point_names, - **kwargs) + try: + return self.interface.get_multiple_points(self.device_name, point_names, **kwargs) + except AttributeError as e: + _log.warning(e) def set_multiple_points(self, point_names_values, **kwargs): - return self.interface.set_multiple_points(self.device_name, - point_names_values, - **kwargs) + try: + return self.interface.set_multiple_points(self.device_name, point_names_values, **kwargs) + except AttributeError as e: + _log.warning(e) def revert_point(self, point_name, **kwargs): - self.interface.revert_point(point_name, **kwargs) + try: + self.interface.revert_point(point_name, **kwargs) + except AttributeError as e: + _log.warning(e) def revert_all(self, **kwargs): - self.interface.revert_all(**kwargs) + try: + self.interface.revert_all(**kwargs) + except AttributeError as e: + _log.warning(e) def publish_cov_value(self, point_name, point_values): """ diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py index 128bc617d1..e2713a65f8 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py @@ -237,12 +237,12 @@ class BaseInterface(object, metaclass=abc.ABCMeta): """ - def __init__(self, vip=None, core=None, **kwargs): + def __init__(self, vip=None, core=None, device_path=None, **kwargs): # Object does not take any arguments to the init. super(BaseInterface, self).__init__() self.vip = vip self.core = core - + self.device_path = device_path self.point_map = {} self.build_register_map() diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index b2fc973e10..370a3d2d9d 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -22,7 +22,7 @@ # ===----------------------------------------------------------------------=== # }}} - +import gevent import logging from datetime import datetime, timedelta @@ -97,10 +97,10 @@ def ping_target(self): self.vip.rpc.call(self.proxy_address, 'ping_device', self.target_address, self.device_id).get(timeout=self.timeout) pinged = True except errors.Unreachable: - _log.warning("Unable to reach BACnet proxy.") - - except errors.VIPError: - _log.warning("Error trying to ping device.") + _log.warning(f"Unable to reach BACnet proxy at: {self.proxy_address}.") + except (Exception, gevent.Timeout) as e: + _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}" + f"through proxy {self.proxy_address}: {e}") self.scheduled_ping = None @@ -108,16 +108,25 @@ def ping_target(self): if not pinged: self.schedule_ping() - def get_point(self, point_name, get_priority_array=False): + def get_point(self, point_name, on_property=None): register = self.get_register_by_name(point_name) - property_name = "priorityArray" if get_priority_array else register.property - register_index = None if get_priority_array else register.index - result = self.vip.rpc.call(self.proxy_address, 'read_property', - self.target_address, register.object_type, - register.instance_number, property_name, register_index).get(timeout=self.timeout) + if on_property is None: + result = self.vip.rpc.call(self.proxy_address, 'read_property', + self.target_address, register.object_type, + register.instance_number, register.property, register.index).get(timeout=self.timeout) + else: + point_map = {} + point_map[register.point_name] = [register.object_type, + register.instance_number, + on_property, + register.index] + result = self.vip.rpc.call(self.proxy_address, 'read_properties', + self.target_address, point_map, + self.max_per_request, True).get(timeout=self.timeout) + result = list(result.values())[0] return result - def set_point(self, point_name, value, priority=None): + def set_point(self, point_name, value, priority=None, on_property=None): # TODO: support writing from an array. register = self.get_register_by_name(point_name) if register.read_only: @@ -130,7 +139,7 @@ def set_point(self, point_name, value, priority=None): args = [self.target_address, value, register.object_type, register.instance_number, - register.property, + on_property if on_property is not None else register.property, priority if priority is not None else register.priority, register.index] result = self.vip.rpc.call(self.proxy_address, 'write_property', *args).get(timeout=self.timeout) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst index 218b1dba8d..4da6ce77a3 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst @@ -12,7 +12,7 @@ activated environment: :: - pip install suds-jurko + pip install zeep Alternatively requirements can be installed from requirements.txt using: diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index 0f8d26e5bb..a106540650 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -27,12 +27,11 @@ import logging import abc import sys - from . import service as cps -from . import async_service as async - +from . import async_service as async_service from .. import BaseInterface, BaseRegister, BasicRevert, DriverInterfaceError -from suds.sudsobject import asdict +#from suds.sudsobject import asdict +from zeep.helpers import serialize_object _log = logging.getLogger(__name__) @@ -54,7 +53,7 @@ point_name_mapping = {"Status.TimeStamp": "TimeStamp"} service = {} -gevent.spawn(async.web_service) +gevent.spawn(async_service.web_service) def recursive_asdict(d): @@ -64,7 +63,7 @@ def recursive_asdict(d): http://stackoverflow.com/questions/2412486/serializing-a-suds-object-in-python """ out = {} - for k, v in asdict(d).items(): + for k, v in serialize_object(d, dict).items(): if hasattr(v, '__keylist__'): out[k] = recursive_asdict(v) elif isinstance(v, list): @@ -139,6 +138,16 @@ def read_only_check(self): raise IOError("Trying to write to a point configured read only: {0}".format(self.attribute_name)) return True + def get_last_non_none_value(self,lst): + """ + Depends on port number, the result could be a list with None value + get last non-None value as result + """ + for item in reversed(lst): + if item is not None: + return item + return None + def get_register(self, result, method, port_flag=True): """Gets correct register from API response. @@ -151,9 +160,10 @@ def get_register(self, result, method, port_flag=True): :return: Correct register value cast to appropriate python type. Returns None if there is an error. """ try: - value = getattr(result, self.attribute_name)(self.port)[0] \ + _log.debug(f'In get_register, to get {self.attribute_name}, the port_flag is {port_flag}') + value = self.get_last_non_none_value(getattr(result, self.attribute_name)(self.port)) \ if port_flag \ - else getattr(result, self.attribute_name)(None)[0] + else self.get_last_non_none_value(getattr(result, self.attribute_name)(None)) return self.sanitize_output(self.data_type, value) except cps.CPAPIException as exception: if exception._responseCode not in ['153']: @@ -196,7 +206,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStations - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() return self.get_register(result.value, method) @@ -237,7 +247,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getLoad - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() return self.get_register(result.value, method) @@ -263,7 +273,7 @@ def value(self, x): kwargs = {'stationID': self.station_id} if self.attribute_name == 'shedState' and not value: method = service[self.username].clearShedState - result = async.CPRequest.request(method, 0, stationID=self.station_id) + result = async_service.CPRequest.request(method, 0, stationID=self.station_id) elif self.attribute_name == 'shedState': _log.error('shedState may only be written with value 0. If you want to shedLoad, write to ' 'allowedLoad or percentShed') @@ -273,7 +283,7 @@ def value(self, x): kwargs[self.attribute_name] = value if self.port: kwargs['portNumber'] = self.port - result = async.CPRequest.request(method, 0, **kwargs) + result = async_service.CPRequest.request(method, 0, **kwargs) result.wait() if result.value.responseCode != "100": @@ -322,7 +332,7 @@ def value(self): if self.port: kwargs['portNumber'] = self.port - result = async.CPRequest.request(method, self.timeout, **kwargs) + result = async_service.CPRequest.request(method, self.timeout, **kwargs) result.wait() return self.get_register(result.value, method, False) @@ -348,7 +358,7 @@ def value(self, x): if self.attribute_name == 'clearAlarms' and value: kwargs = {'stationID': self.station_id} method = service[self.username].clearAlarms - result = async.CPRequest.request(method, 0, **kwargs) + result = async_service.CPRequest.request(method, 0, **kwargs) result.wait() if result.value.responseCode not in ['100', '153']: @@ -383,11 +393,12 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getChargingSessionData - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() # Of Note, due to API limitations, port number is ignored for these calls - return self.get_register(result.value, method, False) + # NOTE: Change this port number for Chargingsession data. + return self.get_register(result.value, method) @value.setter def value(self, x): @@ -418,7 +429,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStationStatus - result = async.CPRequest.request(method, self.timeout, self.station_id) + result = async_service.CPRequest.request(method, self.timeout, self.station_id) result.wait() return self.get_register(result.value, method) @@ -455,7 +466,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStationRights - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() # Note: this does not go through get_register, as it is of a unique type, 'dictionary.' @@ -558,7 +569,7 @@ def parse_config(self, config_dict, registry_config_str): description=description, port_number=port_num, username=config_dict['username'], - timeout=config_dict['cacheExpiration'] + timeout=config_dict.get('cacheExpiration',0) ) self.insert_register(register) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py index ca98c8a753..e142a30d86 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py @@ -47,15 +47,14 @@ import gevent.event import gevent.queue import logging -import suds +import zeep from gevent import monkey from .service import CPAPIException from datetime import datetime, timedelta -monkey.patch_all() _log = logging.getLogger(__name__) -SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" +SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.1.wsdl" # Queue for Web API requests and responses. It is managed by the long running # web_service() greenlet. web_service_queue = gevent.queue.Queue() @@ -253,7 +252,7 @@ def web_service(): web_cache[item_key] = cache_item if not client_set: - client_set.add(suds.client.Client(SERVICE_WSDL_URL)) + client_set.add(zeep.Client(SERVICE_WSDL_URL)) client = client_set.pop() gevent.spawn(web_call, item, client) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py index 9687d08d63..e06c51b25d 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py @@ -1,5 +1,5 @@ from . import service as cps -import suds +import zeep import io station_csv = { @@ -176,5 +176,5 @@ else: print("Some other error happened") - except suds.WebFault as a: + except zeep.exception.Fault as e: print("Sorry, your API credentials are invalid. Please contact Chargepoint for assistance.") diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt index 274ad9e069..f61b24a644 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt @@ -1 +1 @@ -suds-jurko==0.6 \ No newline at end of file +zeep==4.2.1 \ No newline at end of file diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index da845c676e..f9b404a723 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -22,14 +22,14 @@ # ===----------------------------------------------------------------------=== # }}} -import suds.client -import suds.wsse +import zeep +from zeep.wsse.username import UsernameToken +from zeep import Settings import logging logger = logging.getLogger('chargepoint') -SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" - +SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.1.wsdl" CPAPI_SUCCESS = '100' XMPP_EVENTS = [ @@ -156,13 +156,13 @@ class CPStation: """Wrapper around the getStations() return by Chargepoint API. Data surrounding a Chargepoint Station can generally be categorized as static or dynamic. Chargepoint API has two - basic calls, getLoad and getStation, that each return station data. getLoad returns the stationLoadData SUDS - object, and getStation returns the stationDataExtended SUDS object. These are each kept as separate meta-data + basic calls, getLoad and getStation, that each return station data. getLoad returns the stationLoadData object, + and getStation returns the stationDataExtended object. These are each kept as separate meta-data parameters. :param cps: Chargepoint Service object. - :param sld: stationLoadData SUDS object. - :param sde: stationDataExtended SUDS object. + :param sld: stationLoadData object. + :param sde: stationDataExtended object. (stationDataExtended){ stationID = "1:00001" @@ -416,8 +416,17 @@ def get_port_value(port_number, data, attribute): if flag: logger.warning("Station does not have a definition for port {0}".format(port_number)) else: - logger.warning("Response does not have Ports defined") - return None + if (attribute in ['sessionID', 'startTime', 'endTime', 'Energy', 'rfidSerialNumber', 'driverAccountNumber', + 'driverName']) and int(data['portNumber']) == port_number: + try: + data_attribute = data[attribute] + return data_attribute + except: + logger.warning(f'Response does not have {attribute} field') + return None + else: + logger.warning("Response does not have Ports defined") + return None @staticmethod def check_output(attribute, parent_dict): @@ -441,6 +450,7 @@ def get_attr_from_response(name_string, response, portNum=None): else CPAPIResponse.is_not_found(name_string)) else: list.append(CPAPIResponse.get_port_value(portNum, item, name_string)) + logger.debug(f'{name_string} list for {portNum} is {list}') return list @@ -632,12 +642,14 @@ def Type(self, port=None): def startTime(self, port=None): if port: + logger.debug(f'startTime port is {port}') return CPAPIResponse.get_attr_from_response('startTime', self.stations, port) else: return [self.pricing_helper('startTime', station) for station in self.stations] def endTime(self, port=None): if port: + logger.debug(f'endTime port is {port}') return CPAPIResponse.get_attr_from_response('endTime', self.stations, port) else: return [self.pricing_helper('endTime', station) for station in self.stations] @@ -758,8 +770,8 @@ class CPService: """ Python wrapper around the Chargepoint WebServices API. - Current Version: 5.0 - Docs: ChargePoint_Web_Services_API_Guide_Ver4.1_Rev5.pdf + Current Version: 5.1 + Docs: ChargePoint_Web_Services_API_Guide_Ver5.1_Rev1.13.pdf """ def __init__(self, username=None, password=None): @@ -769,18 +781,18 @@ def __init__(self, username=None, password=None): """ self._username = username self._password = password - self._suds_client = None + self._zeep_client = None @property def _client(self): - """Initialize the SUDS client if necessary.""" + """Initialize the ZEEP client if necessary.""" - if self._suds_client is None: - self._suds_client = suds.client.Client(SERVICE_WSDL_URL) + if self._zeep_client is None: + self._zeep_client = zeep.Client(SERVICE_WSDL_URL) # Add SOAP Security tokens self.set_security_token() - return self._suds_client + return self._zeep_client @property def _soap_service(self): @@ -788,13 +800,13 @@ def _soap_service(self): def set_security_token(self): # Add SOAP Security tokens - security = suds.wsse.Security() - token = suds.wsse.UsernameToken(self._username, self._password) - security.tokens.append(token) - self._suds_client.set_options(wsse=security) + #TODO:might need to put this in config + #NOTE: wihtout this setting, zeep will not get result + settins = Settings(strict=False, xml_huge_tree=True, xsd_ignore_sequence_order=True) + self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password),settings=settins) def set_client(self, client): - self._suds_client = client + self._zeep_client = client self.set_security_token() def clearAlarms(self, **kwargs): @@ -820,7 +832,7 @@ def clearAlarms(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('clearAlarmsSearchQuery') + searchQuery = self._client.get_type('ns0:clearAlarmsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.clearAlarms(searchQuery) @@ -836,7 +848,7 @@ def clearShedState(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('shedQueryInputData') + searchQuery = self._client.get_type('ns0:shedQueryInputData')() if 'stationID' in kwargs.keys(): setattr(searchQuery, 'shedStation', {'stationID': kwargs['stationID']}) elif 'sgID' in kwargs.keys(): @@ -890,7 +902,7 @@ def getAlarms(self, **kwargs): } """ - searchQuery = self._client.factory.create('getAlarmsSearchQuery') + searchQuery = self._client.get_type('ns0:getAlarmsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getAlarms(searchQuery) @@ -965,7 +977,7 @@ def getChargingSessionData(self, **kwargs): } """ - searchQuery = self._client.factory.create('sessionSearchdata') + searchQuery = self._client.get_type('ns0:sessionSearchdata')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getChargingSessionData(searchQuery) @@ -1018,7 +1030,8 @@ def getLoad(self, **kwargs): """ # @ToDo: Figure out what type of request searchQuery should be here. - searchQuery = self._client.factory.create('stationSearchRequestExtended') + # @Note: Looks like it should be {sgID: xsd:int, stationID: xsd:string, sessionID: xsd:long} + searchQuery = {} for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getLoad(searchQuery) @@ -1059,7 +1072,7 @@ def getOrgsAndStationGroups(self, **kwargs): } """ - searchQuery = self._client.factory.create('getOrgsAndStationGroupsSearchQuery') + searchQuery = self._client.get_type('ns0:getOrgsAndStationGroupsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getOrgsAndStationGroups(searchQuery) @@ -1214,7 +1227,7 @@ def getStationRights(self, **kwargs): } """ - searchQuery = self._client.factory.create('stationRightsSearchRequest') + searchQuery = self._client.get_type('ns0:stationRightsSearchRequest')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getStationRights(searchQuery) @@ -1359,7 +1372,8 @@ def getStations(self, **kwargs): moreFlag = 0 } """ - searchQuery = self._client.factory.create('stationSearchRequestExtended') + + searchQuery = self._client.get_type('ns0:stationSearchRequestExtended')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getStations(searchQuery) @@ -1446,7 +1460,7 @@ def getUsers(self, **kwargs): } """ - searchQuery = self._client.factory.create('getUsersSearchRequest') + searchQuery = self._client.get_type('ns0:getUsersSearchRequest')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getUsers(searchQuery) @@ -1484,7 +1498,8 @@ def shedLoad(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('shedLoadQueryInputData') + + searchQuery = self._client.get_type('ns0:shedLoadQueryInputData')() port = kwargs.pop('portNumber', None) query_params = {'stationID': kwargs['stationID']} if port: diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index beea69b023..6581e3cff2 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -669,9 +669,10 @@ def read_request(self, request): ) self._data.update(request.parse_values(results)) except (AttributeError, ModbusError) as err: - if "Exception code" in err.message: - raise Exception("{0}: {1}".format(err.message, - helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED"))) + if err is ModbusError: + code = err.get_exception_code() + raise Exception(f'{err.args[0]}, {helpers.TABLE_EXCEPTION_CODE.get(code, "UNDEFINED")}') + logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) def read_all(self): diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py index 5c7a868b66..0ac107061f 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py @@ -387,7 +387,7 @@ def scrape_all(self, agent, device_name): return agent.vip.rpc.call(PLATFORM_DRIVER, 'scrape_all', device_name)\ .get(timeout=10) - @pytest.mark.xfail(is_running_in_container(), reason='Fails to set points on this test setup, only in Docker.') + @pytest.mark.skip('This test has been unreliable.') def test_scrape_all(self, agent): for key in registers_dict.keys(): self.set_point(agent, 'modbus_tk', key, registers_dict[key]) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py index ea58ded782..593c2a684f 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py @@ -2,6 +2,9 @@ import gevent import pytest + +from pathlib import Path + from volttron.platform.agent.known_identities import CONFIGURATION_STORE, PLATFORM_DRIVER from volttron.platform import jsonapi from volttrontesting.utils.platformwrapper import PlatformWrapper diff --git a/services/core/PlatformDriverAgent/tests/test_bacnet.py b/services/core/PlatformDriverAgent/tests/test_bacnet.py index 6c608e42e9..476a4fc5f2 100644 --- a/services/core/PlatformDriverAgent/tests/test_bacnet.py +++ b/services/core/PlatformDriverAgent/tests/test_bacnet.py @@ -77,7 +77,15 @@ def test_get_point_should_succeed(bacnet_test_agent): @pytest.fixture(scope="module") def bacnet_proxy_agent(volttron_instance): - device_address = socket.gethostbyname(socket.gethostname() + ".local") + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(0) + try: + s.connect(('8.8.8.8', 1)) + device_address = s.getsockname()[0] + except Exception: + device_address = '127.0.0.1' + finally: + s.close() print(f"Device address for proxy agent for testing: {device_address}") bacnet_proxy_agent_config = { "device_address": device_address, diff --git a/services/ops/EmailerAgent/emailer/agent.py b/services/ops/EmailerAgent/emailer/agent.py index 00cdef0b14..cf0dd5de77 100644 --- a/services/ops/EmailerAgent/emailer/agent.py +++ b/services/ops/EmailerAgent/emailer/agent.py @@ -25,6 +25,7 @@ # Import the email modules we'll need from email.mime.text import MIMEText +from smtplib import SMTPException import logging import socket @@ -251,17 +252,27 @@ def _send_email(self, from_address, to_addresses, mime_message): self.vip.health.set_status(STATUS_GOOD, "Successfully sent email.") send_successful = True + except SMTPException as smtp_err: + _log.error(f"SMTP error occurred: {smtp_err}") + _log.error(f"Unable to send email message: {mime_message.as_string()}") + self.vip.health.set_status( + STATUS_BAD, + "SMTP configuration or authentication issue. Please check your SMTP settings and credentials.") + + except OSError as os_err: + _log.error(f"Network-related error occurred: {os_err}") + _log.error(f"Unable to send email message: {mime_message.as_string()}") + self.vip.health.set_status( + STATUS_BAD, "Network issue. Please check your internet connection and SMTP server accessibility.") + except Exception as e: - _log.error( - 'Unable to send email message: %s' % mime_message.as_string()) - _log.error(e.args) - self.vip.health.set_status(STATUS_BAD, - "Unable to send email to recipients") + _log.error(f"An unexpected error occurred: {e}") + _log.error(f"Unable to send email message: {mime_message.as_string()}") + self.vip.health.set_status(STATUS_BAD, f"Unable to send email to recipients: {e}") finally: if sent_email_record is not None: sent_email_record['successful'] = send_successful - self.vip.pubsub.publish("pubsub", "record/sent_email", - message=sent_email_record) + self.vip.pubsub.publish("pubsub", "record/sent_email", message=sent_email_record) def send_email(self, from_address, to_addresses, subject, message): """ diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index b56f6f309a..b7b950618c 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -35,7 +35,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '8.2' +__version__ = '9.0.3' _log = logging.getLogger(__name__) diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index eb9f2ec238..dfa9f7c560 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -466,7 +466,7 @@ def vip_main(agent_class, identity=None, version='0.1', **kwargs): # TODO: Make required for all agents. Handle it through vctl and aip. if not os.environ.get("_LAUNCHED_BY_PLATFORM"): if not publickey or not secretkey: - raise ValueError("AGENT_PUBLIC and AGENT_SECRET environmental variables must " + raise ValueError("AGENT_PUBLICKEY and AGENT_SECRETKEY environmental variables must " "be set to run without the platform.") address = get_address() diff --git a/volttron/platform/aip.py b/volttron/platform/aip.py index dc7859ec89..07e8219126 100644 --- a/volttron/platform/aip.py +++ b/volttron/platform/aip.py @@ -247,6 +247,7 @@ def __init__(self, env, **kwargs): if self.message_bus == 'rmq': self.rmq_mgmt = RabbitMQMgmt() self.instance_name = get_platform_instance_name() + self.agent_uuid_name_map = {} def add_agent_user_group(self): user = pwd.getpwuid(os.getuid()) @@ -682,11 +683,14 @@ def remove_agent(self, agent_uuid, remove_auth=True): self.remove_agent_user(volttron_agent_user) def agent_name(self, agent_uuid): + if cached_name := self.agent_uuid_name_map.get(agent_uuid): + return cached_name agent_path = os.path.join(self.install_dir, agent_uuid) for agent_name in os.listdir(agent_path): dist_info = os.path.join( agent_path, agent_name, agent_name + '.dist-info') if os.path.exists(dist_info): + self.agent_uuid_name_map[agent_uuid] = agent_name return agent_name raise KeyError(agent_uuid) diff --git a/volttron/platform/control/control_parser.py b/volttron/platform/control/control_parser.py index 0c10fd44a9..5a256800d2 100644 --- a/volttron/platform/control/control_parser.py +++ b/volttron/platform/control/control_parser.py @@ -24,27 +24,26 @@ # Monkeypatch for gevent from volttron.utils import monkey_patch + monkey_patch() import argparse import collections import logging -import logging.handlers import logging.config +import logging.handlers import os import sys - -from datetime import timedelta, datetime +from datetime import datetime, timedelta import gevent import gevent.event -# noinspection PyUnresolvedReferences - from volttron.platform import aip as aipmod -from volttron.platform import config -from volttron.platform import get_home, get_address -from volttron.platform import jsonapi +from volttron.platform import config, get_address, get_home, is_rabbitmq_available, jsonapi +from volttron.platform.agent import utils +from volttron.platform.agent.known_identities import PLATFORM_HEALTH +from volttron.platform.agent.utils import is_secure_mode, wait_for_volttron_shutdown from volttron.platform.control.control_auth import add_auth_parser from volttron.platform.control.control_certs import add_certs_parser from volttron.platform.control.control_config import add_config_store_parser @@ -52,25 +51,18 @@ from volttron.platform.control.control_rmq import add_rabbitmq_parser from volttron.platform.control.control_rpc import add_rpc_agent_parser from volttron.platform.control.control_utils import ( - _list_agents, - _show_filtered_agents, - _show_filtered_agents_status, - filter_agent, - filter_agents, - get_filtered_agents - ) -from volttron.platform.agent import utils -from volttron.platform.agent.known_identities import PLATFORM_HEALTH + _list_agents, _show_filtered_agents, _show_filtered_agents_status, + filter_agent, filter_agents, get_filtered_agents) +from volttron.platform.control.install_agents import InstallRuntimeError, add_install_agent_parser from volttron.platform.jsonrpc import RemoteError from volttron.platform.keystore import KeyStore, KnownHostsStore +from volttron.platform.vip.agent.errors import Unreachable, VIPError + +# noinspection PyUnresolvedReferences -from volttron.platform.vip.agent.errors import VIPError, Unreachable -from volttron.platform.agent.utils import is_secure_mode, wait_for_volttron_shutdown -from volttron.platform.control.install_agents import add_install_agent_parser, InstallRuntimeError -from volttron.platform import is_rabbitmq_available if is_rabbitmq_available(): - from volttron.utils.rmq_setup import check_rabbit_status from volttron.utils.rmq_config_params import RMQConfig + from volttron.utils.rmq_setup import check_rabbit_status try: import volttron.restricted @@ -86,8 +78,7 @@ # will be volttron.platform.main or main.py instead of __main__ _log = logging.getLogger( - os.path.basename(sys.argv[0]) if __name__ == "__main__" else __name__ -) + os.path.basename(sys.argv[0]) if __name__ == "__main__" else __name__) # Allows server side logging. # _log.setLevel(logging.DEBUG) @@ -96,16 +87,16 @@ CHUNK_SIZE = 4096 -def log_to_file(file, level=logging.WARNING, + +def log_to_file(file, + level=logging.WARNING, handler_class=logging.StreamHandler): """Direct log output to a file (or something like one).""" handler = handler_class(file) handler.setLevel(level) handler.setFormatter( utils.AgentFormatter( - "%(asctime)s %(composite_name)s %(levelname)s: %(message)s" - ) - ) + "%(asctime)s %(composite_name)s %(levelname)s: %(message)s")) root = logging.getLogger() root.setLevel(level) root.addHandler(handler) @@ -123,17 +114,17 @@ def tag_agent(opts): msg = "multiple agents selected" else: msg = "agent not found" - _stderr.write( - "{}: error: {}: {}\n".format(opts.command, msg, opts.agent)) + _stderr.write("{}: error: {}: {}\n".format(opts.command, msg, + opts.agent)) return 10 - (agent,) = agents + (agent, ) = agents if opts.tag: _stdout.write("Tagging {} {}\n".format(agent.uuid, agent.name)) opts.aip.tag_agent(agent.uuid, opts.tag) elif opts.remove: if agent.tag is not None: - _stdout.write( - "Removing tag for {} {}\n".format(agent.uuid, agent.name)) + _stdout.write("Removing tag for {} {}\n".format( + agent.uuid, agent.name)) opts.aip.tag_agent(agent.uuid, None) else: if agent.tag is not None: @@ -144,27 +135,25 @@ def remove_agent(opts, remove_auth=True): agents = _list_agents(opts.aip) for pattern, match in filter_agents(agents, opts.pattern, opts): if not match: - _stderr.write( - "{}: error: agent not found: {}\n".format(opts.command, - pattern) - ) + _stderr.write("{}: error: agent not found: {}\n".format( + opts.command, pattern)) elif len(match) > 1 and not opts.force: _stderr.write( "{}: error: pattern returned multiple agents: {}\n".format( - opts.command, pattern - ) - ) + opts.command, pattern)) _stderr.write( "Use -f or --force to force removal of multiple agents.\n") return 10 for agent in match: _stdout.write("Removing {} {}\n".format(agent.uuid, agent.name)) - opts.connection.call("remove_agent", agent.uuid, + opts.connection.call("remove_agent", + agent.uuid, remove_auth=remove_auth) # TODO: Remove AIP def list_agents(opts): + def get_priority(agent): return opts.aip.agent_priority(agent.uuid) or "" @@ -193,11 +182,8 @@ def update_health_cache(opts): do_update = True # Make sure we update if we don't have any health dicts, or if the cache # has timed out. - if ( - health_cache_timeout_date is not None - and t_now < health_cache_timeout_date - and health_cache - ): + if (health_cache_timeout_date is not None + and t_now < health_cache_timeout_date and health_cache): do_update = False if do_update: @@ -205,12 +191,10 @@ def update_health_cache(opts): if opts.connection.server: health_cache.update( opts.connection.server.vip.rpc.call( - PLATFORM_HEALTH, "get_platform_health" - ).get(timeout=4) - ) + PLATFORM_HEALTH, "get_platform_health").get(timeout=4)) health_cache_timeout_date = datetime.now() + timedelta( - seconds=health_cache_timeout - ) + seconds=health_cache_timeout) + # TODO: Remove AIP def status_agents(opts): @@ -229,12 +213,14 @@ def status_agents(opts): agent = agents[uuid] agents[uuid] = agent._replace(agent_user=agent_user) except KeyError: - Agent = collections.namedtuple("Agent", - "name tag uuid vip_identity " - "agent_user") - agents[uuid] = agent = Agent( - name, None, uuid, vip_identity=identity, agent_user=agent_user - ) + Agent = collections.namedtuple( + "Agent", "name tag uuid vip_identity " + "agent_user") + agents[uuid] = agent = Agent(name, + None, + uuid, + vip_identity=identity, + agent_user=agent_user) status[uuid] = stat agents = list(agents.values()) @@ -268,6 +254,7 @@ def get_health(agent): _show_filtered_agents_status(opts, get_status, get_health, agents) + #TODO: Remove AIP def agent_health(opts): agents = {agent.uuid: agent for agent in _list_agents(opts.aip)}.values() @@ -295,21 +282,17 @@ def agent_health(opts): def clear_status(opts): opts.connection.call("clear_status", opts.clear_all) + # TODO: Remove AIP def enable_agent(opts): agents = _list_agents(opts.aip) for pattern, match in filter_agents(agents, opts.pattern, opts): if not match: - _stderr.write( - "{}: error: agent not found: {}\n".format(opts.command, - pattern) - ) + _stderr.write("{}: error: agent not found: {}\n".format( + opts.command, pattern)) for agent in match: - _stdout.write( - "Enabling {} {} with priority {}\n".format( - agent.uuid, agent.name, opts.priority - ) - ) + _stdout.write("Enabling {} {} with priority {}\n".format( + agent.uuid, agent.name, opts.priority)) opts.aip.prioritize_agent(agent.uuid, opts.priority) @@ -317,15 +300,13 @@ def disable_agent(opts): agents = _list_agents(opts.aip) for pattern, match in filter_agents(agents, opts.pattern, opts): if not match: - _stderr.write( - "{}: error: agent not found: {}\n".format(opts.command, - pattern) - ) + _stderr.write("{}: error: agent not found: {}\n".format( + opts.command, pattern)) for agent in match: priority = opts.aip.agent_priority(agent.uuid) if priority is not None: - _stdout.write( - "Disabling {} {}\n".format(agent.uuid, agent.name)) + _stdout.write("Disabling {} {}\n".format( + agent.uuid, agent.name)) opts.aip.prioritize_agent(agent.uuid, None) @@ -358,12 +339,16 @@ def act_on_agent(action, opts): agents, pattern_to_use = [a for a in agents if a.tag is not None], '*' # filter agents and update regex pattern - for pattern, filtered_agents in filter_agents(agents, pattern_to_use, opts): + for pattern, filtered_agents in filter_agents(agents, pattern_to_use, + opts): if not filtered_agents: - _stderr.write(f"Agents NOT found using 'vctl {opts.command}' on pattern: {pattern}\n") + _stderr.write( + f"Agents NOT found using 'vctl {opts.command}' on pattern: {pattern}\n" + ) for agent in filtered_agents: pid, status = call("agent_status", agent.uuid) - _call_action_on_agent(agent, pid, status, call, action) + _call_action_on_agent(agent, pid, status, call, action) + def _call_action_on_agent(agent, pid, status, call, action): if action == "start_agent": @@ -389,23 +374,17 @@ def shutdown_agents(opts): if "rmq" == utils.get_messagebus(): if not check_rabbit_status(): rmq_cfg = RMQConfig() - wait_period = ( - rmq_cfg.reconnect_delay() if rmq_cfg.reconnect_delay() < 60 - else 60 - ) + wait_period = (rmq_cfg.reconnect_delay() + if rmq_cfg.reconnect_delay() < 60 else 60) _stderr.write( "RabbitMQ server is not running.\n" "Waiting for {} seconds for possible reconnection and to " - "perform normal shutdown\n".format( - wait_period - ) - ) + "perform normal shutdown\n".format(wait_period)) gevent.sleep(wait_period) if not check_rabbit_status(): _stderr.write( "RabbitMQ server is still not running.\nShutting down " - "the platform forcefully\n" - ) + "the platform forcefully\n") opts.aip.brute_force_platform_shutdown() return opts.connection.call("shutdown") @@ -442,9 +421,8 @@ def send(): wheel.close() channel.close(linger=0) - result = connection.vip.rpc.call( - peer, "install_agent", os.path.basename(path), channel.name - ) + result = connection.vip.rpc.call(peer, "install_agent", + os.path.basename(path), channel.name) task = gevent.spawn(send) result.rawlink(lambda glt: task.kill(block=False)) _log.debug(f"Result is {result}") @@ -458,9 +436,6 @@ def send_agent(opts): return uuid - - - def do_stats(opts): call = opts.connection.call if opts.op == "status": @@ -478,7 +453,6 @@ def do_stats(opts): _stdout.write("%sabled\n" % ("en" if call("stats.enabled") else "dis")) - def priority(value): n = int(value) if not 0 <= n < 100: @@ -493,17 +467,18 @@ def get_keys(opts): key_store = KeyStore() publickey = key_store.public secretkey = key_store.secret - return {"publickey": publickey, "secretkey": secretkey, - "serverkey": serverkey} + return { + "publickey": publickey, + "secretkey": secretkey, + "serverkey": serverkey + } def main(): # Refuse to run as root if not getattr(os, "getuid", lambda: -1)(): - sys.stderr.write( - "%s: error: refusing to run as root to prevent " - "potential damage.\n" % os.path.basename(sys.argv[0]) - ) + sys.stderr.write("%s: error: refusing to run as root to prevent " + "potential damage.\n" % os.path.basename(sys.argv[0])) sys.exit(77) volttron_home = get_home() @@ -534,8 +509,7 @@ def main(): help="timeout in seconds for remote calls (default: %(default)g)", ) global_args.add_argument( - "--msgdebug", help="route all messages to an agent while debugging" - ) + "--msgdebug", help="route all messages to an agent while debugging") global_args.add_argument( "--vip-address", metavar="ZMQADDR", @@ -554,21 +528,24 @@ def main(): action="store_true", help="filter/search by agent name", ) - filterable.add_argument( - "--tag", dest="by_tag", action="store_true", - help="filter/search by tag name" - ) - filterable.add_argument( - "--all-tagged", dest="by_all_tagged", action="store_true", - help="filter/search by all tagged agents" - ) + filterable.add_argument("--tag", + dest="by_tag", + action="store_true", + help="filter/search by tag name") + filterable.add_argument("--all-tagged", + dest="by_all_tagged", + action="store_true", + help="filter/search by all tagged agents") filterable.add_argument( "--uuid", dest="by_uuid", action="store_true", help="filter/search by UUID (default)", ) - filterable.set_defaults(by_name=False, by_tag=False, by_all_tagged=False, by_uuid=False) + filterable.set_defaults(by_name=False, + by_tag=False, + by_all_tagged=False, + by_uuid=False) parser = config.ArgumentParser( prog=os.path.basename(sys.argv[0]), @@ -614,12 +591,13 @@ def main(): default=logging.WARNING, help="set logger verboseness", ) - parser.add_argument("--show-config", action="store_true", + parser.add_argument("--show-config", + action="store_true", help=argparse.SUPPRESS) - parser.add_argument( - "--json", action="store_true", default=False, - help="format output to json" - ) + parser.add_argument("--json", + action="store_true", + default=False, + help="format output to json") parser.add_help_argument() parser.set_defaults( @@ -627,9 +605,9 @@ def main(): volttron_home=volttron_home, ) - top_level_subparsers = parser.add_subparsers( - title="commands", metavar="", dest="command" - ) + top_level_subparsers = parser.add_subparsers(title="commands", + metavar="", + dest="command") def add_parser(*args, **kwargs) -> argparse.ArgumentParser: parents = kwargs.get("parents", []) @@ -643,28 +621,32 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: # ==================================================== add_install_agent_parser(add_parser, HAVE_RESTRICTED) - tag = add_parser("tag", parents=[filterable], + tag = add_parser("tag", + parents=[filterable], help="set, show, or remove agent tag") tag.add_argument("agent", help="UUID or name of agent") group = tag.add_mutually_exclusive_group() group.add_argument("tag", nargs="?", const=None, help="tag to give agent") - group.add_argument("-r", "--remove", action="store_true", + group.add_argument("-r", + "--remove", + action="store_true", help="remove tag") tag.set_defaults(func=tag_agent, tag=None, remove=False) remove = add_parser("remove", parents=[filterable], help="remove agent") remove.add_argument("pattern", nargs="+", help="UUID or name of agent") - remove.add_argument( - "-f", "--force", action="store_true", - help="force removal of multiple agents" - ) + remove.add_argument("-f", + "--force", + action="store_true", + help="force removal of multiple agents") remove.set_defaults(func=remove_agent, force=False) peers = add_parser("peerlist", help="list the peers connected to the platform") peers.set_defaults(func=list_peers) - list_ = add_parser("list", parents=[filterable], + list_ = add_parser("list", + parents=[filterable], help="list installed agent") list_.add_argument("pattern", nargs="*", help="UUID or name of agent") list_.add_argument( @@ -676,7 +658,8 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) list_.set_defaults(func=list_agents, min_uuid_len=1) - status = add_parser("status", parents=[filterable], + status = add_parser("status", + parents=[filterable], help="show status of agents") status.add_argument("pattern", nargs="*", help="UUID or name of agent") status.add_argument( @@ -688,9 +671,9 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) status.set_defaults(func=status_agents, min_uuid_len=1) - health = add_parser( - "health", parents=[filterable], help="show agent health as JSON" - ) + health = add_parser("health", + parents=[filterable], + help="show agent health as JSON") health.add_argument("pattern", nargs=1, help="UUID or name of agent") health.set_defaults(func=agent_health, min_uuid_len=1) @@ -704,27 +687,29 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) clear.set_defaults(func=clear_status, clear_all=False) - enable = add_parser( - "enable", parents=[filterable], - help="enable agent to start automatically" - ) + enable = add_parser("enable", + parents=[filterable], + help="enable agent to start automatically") enable.add_argument("pattern", nargs="+", help="UUID or name of agent") - enable.add_argument( - "-p", "--priority", type=priority, - help="2-digit priority from 00 to 99" - ) + enable.add_argument("-p", + "--priority", + type=priority, + help="2-digit priority from 00 to 99") enable.set_defaults(func=enable_agent, priority="50") - disable = add_parser( - "disable", parents=[filterable], - help="prevent agent from start automatically" - ) + disable = add_parser("disable", + parents=[filterable], + help="prevent agent from start automatically") disable.add_argument("pattern", nargs="+", help="UUID or name of agent") disable.set_defaults(func=disable_agent) - start = add_parser("start", parents=[filterable], + start = add_parser("start", + parents=[filterable], help="start installed agent") - start.add_argument("pattern", nargs="*", help="UUID or name of agent", default='') + start.add_argument("pattern", + nargs="*", + help="UUID or name of agent", + default='') if HAVE_RESTRICTED: start.add_argument( "--verify", @@ -741,11 +726,17 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: start.set_defaults(func=start_agent) stop = add_parser("stop", parents=[filterable], help="stop agent") - stop.add_argument("pattern", nargs="*", help="UUID or name of agent", default='') + stop.add_argument("pattern", + nargs="*", + help="UUID or name of agent", + default='') stop.set_defaults(func=stop_agent) restart = add_parser("restart", parents=[filterable], help="restart agent") - restart.add_argument("pattern", nargs="*", help="UUID or name of agent", default='') + restart.add_argument("pattern", + nargs="*", + help="UUID or name of agent", + default='') restart.set_defaults(func=restart_agent) run = add_parser("run", help="start any agent by path") @@ -765,7 +756,6 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) run.set_defaults(func=run_agent) - # ==================================================== # rpc commands # ==================================================== @@ -787,10 +777,9 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: add_config_store_parser(add_parser) shutdown = add_parser("shutdown", help="stop all agents") - shutdown.add_argument( - "--platform", action="store_true", - help="also stop the platform process" - ) + shutdown.add_argument("--platform", + action="store_true", + help="also stop the platform process") shutdown.set_defaults(func=shutdown_agents, platform=False) send = add_parser("send", help="send agent and start on a remote platform") @@ -800,9 +789,9 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: stats = add_parser("stats", help="manage router message statistics tracking") op = stats.add_argument( - "op", choices=["status", "enable", "disable", "dump", "pprint"], - nargs="?" - ) + "op", + choices=["status", "enable", "disable", "dump", "pprint"], + nargs="?") stats.set_defaults(func=do_stats, op="status") # ============================================================================== @@ -820,12 +809,14 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: "create-cgroups", help="setup VOLTTRON control group for restricted execution", ) - cgroup.add_argument( - "-u", "--user", metavar="USER", help="owning user name or ID" - ) - cgroup.add_argument( - "-g", "--group", metavar="GROUP", help="owning group name or ID" - ) + cgroup.add_argument("-u", + "--user", + metavar="USER", + help="owning user name or ID") + cgroup.add_argument("-g", + "--group", + metavar="GROUP", + help="owning group name or ID") cgroup.set_defaults(func=create_cgroups, user=None, group=None) # Parse and expand options @@ -840,10 +831,8 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: if args[0] not in ("list", "tag", "auth", "rabbitmq", "certs"): # check pid file if not utils.is_volttron_running(volttron_home): - _stderr.write( - "VOLTTRON is not running. This command " - "requires VOLTTRON platform to be running\n" - ) + _stderr.write("VOLTTRON is not running. This command " + "requires VOLTTRON platform to be running\n") return 10 conf = os.path.join(volttron_home, "config") @@ -868,13 +857,18 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: elif opts.log == "-": log_to_file(sys.stdout, level) elif opts.log: - log_to_file(opts.log, level, + log_to_file(opts.log, + level, handler_class=logging.handlers.WatchedFileHandler) else: log_to_file(None, 100, handler_class=lambda x: logging.NullHandler()) if opts.log_config: logging.config.fileConfig(opts.log_config) + if not hasattr(opts, "func"): + parser.print_help() + sys.exit(0) + opts.aip = aipmod.AIPplatform(opts) opts.aip.setup() diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index a022639ec2..56cbdb26a3 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -39,7 +39,6 @@ from gevent.subprocess import Popen from zmq import green as zmq -from requirements import extras_require from volttron.platform import is_rabbitmq_available from volttron.platform.auth import certs from volttron.platform import jsonapi @@ -336,12 +335,7 @@ def _get_dependencies(): return dependencies -def _check_dependencies_met(requirement): - try: - dependencies_needed = extras_require[requirement] - except KeyError: - print(f"ERROR: Requirement {requirement} was not found in requirements.py") - return False +def _check_dependencies_met(dependencies_needed): current_dependencies = _get_dependencies() for dependency in dependencies_needed: if "==" in dependency: @@ -358,15 +352,21 @@ def _check_dependencies_met(requirement): def set_dependencies(requirement): try: + # go up two level above env/bin + sys.path.append(os.path.dirname(os.path.dirname(sys.path[0]))) + from requirements import extras_require dependencies_needed = extras_require[requirement] except KeyError: - print("ERROR: Incorrect requirement chosen") + print(f"ERROR: Incorrect requirement chosen: {requirement}") + return + + if not _check_dependencies_met(dependencies_needed): + print(f"Installing {requirement} dependencies...") + cmds = [sys.executable, "-m", "pip", "install"] + for dependency in dependencies_needed: + cmds.append(dependency) + subprocess.check_call(cmds) return - cmds = [sys.executable, "-m", "pip", "install"] - for dependency in dependencies_needed: - cmds.append(dependency) - subprocess.check_call(cmds) - return def _create_web_certs(): @@ -893,10 +893,7 @@ def wizard(): prompt = 'Is this instance web enabled?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met('web'): - print("Web dependencies not installed. Installing now...") - set_dependencies('web') - print("Done!") + set_dependencies('web') if config_opts['message-bus'] == 'rmq': do_web_enabled_rmq(volttron_home) elif config_opts['message-bus'] == 'zmq': @@ -916,13 +913,9 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): - print("VCP dependencies not installed. Installing now...") - if not _check_dependencies_met("drivers"): - set_dependencies("drivers") - if not _check_dependencies_met("web"): - set_dependencies("web") - print("Done!") + print("Checking for VCP dependencies.....") + set_dependencies("drivers") + set_dependencies("web") do_vcp() prompt = 'Would you like to install a platform historian?' @@ -932,10 +925,8 @@ def wizard(): prompt = 'Would you like to install a platform driver?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met("drivers"): - print("Driver dependencies not installed. Installing now...") - set_dependencies("drivers") - print("Done!") + print("Checking Driver dependencies...") + set_dependencies("drivers") do_platform_driver() prompt = 'Would you like to install a listener agent?' @@ -954,13 +945,9 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): - print("VCP dependencies not installed. Installing now...") - if not _check_dependencies_met("drivers"): - set_dependencies("drivers") - if not _check_dependencies_met("web"): - set_dependencies("web") - print("Done!") + print("Checking VCP dependencies...") + set_dependencies("drivers") + set_dependencies("web") do_vcp() prompt = 'Would you like to install a platform historian?' @@ -970,10 +957,8 @@ def wizard(): prompt = 'Would you like to install a platform driver?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met("drivers"): - print("Driver dependencies not installed. Installing now...") - set_dependencies("drivers") - print("Done!") + print("Checking Driver dependencies...") + set_dependencies("drivers") do_platform_driver() prompt = 'Would you like to install a listener agent?' @@ -1114,10 +1099,9 @@ def process_rmq_inputs(args_dict, instance_name=None): vhome = get_home() - if args_dict['installation-type'] in ['federation', 'shovel'] and not _check_dependencies_met('web'): - print("Web dependencies not installed. Installing now...") + if args_dict['installation-type'] in ['federation', 'shovel']: + print("Checking Web dependencies...") set_dependencies('web') - print("Done!") if args_dict['config'] is not None: if not os.path.exists(vhome): diff --git a/volttron/platform/main.py b/volttron/platform/main.py index a0fdd7ecf4..16db2285d1 100644 --- a/volttron/platform/main.py +++ b/volttron/platform/main.py @@ -435,9 +435,6 @@ def issue(self, topic, frames, extra=None): # return result def handle_subsystem(self, frames, user_id): - _log.debug( - f"Handling subsystem with frames: {frames} user_id: {user_id}") - subsystem = frames[5] if subsystem == 'quit': sender = frames[0] diff --git a/volttron/platform/vip/agent/subsystems/auth.py b/volttron/platform/vip/agent/subsystems/auth.py index 8ba4e2dcba..9337d0e513 100644 --- a/volttron/platform/vip/agent/subsystems/auth.py +++ b/volttron/platform/vip/agent/subsystems/auth.py @@ -265,6 +265,7 @@ def update_rpc_method_capabilities(self): """ rpc_method_authorizations = {} rpc_methods = self.get_rpc_exports() + updated_rpc_authorizations = None for method in rpc_methods: if len(method.split(".")) > 1: pass @@ -295,9 +296,7 @@ def update_rpc_method_capabilities(self): _log.info( f"Skipping updating rpc auth capabilities for agent " f"{self._core().identity} connecting to remote address: {self._core().address} ") - updated_rpc_authorizations = None except gevent.timeout.Timeout: - updated_rpc_authorizations = None _log.warning(f"update_id_rpc_authorization rpc call timed out for {self._core().identity} {rpc_method_authorizations}") except MethodNotFound: _log.warning("update_id_rpc_authorization method is missing from " @@ -306,7 +305,6 @@ def update_rpc_method_capabilities(self): "dynamic RPC authorizations.") return except Exception as e: - updated_rpc_authorizations = None _log.exception(f"Exception when calling rpc method update_id_rpc_authorizations for identity: " f"{self._core().identity} Exception:{e}") if updated_rpc_authorizations is None: @@ -318,7 +316,7 @@ def update_rpc_method_capabilities(self): f"the identity of the agent" ) return - if rpc_method_authorizations != updated_rpc_authorizations: + if rpc_method_authorizations != updated_rpc_authorizations and updated_rpc_authorizations is not None: for method in updated_rpc_authorizations: self.set_rpc_authorizations( method, updated_rpc_authorizations[method] diff --git a/volttron/platform/vip/agent/subsystems/rpc.py b/volttron/platform/vip/agent/subsystems/rpc.py index 510ce9d099..d77108a481 100644 --- a/volttron/platform/vip/agent/subsystems/rpc.py +++ b/volttron/platform/vip/agent/subsystems/rpc.py @@ -278,8 +278,8 @@ def _iterate_exports(self): for method_name in self._exports: method = self._exports[method_name] caps = annotations(method, set, "rpc.allow_capabilities") - # if caps: - # self._exports[method_name] = self._add_auth_check(method, caps) + if caps: + self._exports[method_name] = self._add_auth_check(method, caps) def _add_auth_check(self, method, required_caps): """ diff --git a/volttron/platform/web/admin_endpoints.py b/volttron/platform/web/admin_endpoints.py index 9de8d05a5c..d307b2f353 100644 --- a/volttron/platform/web/admin_endpoints.py +++ b/volttron/platform/web/admin_endpoints.py @@ -46,7 +46,6 @@ from volttron.platform import get_home from volttron.platform import jsonapi from volttron.utils import VolttronHomeFileReloader -from volttron.utils.persistance import PersistentDict _log = logging.getLogger(__name__) @@ -84,7 +83,7 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None) else: self._ssl_public_key = None - self._userdict = None + self._userdict = {} self.reload_userdict() self._observer = Observer() @@ -96,7 +95,14 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None) def reload_userdict(self): webuserpath = os.path.join(get_home(), 'web-users.json') - self._userdict = PersistentDict(webuserpath, format="json") + if os.path.exists(webuserpath): + with open(webuserpath) as fp: + try: + self._userdict = jsonapi.loads(fp.read()) + except json.decoder.JSONDecodeError: + self._userdict = {} + # Keep same behavior as with PersistentDict + raise ValueError("File not in a supported format") def get_routes(self): """ @@ -339,4 +345,5 @@ def add_user(self, username, unencrypted_pw, groups=None, overwrite=False): groups=groups ) - self._userdict.sync() + with open(os.path.join(get_home(), 'web-users.json'), 'w') as fp: + fp.write(jsonapi.dumps(self._userdict, indent=2)) diff --git a/volttron/utils/__init__.py b/volttron/utils/__init__.py index daa1e1aaa7..e236d5185f 100644 --- a/volttron/utils/__init__.py +++ b/volttron/utils/__init__.py @@ -108,7 +108,7 @@ def __init__(self, filetowatch, callback): _log.debug("patterns is {}".format([get_home() + '/' + filetowatch])) self._callback = callback - def on_any_event(self, event): + def on_closed(self, event): _log.debug("Calling callback on event {}. Calling {}".format(event, self._callback)) try: self._callback() @@ -133,7 +133,7 @@ def __init__(self, filetowatch, callback): def watchfile(self): return self._filetowatch - def on_any_event(self, event): + def on_closed(self, event): _log.debug("Calling callback on event {}. Calling {}".format(event, self._callback)) try: self._callback(self._filetowatch) diff --git a/volttrontesting/platform/auth_tests/test_auth_control.py b/volttrontesting/platform/auth_tests/test_auth_control.py index e76d63df8f..a17c6bf1b0 100644 --- a/volttrontesting/platform/auth_tests/test_auth_control.py +++ b/volttrontesting/platform/auth_tests/test_auth_control.py @@ -375,20 +375,6 @@ def test_auth_rpc_method_remove(auth_instance): assert entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]} -@pytest.mark.control -def test_group_cmds(auth_instance): - """Test add-group, list-groups, update-group, and remove-group""" - _run_group_or_role_cmds(auth_instance, _add_group, _list_groups, - _update_group, _remove_group) - - -@pytest.mark.control -def test_role_cmds(auth_instance): - """Test add-role, list-roles, update-role, and remove-role""" - _run_group_or_role_cmds(auth_instance, _add_role, _list_roles, - _update_role, _remove_role) - - def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn): expected = [] key = '0' diff --git a/volttrontesting/platform/auth_tests/test_auth_group_roles.py b/volttrontesting/platform/auth_tests/test_auth_group_roles.py new file mode 100644 index 0000000000..bfcad699c5 --- /dev/null +++ b/volttrontesting/platform/auth_tests/test_auth_group_roles.py @@ -0,0 +1,174 @@ + +import os +import re +import subprocess + +import gevent +import pytest +from mock import MagicMock +from volttron.platform.auth.auth_protocols.auth_zmq import ZMQAuthorization, ZMQServerAuthentication + +from volttrontesting.platform.auth_tests.conftest import assert_auth_entries_same +from volttrontesting.utils.platformwrapper import with_os_environ +from volttrontesting.utils.utils import AgentMock +from volttron.platform.vip.agent import Agent +from volttron.platform.auth import AuthService +from volttron.platform.auth import AuthEntry +from volttron.platform import jsonapi + +@pytest.fixture(autouse=True) +def auth_instance(volttron_instance): + if not volttron_instance.auth_enabled: + pytest.skip("AUTH tests are not applicable if auth is disabled") + with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'r') as f: + auth_file = jsonapi.load(f) + print(auth_file) + try: + yield volttron_instance + finally: + with with_os_environ(volttron_instance.env): + with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'w') as f: + jsonapi.dump(auth_file, f) + + +def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn): + expected = [] + key = '0' + values = ['0', '1'] + expected.extend(values) + + add_fn(platform, key, values) + gevent.sleep(4) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update add single value + values = ['2'] + expected.extend(values) + update_fn(platform, key, values) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update add multiple values + values = ['3', '4'] + expected.extend(values) + update_fn(platform, key, values) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update remove single value + value = '0' + expected.remove(value) + update_fn(platform, key, [value], remove=True) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update remove single value + values = ['1', '2'] + for value in values: + expected.remove(value) + update_fn(platform, key, values, remove=True) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Remove key + remove_fn(platform, key) + gevent.sleep(2) + keys = list_fn(platform) + assert key not in keys + + + +def _add_group_or_role(platform, cmd, name, list_): + with with_os_environ(platform.env): + args = ['volttron-ctl', 'auth', cmd, name] + args.extend(list_) + p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True) + p.communicate() + assert p.returncode == 0 + + +def _add_group(platform, group, roles): + _add_group_or_role(platform, 'add-group', group, roles) + + +def _add_role(platform, role, capabilities): + _add_group_or_role(platform, 'add-role', role, capabilities) + + +def _list_groups_or_roles(platform, cmd): + with with_os_environ(platform.env): + output = subprocess.check_output(['volttron-ctl', 'auth', cmd], + env=platform.env, universal_newlines=True) + # For these tests don't use names that contain space, [, comma, or ' + output = output.replace('[', '').replace("'", '').replace(']', '') + output = output.replace(',', '') + lines = output.split('\n') + + dict_ = {} + for line in lines[2:-1]: # skip two header lines and last (empty) line + list_ = ' '.join(line.split()).split() # combine multiple spaces + dict_[list_[0]] = list_[1:] + return dict_ + + +def _list_groups(platform): + return _list_groups_or_roles(platform, 'list-groups') + + +def _list_roles(platform): + return _list_groups_or_roles(platform, 'list-roles') + + +def _update_group_or_role(platform, cmd, key, values, remove): + with with_os_environ(platform.env): + args = ['volttron-ctl', 'auth', cmd, key] + args.extend(values) + if remove: + args.append('--remove') + p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True) + p.communicate() + assert p.returncode == 0 + + +def _update_group(platform, group, roles, remove=False): + _update_group_or_role(platform, 'update-group', group, roles, remove) + + +def _update_role(platform, role, caps, remove=False): + _update_group_or_role(platform, 'update-role', role, caps, remove) + + +def _remove_group_or_role(platform, cmd, key): + with with_os_environ(platform.env): + args = ['volttron-ctl', 'auth', cmd, key] + p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True) + p.communicate() + assert p.returncode == 0 + + +def _remove_group(platform, group): + _remove_group_or_role(platform, 'remove-group', group) + + +def _remove_role(platform, role): + _remove_group_or_role(platform, 'remove-role', role) + + +@pytest.mark.control +def test_group_cmds(auth_instance): + """Test add-group, list-groups, update-group, and remove-group""" + _run_group_or_role_cmds(auth_instance, _add_group, _list_groups, + _update_group, _remove_group) + + +@pytest.mark.control +def test_role_cmds(auth_instance): + """Test add-role, list-roles, update-role, and remove-role""" + _run_group_or_role_cmds(auth_instance, _add_role, _list_roles, + _update_role, _remove_role) + diff --git a/volttrontesting/platform/auth_tests/test_auth_integration.py b/volttrontesting/platform/auth_tests/test_auth_integration.py new file mode 100644 index 0000000000..bc9dedc9b3 --- /dev/null +++ b/volttrontesting/platform/auth_tests/test_auth_integration.py @@ -0,0 +1,224 @@ +import os +import subprocess +import sys +import tempfile +import gevent +import pytest +from volttron.platform.agent.known_identities import AUTH +from volttron.platform import jsonrpc +from volttron.platform.messaging.health import STATUS_BAD + +called_agent_src = """ +import sys +from volttron.platform.agent import utils +from volttron.platform.vip.agent import Agent, Core +from volttron.platform.vip.agent.subsystems import RPC +import gevent +class CalledAgent(Agent): + def __init__(self, config_path, **kwargs): + super(CalledAgent, self).__init__(**kwargs) + @RPC.export + @RPC.allow("can_call_method") + def restricted_method(self, sender, **kwargs): + print("test") +def main(argv=sys.argv): + try: + utils.vip_main(CalledAgent, version='0.1') + except Exception as e: + print('unhandled exception: {}'.format(e)) +if __name__ == '__main__': + # Entry point for script + sys.exit(main()) +""" + +called_agent_setup = """ +from setuptools import setup +setup( + name='calledagent', + version='0.1', + install_requires=['volttron'], + packages=['calledagent'], + entry_points={ + 'setuptools.installation': [ + 'eggsecutable=calledagent.calledagent:main', + ] + } +) +""" + +caller_agent_src = """ +import sys +import gevent +import logging +from volttron.platform.agent import utils +from volttron.platform.vip.agent import Agent, Core +from volttron.platform.vip.agent.subsystems import RPC +from volttron.platform.scheduling import periodic +from volttron.platform.messaging.health import (STATUS_BAD, + STATUS_GOOD, Status) +from volttron.platform.agent.known_identities import AUTH +from volttron.platform import jsonrpc +from volttron.platform.messaging.health import STATUS_BAD + +_log = logging.getLogger(__name__) +class CallerAgent(Agent): + def __init__(self, config_path, **kwargs): + super(CallerAgent, self).__init__(**kwargs) + + # @Core.schedule(periodic(3)) + # def call_rpc_method(self): + @Core.receiver("onstart") + def onstart(self, sender, **kwargs): + try: + self.vip.rpc.call('called_agent', 'restricted_method').get(timeout=3) + except Exception as e: + self.vip.health.set_status(STATUS_BAD, f"{e}") +def main(argv=sys.argv): + try: + utils.vip_main(CallerAgent, version='0.1') + except Exception as e: + print('unhandled exception: {}'.format(e)) +if __name__ == '__main__': + # Entry point for script + sys.exit(main()) +""" + +caller_agent_setup = """ +from setuptools import setup +setup( + name='calleragent', + version='0.1', + install_requires=['volttron'], + packages=['calleragent'], + entry_points={ + 'setuptools.installation': [ + 'eggsecutable=calleragent.calleragent:main', + ] + } +) +""" + +@pytest.fixture +def install_two_agents(volttron_instance): + """Returns two agents for testing authorization + + The first agent is the "RPC callee." + The second agent is the unauthorized "RPC caller." + """ + """ + Test if control agent periodically monitors and restarts any crashed agents + :param volttron_instance: + :return: + """ + + tmpdir = volttron_instance.volttron_home+"/tmpdir" + os.mkdir(tmpdir) + tmpdir = volttron_instance.volttron_home+"/tmpdir" + "/called" + os.mkdir(tmpdir) + os.chdir(tmpdir) + + os.mkdir("calledagent") + with open(os.path.join("calledagent", "__init__.py"), "w") as file: + pass + with open(os.path.join("calledagent", "calledagent.py"), "w") as file: + file.write(called_agent_src) + with open(os.path.join("setup.py"), "w") as file: + file.write(called_agent_setup) + p = subprocess.Popen( + [sys.executable, "setup.py", "bdist_wheel"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + # print("out {}".format(stdout)) + # print("err {}".format(stderr)) + + wheel = os.path.join(tmpdir, "dist", "calledagent-0.1-py3-none-any.whl") + assert os.path.exists(wheel) + called_uuid = volttron_instance.install_agent(agent_wheel=wheel, + vip_identity="called_agent", + start=False) + assert called_uuid + gevent.sleep(1) + + + tmpdir = volttron_instance.volttron_home+"/tmpdir" + "/caller" + os.mkdir(tmpdir) + os.chdir(tmpdir) + os.mkdir("calleragent") + with open(os.path.join("calleragent", "__init__.py"), "w") as file: + pass + with open(os.path.join("calleragent", "calleragent.py"), "w") as file: + file.write(caller_agent_src) + with open(os.path.join("setup.py"), "w") as file: + file.write(caller_agent_setup) + p = subprocess.Popen( + [sys.executable, "setup.py", "bdist_wheel"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + # print("out {}".format(stdout)) + # print("err {}".format(stderr)) + + wheel = os.path.join(tmpdir, "dist", "calleragent-0.1-py3-none-any.whl") + assert os.path.exists(wheel) + caller_uuid = volttron_instance.install_agent(agent_wheel=wheel, + vip_identity="caller_agent", + start=False) + assert caller_uuid + gevent.sleep(1) + + try: + yield caller_uuid, called_uuid + finally: + #volttron_instance.remove_agent(caller_uuid) + #volttron_instance.remove_agent(called_uuid) + # TODO if we have to wait for auth propagation anyways why do we create new agents for each test case + # we should just update capabilities, at least we will save on agent creation and tear down time + gevent.sleep(1) + + +@pytest.fixture(autouse=True) +def build_volttron_instance(volttron_instance): + if not volttron_instance.auth_enabled: + pytest.skip("AUTH tests are not applicable if auth is disabled") + + +@pytest.mark.auth +def test_unauthorized_rpc_call(volttron_instance, install_two_agents): + """Tests an agent with no capabilities calling a method that + requires one capability ("can_call_foo") + """ + (caller_agent_uuid, called_agent_uuid) = install_two_agents + + # check auth error for newly installed agents + check_auth_error(volttron_instance, caller_agent_uuid, called_agent_uuid) + + volttron_instance.restart_platform() + gevent.sleep(3) + + # check auth error for already installed agent + check_auth_error(volttron_instance, caller_agent_uuid, called_agent_uuid) + +def check_auth_error(volttron_instance, caller_agent_uuid, called_agent_uuid): + + expected_auth_err = ('volttron.platform.jsonrpc.Error(' + '-32001, "method \'restricted_method\' ' + 'requires capabilities {\'can_call_method\'}, ' + 'but capability {\'edit_config_store\': {\'identity\': \'caller_agent\'}}' + ' was provided for user caller_agent")') + volttron_instance.start_agent(called_agent_uuid) + gevent.sleep(1) + volttron_instance.start_agent(caller_agent_uuid) + + # If the agent is not authorized health status is updated + health = volttron_instance.dynamic_agent.vip.rpc.call( + "caller_agent", "health.get_status").get(timeout=2) + + assert health.get('status') == STATUS_BAD + assert health.get('context') == expected_auth_err + + + + diff --git a/volttrontesting/services/historian/test_historian.py b/volttrontesting/services/historian/test_historian.py index afa49acf41..352cbfea6a 100644 --- a/volttrontesting/services/historian/test_historian.py +++ b/volttrontesting/services/historian/test_historian.py @@ -1443,6 +1443,8 @@ def test_invalid_query(request, historian, publish_agent, query_agent, start=now, count=20, order="LAST_TO_FIRST").get(timeout=10) + except gevent.timeout.Timeout: + assert True except Exception as error: print("exception: {}".format(error)) assert "No route to host:" in str(error) diff --git a/volttrontesting/utils/platformwrapper.py b/volttrontesting/utils/platformwrapper.py index 5fc82bf4b0..79cd4e4267 100644 --- a/volttrontesting/utils/platformwrapper.py +++ b/volttrontesting/utils/platformwrapper.py @@ -1564,17 +1564,20 @@ def shutdown_platform(self): return running_pids = [] - if self.dynamic_agent: # because we are not creating dynamic agent in setupmode - for agnt in self.list_agents(): - pid = self.agent_pid(agnt['uuid']) - if pid is not None and int(pid) > 0: - running_pids.append(int(pid)) - if not self.skip_cleanup: - self.remove_all_agents() - # don't wait indefinetly as shutdown will not throw an error if RMQ is down/has cert errors - self.dynamic_agent.vip.rpc(CONTROL, 'shutdown').get(timeout=10) - self.dynamic_agent.core.stop() - self.dynamic_agent = None + if self.dynamic_agent: + try:# because we are not creating dynamic agent in setupmode + for agnt in self.list_agents(): + pid = self.agent_pid(agnt['uuid']) + if pid is not None and int(pid) > 0: + running_pids.append(int(pid)) + if not self.skip_cleanup: + self.remove_all_agents() + # don't wait indefinetly as shutdown will not throw an error if RMQ is down/has cert errors + self.dynamic_agent.vip.rpc(CONTROL, 'shutdown').get(timeout=10) + self.dynamic_agent.core.stop() + self.dynamic_agent = None + except BaseException as e: + self.logit(f"Exception while shutting down. {e}") if self.p_process is not None: try: