From f5d2b63cf29fad189981b29bfea945d6e58f6e58 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 8 Jan 2026 10:43:46 -0600 Subject: [PATCH 1/8] allowing ext_pkg to be defined anywhere --- nodescraper/cli/cli.py | 84 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 81c18f19..152de6a2 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -30,6 +30,7 @@ import os import platform import sys +from importlib import import_module from typing import Optional import nodescraper @@ -54,12 +55,87 @@ from nodescraper.pluginexecutor import PluginExecutor from nodescraper.pluginregistry import PluginRegistry -try: - import ext_nodescraper_plugins as ext_pkg - extra_pkgs = [ext_pkg] -except ImportError: +def discover_external_plugins(): + """Discover ext_nodescraper_plugins from all installed packages. + + This function searches for ext_nodescraper_plugins in: + 1. Top-level ext_nodescraper_plugins package + 2. Any installed package that has an ext_nodescraper_plugins submodule + + Returns: + list: List of discovered plugin packages + """ extra_pkgs = [] + seen_paths = set() # Track paths to avoid duplicates + + # Try top-level ext_nodescraper_plugins first (original behavior) + try: + import ext_nodescraper_plugins as ext_pkg + extra_pkgs.append(ext_pkg) + if hasattr(ext_pkg, '__file__') and ext_pkg.__file__: + seen_paths.add(ext_pkg.__file__) + except ImportError: + pass + + # Discover ext_nodescraper_plugins from installed packages + try: + from importlib.metadata import distributions + + for dist in distributions(): + # Get package name and try different variations + pkg_name = dist.metadata.get('Name', '') + if not pkg_name: + continue + + # Try multiple name variations (with hyphens, underscores, and top-level module name) + name_variants = [ + pkg_name.replace('-', '_'), # amd-error-scraper -> amd_error_scraper + pkg_name.replace('_', '-'), # amd_error_scraper -> amd-error-scraper + ] + + # Try to find the actual top-level module name + try: + top_level = dist.read_text('top_level.txt') + if top_level: + name_variants.extend(top_level.strip().split('\n')) + except Exception: + pass + + # Try each variant + for variant in name_variants: + if not variant: + continue + + try: + module_path = f"{variant}.ext_nodescraper_plugins" + ext_pkg = import_module(module_path) + + # Check if we already have this package (by file path) + pkg_path = getattr(ext_pkg, '__file__', None) + if pkg_path and pkg_path in seen_paths: + continue + + # Add the package + extra_pkgs.append(ext_pkg) + if pkg_path: + seen_paths.add(pkg_path) + + # Found it, no need to try other variants + break + + except (ImportError, AttributeError, ModuleNotFoundError): + # This variant doesn't have ext_nodescraper_plugins, try next + continue + + except Exception: + # If discovery fails, just use what we found with top-level import + pass + + return extra_pkgs + + +extra_pkgs = discover_external_plugins() def build_parser( From 752365568996c68cd45eee5761e94f589b79dea3 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 8 Jan 2026 11:10:26 -0600 Subject: [PATCH 2/8] cleanup --- nodescraper/cli/cli.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 152de6a2..8caa1855 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -59,17 +59,12 @@ def discover_external_plugins(): """Discover ext_nodescraper_plugins from all installed packages. - This function searches for ext_nodescraper_plugins in: - 1. Top-level ext_nodescraper_plugins package - 2. Any installed package that has an ext_nodescraper_plugins submodule - Returns: list: List of discovered plugin packages """ extra_pkgs = [] seen_paths = set() # Track paths to avoid duplicates - # Try top-level ext_nodescraper_plugins first (original behavior) try: import ext_nodescraper_plugins as ext_pkg extra_pkgs.append(ext_pkg) @@ -83,18 +78,15 @@ def discover_external_plugins(): from importlib.metadata import distributions for dist in distributions(): - # Get package name and try different variations pkg_name = dist.metadata.get('Name', '') if not pkg_name: continue - # Try multiple name variations (with hyphens, underscores, and top-level module name) name_variants = [ - pkg_name.replace('-', '_'), # amd-error-scraper -> amd_error_scraper - pkg_name.replace('_', '-'), # amd_error_scraper -> amd-error-scraper + pkg_name.replace('-', '_'), + pkg_name.replace('_', '-'), ] - # Try to find the actual top-level module name try: top_level = dist.read_text('top_level.txt') if top_level: @@ -102,7 +94,6 @@ def discover_external_plugins(): except Exception: pass - # Try each variant for variant in name_variants: if not variant: continue @@ -121,15 +112,12 @@ def discover_external_plugins(): if pkg_path: seen_paths.add(pkg_path) - # Found it, no need to try other variants break except (ImportError, AttributeError, ModuleNotFoundError): - # This variant doesn't have ext_nodescraper_plugins, try next continue except Exception: - # If discovery fails, just use what we found with top-level import pass return extra_pkgs From 91b16854ad9df2fcb382fa9f95a4836b88418539 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 8 Jan 2026 13:59:04 -0600 Subject: [PATCH 3/8] added utest --- test/unit/framework/test_cli.py | 228 +++++++++++++++++++++++++++++++- 1 file changed, 227 insertions(+), 1 deletion(-) diff --git a/test/unit/framework/test_cli.py b/test/unit/framework/test_cli.py index cd266ed9..095a852b 100644 --- a/test/unit/framework/test_cli.py +++ b/test/unit/framework/test_cli.py @@ -25,13 +25,21 @@ ############################################################################### import argparse import os +import sys +import tempfile +import types +from pathlib import Path +from unittest import mock import pytest from pydantic import BaseModel +from nodescraper.base import InBandDataPlugin from nodescraper.cli import cli, inputargtypes -from nodescraper.enums import SystemLocation +from nodescraper.enums import ExecutionStatus, SystemLocation +from nodescraper.interfaces import DataAnalyzer from nodescraper.models import SystemInfo +from nodescraper.pluginregistry import PluginRegistry def test_log_path_arg(): @@ -150,3 +158,221 @@ def test_system_info_builder(): ) def test_process_args(raw_arg_input, plugin_names, exp_output): assert cli.process_args(raw_arg_input, plugin_names) == exp_output + + +def test_discover_external_plugins_top_level(): + """Test discovering ext_nodescraper_plugins as a top-level import.""" + mock_ext_pkg = mock.MagicMock() + mock_ext_pkg.__file__ = "/path/to/ext_nodescraper_plugins/__init__.py" + + with mock.patch("nodescraper.cli.cli.import_module"): + with mock.patch.dict("sys.modules", {"ext_nodescraper_plugins": mock_ext_pkg}): + result = cli.discover_external_plugins() + + assert len(result) >= 1 + assert mock_ext_pkg in result + + +def test_discover_external_plugins_no_plugins(): + """Test when no external plugins are installed.""" + with mock.patch("nodescraper.cli.cli.import_module") as mock_import: + mock_import.side_effect = ImportError("No module named 'ext_nodescraper_plugins'") + + with mock.patch("importlib.metadata.distributions", return_value=[]): + result = cli.discover_external_plugins() + + assert result == [] + + +def test_discover_external_plugins_from_installed_package(): + """Test discovering plugins from installed packages (not top-level).""" + mock_dist = mock.MagicMock() + mock_dist.metadata.get.return_value = "amd-custom-package" + mock_dist.read_text.return_value = "custompackage" + + mock_plugin = mock.MagicMock() + mock_plugin.__file__ = "/path/to/custompackage/ext_nodescraper_plugins/__init__.py" + + def mock_import_func(module_path): + if module_path == "custompackage.ext_nodescraper_plugins": + return mock_plugin + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist]): + result = cli.discover_external_plugins() + + assert mock_plugin in result + + +def test_discover_external_plugins_deduplication(): + """Test that duplicate plugins are not added multiple times.""" + mock_ext_pkg = mock.MagicMock() + mock_ext_pkg.__file__ = "/path/to/ext_nodescraper_plugins/__init__.py" + + mock_dist1 = mock.MagicMock() + mock_dist1.metadata.get.return_value = "package-one" + mock_dist1.read_text.return_value = "package_one" + + mock_dist2 = mock.MagicMock() + mock_dist2.metadata.get.return_value = "package-two" + mock_dist2.read_text.return_value = "package_one" + + def mock_import_func(module_path): + if "package_one.ext_nodescraper_plugins" in module_path: + return mock_ext_pkg + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist1, mock_dist2]): + result = cli.discover_external_plugins() + + file_paths = [pkg.__file__ for pkg in result if hasattr(pkg, "__file__")] + assert file_paths.count(mock_ext_pkg.__file__) == 1 + + +def test_discover_external_plugins_name_variants(): + """Test that different package name variants are tried (hyphens vs underscores).""" + mock_dist = mock.MagicMock() + mock_dist.metadata.get.return_value = "amd-error-scraper" + mock_dist.read_text.side_effect = Exception("No top_level.txt") + + mock_plugin = mock.MagicMock() + mock_plugin.__file__ = "/path/to/amd_error_scraper/ext_nodescraper_plugins/__init__.py" + + call_count = {"count": 0} + + def mock_import_func(module_path): + call_count["count"] += 1 + if module_path == "amd_error_scraper.ext_nodescraper_plugins": + return mock_plugin + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist]): + result = cli.discover_external_plugins() + + assert mock_plugin in result + assert call_count["count"] >= 1 + + +def test_discover_external_plugins_handles_exceptions(): + """Test that discovery continues even if some packages fail.""" + mock_dist1 = mock.MagicMock() + mock_dist1.metadata.get.return_value = "good-package" + mock_dist1.read_text.return_value = "goodpackage" + + mock_dist2 = mock.MagicMock() + mock_dist2.metadata.get.side_effect = Exception("Corrupted metadata") + + mock_plugin = mock.MagicMock() + mock_plugin.__file__ = "/path/to/goodpackage/ext_nodescraper_plugins/__init__.py" + + def mock_import_func(module_path): + if module_path == "goodpackage.ext_nodescraper_plugins": + return mock_plugin + raise ImportError(f"No module named '{module_path}'") + + with mock.patch("nodescraper.cli.cli.import_module", side_effect=mock_import_func): + with mock.patch("importlib.metadata.distributions", return_value=[mock_dist1, mock_dist2]): + result = cli.discover_external_plugins() + + assert mock_plugin in result + + +def test_external_plugins_integration(): + """Integration test: Create a temporary external plugin and verify it's picked up.""" + with tempfile.TemporaryDirectory() as tmpdir: + pkg_dir = Path(tmpdir) / "test_external_pkg" + pkg_dir.mkdir() + (pkg_dir / "__init__.py").write_text("# Test external package\n") + + ext_plugins_dir = pkg_dir / "ext_nodescraper_plugins" + ext_plugins_dir.mkdir() + (ext_plugins_dir / "__init__.py").write_text("# External plugins package\n") + + plugin_module_dir = ext_plugins_dir / "test_plugin" + plugin_module_dir.mkdir() + + plugin_code = ''' +"""Test external plugin module""" +from nodescraper.base import InBandDataPlugin +from nodescraper.enums import ExecutionStatus +from nodescraper.interfaces import DataAnalyzer + +class TestAnalyzer(DataAnalyzer): + DATA_MODEL = dict + + def analyze_data(self, data): + return ExecutionStatus.SUCCESS, None + +class TestExternalPlugin(InBandDataPlugin): + DATA_MODEL = dict + ANALYZER = TestAnalyzer + + def run(self): + return ExecutionStatus.SUCCESS, {"test": "data"} +''' + (plugin_module_dir / "__init__.py").write_text(plugin_code) + + sys.path.insert(0, tmpdir) + + try: + import test_external_pkg.ext_nodescraper_plugins as test_ext_pkg + + plugin_registry = PluginRegistry(plugin_pkg=[test_ext_pkg]) + + assert ( + "TestExternalPlugin" in plugin_registry.plugins + ), f"External plugin not found. Available plugins: {list(plugin_registry.plugins.keys())}" + + plugin_class = plugin_registry.plugins["TestExternalPlugin"] + assert plugin_class.__name__ == "TestExternalPlugin" + + finally: + sys.path.remove(tmpdir) + modules_to_remove = [ + key for key in sys.modules.keys() if key.startswith("test_external_pkg") + ] + for module in modules_to_remove: + del sys.modules[module] + + +def test_discover_and_load_external_plugins(): + """Test the full flow: discover external plugins using mocked modules.""" + mock_plugin_module = types.ModuleType("mock_ext_nodescraper_plugins") + mock_plugin_module.__file__ = "/fake/path/mock_ext_nodescraper_plugins/__init__.py" + mock_plugin_module.__path__ = ["/fake/path/mock_ext_nodescraper_plugins"] + + mock_submodule = types.ModuleType("mock_ext_nodescraper_plugins.mock_plugin") + mock_submodule.__file__ = "/fake/path/mock_ext_nodescraper_plugins/mock_plugin.py" + + class MockAnalyzer(DataAnalyzer): + DATA_MODEL = dict + + def analyze_data(self, data): + return ExecutionStatus.SUCCESS, None + + class MockExternalPlugin(InBandDataPlugin): + DATA_MODEL = dict + ANALYZER = MockAnalyzer + + def run(self): + return ExecutionStatus.SUCCESS, {} + + mock_submodule.MockExternalPlugin = MockExternalPlugin + + def mock_iter_modules(path, prefix=""): + yield None, f"{prefix}mock_plugin", False + + def mock_import_module(name): + if "mock_plugin" in name: + return mock_submodule + raise ImportError(f"No module named {name}") + + with mock.patch("pkgutil.iter_modules", side_effect=mock_iter_modules): + with mock.patch("importlib.import_module", side_effect=mock_import_module): + plugin_registry = PluginRegistry(plugin_pkg=[mock_plugin_module]) + + assert "MockExternalPlugin" in plugin_registry.plugins + assert plugin_registry.plugins["MockExternalPlugin"] == MockExternalPlugin From 4142cf1b40587d7f550158f022246617c8b1147e Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 8 Jan 2026 14:14:16 -0600 Subject: [PATCH 4/8] cleanup --- test/unit/framework/test_cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/unit/framework/test_cli.py b/test/unit/framework/test_cli.py index 095a852b..49cd8990 100644 --- a/test/unit/framework/test_cli.py +++ b/test/unit/framework/test_cli.py @@ -285,17 +285,16 @@ def test_external_plugins_integration(): with tempfile.TemporaryDirectory() as tmpdir: pkg_dir = Path(tmpdir) / "test_external_pkg" pkg_dir.mkdir() - (pkg_dir / "__init__.py").write_text("# Test external package\n") + (pkg_dir / "__init__.py").write_text("") ext_plugins_dir = pkg_dir / "ext_nodescraper_plugins" ext_plugins_dir.mkdir() - (ext_plugins_dir / "__init__.py").write_text("# External plugins package\n") + (ext_plugins_dir / "__init__.py").write_text("") plugin_module_dir = ext_plugins_dir / "test_plugin" plugin_module_dir.mkdir() - plugin_code = ''' -"""Test external plugin module""" + plugin_code = """ from nodescraper.base import InBandDataPlugin from nodescraper.enums import ExecutionStatus from nodescraper.interfaces import DataAnalyzer @@ -312,7 +311,7 @@ class TestExternalPlugin(InBandDataPlugin): def run(self): return ExecutionStatus.SUCCESS, {"test": "data"} -''' +""" (plugin_module_dir / "__init__.py").write_text(plugin_code) sys.path.insert(0, tmpdir) From fb9cef1cc55887cfc1cd696638545d32adc6ba88 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 8 Jan 2026 14:42:41 -0600 Subject: [PATCH 5/8] formatting --- nodescraper/cli/cli.py | 43 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 8caa1855..072703ca 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -58,68 +58,69 @@ def discover_external_plugins(): """Discover ext_nodescraper_plugins from all installed packages. - + Returns: list: List of discovered plugin packages """ extra_pkgs = [] seen_paths = set() # Track paths to avoid duplicates - + try: import ext_nodescraper_plugins as ext_pkg + extra_pkgs.append(ext_pkg) - if hasattr(ext_pkg, '__file__') and ext_pkg.__file__: + if hasattr(ext_pkg, "__file__") and ext_pkg.__file__: seen_paths.add(ext_pkg.__file__) except ImportError: pass - + # Discover ext_nodescraper_plugins from installed packages try: from importlib.metadata import distributions - + for dist in distributions(): - pkg_name = dist.metadata.get('Name', '') + pkg_name = dist.metadata.get("Name", "") if not pkg_name: continue - + name_variants = [ - pkg_name.replace('-', '_'), - pkg_name.replace('_', '-'), + pkg_name.replace("-", "_"), + pkg_name.replace("_", "-"), ] - + try: - top_level = dist.read_text('top_level.txt') + top_level = dist.read_text("top_level.txt") if top_level: - name_variants.extend(top_level.strip().split('\n')) + name_variants.extend(top_level.strip().split("\n")) except Exception: pass - + for variant in name_variants: if not variant: continue - + try: module_path = f"{variant}.ext_nodescraper_plugins" ext_pkg = import_module(module_path) - + # Check if we already have this package (by file path) - pkg_path = getattr(ext_pkg, '__file__', None) + pkg_path = getattr(ext_pkg, "__file__", None) if pkg_path and pkg_path in seen_paths: continue - + # Add the package extra_pkgs.append(ext_pkg) if pkg_path: seen_paths.add(pkg_path) - + break - + except (ImportError, AttributeError, ModuleNotFoundError): continue - + except Exception: pass - + return extra_pkgs From 55920b2ebf40cad0932ab26edb27c196724c5eba Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Thu, 8 Jan 2026 15:06:23 -0600 Subject: [PATCH 6/8] Relax paramiko requirement to >=3.2.0,<4.0.0 - Changed from paramiko~=3.5.1 to paramiko>=3.2.0,<4.0.0 - Allows broader compatibility with downstream dependencies - Uses only basic paramiko features stable since 3.2.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 358d9cf2..7b0c0f35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = ["Topic :: Software Development"] dependencies = [ "pydantic>=2.8.2", - "paramiko~=3.5.1", + "paramiko>=3.2.0,<4.0.0", "requests", "pytz" ] From 4493e17776524075fe0d93efd6d70c856dc32c9c Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Fri, 9 Jan 2026 11:05:41 -0600 Subject: [PATCH 7/8] urllib3 version downgrade --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b0c0f35..831d83ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dependencies = [ "pydantic>=2.8.2", "paramiko>=3.2.0,<4.0.0", "requests", - "pytz" + "pytz", + "urllib3>=1.26.15,<2.0.0" ] [project.optional-dependencies] From f409344b7796ba1166841e65e34068d695926b9b Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Fri, 9 Jan 2026 11:49:22 -0600 Subject: [PATCH 8/8] When node-scraper runs as an entry point script, Python adds venv/bin (or venv\Scripts on Windows) to sys.path[0] This breaks importlib.metadata.distributions() from discovering editable installs which results in xternal plugins from packages like amd-error-scraper weren't being found --- nodescraper/cli/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 072703ca..ff9eddd6 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -124,8 +124,16 @@ def discover_external_plugins(): return extra_pkgs +# Fix sys.path[0] if it's the venv/bin directory to avoid breaking editable install discovery +_original_syspath0 = sys.path[0] +if _original_syspath0.endswith("/bin") or _original_syspath0.endswith("\\Scripts"): + sys.path[0] = "" + extra_pkgs = discover_external_plugins() +# Restore original sys.path[0] +sys.path[0] = _original_syspath0 + def build_parser( plugin_reg: PluginRegistry,