workflow/examples/no_legacy_dependency_test.py

0001 """
0002 Offline check that the native workflow engine does not depend on the legacy workflow modules.
0003
0004 The native engine and the legacy iDDS-backed workflow processor
0005 (pandaserver.taskbuffer.workflow_processor) are meant to be mutually independent: the processor owns
0006 pcwl_utils, snakeparser and workflow_utils, and through workflow_utils it owns the idds dependency.
0007 Nothing in the native engine may reach any of them, which is what lets the engine run on a
0008 deployment with no idds and no snakemake installed.
0009
0010 The check imports every native module with only the third-party packages the native path
0011 legitimately needs stubbed out, and watches every import that goes past. idds and snakemake are
0012 deliberately left unavailable, and any attempt to import them is recorded even when the importer
0013 wraps it in try/except ImportError, so a coupling reintroduced anywhere in the import graph is
0014 caught here rather than on a deployment that has neither installed.
0015
0016 Run from the repository root:  python3 pandaserver/workflow/examples/no_legacy_dependency_test.py
0017 """
0018
0019 import importlib
0020 import importlib.abc
0021 import importlib.machinery
0022 import os
0023 import sys
0024 import types
0025 import warnings
0026
0027 # The import graph reaches unrelated modules that still carry unescaped regex literals. Their
0028 # SyntaxWarnings say nothing about this check, so they are filtered to keep the output readable.
0029 warnings.filterwarnings("ignore", category=SyntaxWarning)
0030
0031 REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
0032 sys.path.insert(0, REPO_ROOT)
0033
0034 # Native modules that must stand on their own
0035 NATIVE_MODULES = (
0036     "pandaserver.workflow.workflow_base",
0037     "pandaserver.workflow.workflow_native_utils",
0038     "pandaserver.workflow.workflow_parser",
0039     "pandaserver.workflow.workflow_core",
0040     "pandaserver.workflow.data_handler_plugins.base_data_handler",
0041     "pandaserver.workflow.data_handler_plugins.panda_task_data_handler",
0042     "pandaserver.workflow.data_handler_plugins.ddm_collection_data_handler",
0043     "pandaserver.workflow.step_handler_plugins.base_step_handler",
0044     "pandaserver.workflow.step_handler_plugins.panda_task_step_handler",
0045 )
0046
0047 # Third-party packages the native path may legitimately import. idds and snakemake are absent on
0048 # purpose: they must never be imported, so leaving them unstubbed turns a coupling into a failure.
0049 STUB_ROOTS = ("pandaclient", "ruamel", "requests", "rucio")
0050
0051 # Modules whose presence in sys.modules means a legacy dependency crept back in
0052 FORBIDDEN_ROOTS = ("idds", "snakemake")
0053 FORBIDDEN_MODULES = (
0054     "pandaserver.workflow.workflow_utils",
0055     "pandaserver.workflow.pcwl_utils",
0056     "pandaserver.workflow.snakeparser",
0057     "pandaserver.taskbuffer.workflow_processor",
0058 )
0059
0060
0061 # Every attempt to import a forbidden package, recorded here so that one suppressed by a
0062 # try/except ImportError in the importing module is still reported
0063 FORBIDDEN_ATTEMPTS: list[str] = []
0064
0065
0066 class ForbiddenWatcher(importlib.abc.MetaPathFinder):
0067     """Records attempts to import a forbidden package, then lets the import fail as normal"""
0068
0069     def find_spec(self, name, path=None, target=None):
0070         if name.split(".")[0] in FORBIDDEN_ROOTS:
0071             FORBIDDEN_ATTEMPTS.append(name)
0072         return None
0073
0074
0075 class AutoStubFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader):
0076     def find_spec(self, name, path=None, target=None):
0077         if name.split(".")[0] in STUB_ROOTS:
0078             return importlib.machinery.ModuleSpec(name, self, is_package=True)
0079         return None
0080
0081     def create_module(self, spec):
0082         module = types.ModuleType(spec.name)
0083         module.__path__ = []
0084         return module
0085
0086     def exec_module(self, module):
0087         class Anything:
0088             def __init__(self, *a, **k):
0089                 pass
0090
0091             def __call__(self, *a, **k):
0092                 return self
0093
0094         module.__getattr__ = lambda name: Anything
0095
0096
0097 sys.meta_path.insert(0, AutoStubFinder())
0098 sys.meta_path.insert(0, ForbiddenWatcher())
0099
0100
0101 def stub(name, **attrs):
0102     module = types.ModuleType(name)
0103     for key, value in attrs.items():
0104         setattr(module, key, value)
0105     sys.modules[name] = module
0106     return module
0107
0108
0109 class Log:
0110     def __init__(self, *a, **k):
0111         pass
0112
0113     def info(self, m):
0114         pass
0115
0116     debug = info
0117     warning = info
0118     error = info
0119
0120
0121 stub("pandacommon")
0122 stub("pandacommon.pandautils").__path__ = []
0123 stub("pandacommon.pandautils.base", SpecBase=object)
0124 stub("pandacommon.pandautils.PandaUtils", naive_utcnow=lambda: None, get_sql_IN_bind_variables=lambda *a, **k: (None, None))
0125 stub("pandacommon.pandautils.thread_utils", GenericThread=object)
0126 stub("pandacommon.pandalogger").__path__ = []
0127 stub("pandacommon.pandalogger.LogWrapper", LogWrapper=Log)
0128 stub("pandacommon.pandalogger.PandaLogger", PandaLogger=lambda: types.SimpleNamespace(getLogger=lambda n: None))
0129 stub("pandaserver.config", panda_config=types.SimpleNamespace(schemaJEDI="ATLAS_PANDA", schemaDEFT="ATLAS_DEFT"))
0130
0131
0132 def check(label, condition, detail=""):
0133     print(f"  {'PASS' if condition else 'FAIL'}  {label}{'  ' + str(detail) if detail and not condition else ''}")
0134     return condition
0135
0136
0137 def main():
0138     failures = 0
0139
0140     print("\n=== every native module imports without idds or snakemake ===")
0141     for module_name in NATIVE_MODULES:
0142         try:
0143             importlib.import_module(module_name)
0144             failures += not check(module_name, True)
0145         except Exception as exc:
0146             failures += not check(module_name, False, f"{type(exc).__name__}: {exc}")
0147
0148     print("\n=== nothing legacy was pulled into the import graph ===")
0149     reached_roots = sorted({name.split(".")[0] for name in sys.modules} & set(FORBIDDEN_ROOTS))
0150     failures += not check("no idds or snakemake package imported", not reached_roots, reached_roots)
0151     failures += not check("no idds or snakemake import even attempted", not FORBIDDEN_ATTEMPTS, sorted(set(FORBIDDEN_ATTEMPTS)))
0152     for forbidden in FORBIDDEN_MODULES:
0153         reached = sorted(name for name in sys.modules if name == forbidden or name.startswith(forbidden + "."))
0154         failures += not check(f"{forbidden} not imported", not reached, reached)
0155
0156     print(f"\n{'ALL CHECKS PASSED' if not failures else f'{failures} CHECK(S) FAILED'}")
0157     return 1 if failures else 0
0158
0159
0160 if __name__ == "__main__":
0161     sys.exit(main())