v1/tests/harvester_api_tests.py

0001 import json
0002 import unittest
0003 from datetime import datetime, timezone
0004
0005 from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
0006
0007 # Get current UTC time with microseconds. The format needs to be compatible with the one used in the database
0008 now_utc = datetime.now(timezone.utc)
0009 formatted_time = now_utc.strftime("%d.%m.%y %H:%M:%S") + f".{now_utc.microsecond // 1000:02d}"
0010
0011 HARVESTER_ID = "test_fbarreir"
0012 HARVESTER_HOST = "test_fbarreir.cern.ch"
0013 PANDA_QUEUE = "test_queue"
0014
0015
0016 class TestHarvesterAPI(unittest.TestCase):
0017     def setUp(self):
0018         self.http_client = HttpClient()
0019
0020     def test_update_service_metrics(self):
0021         url = f"{api_url_ssl}/harvester/update_service_metrics"
0022         print(f"Testing URL: {url}")
0023         harvester_id = HARVESTER_ID
0024         harvester_host = HARVESTER_HOST
0025         creation_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
0026         metric = {
0027             "rss_mib": 2737.36,
0028             "memory_pc": 39.19,
0029             "cpu_pc": 15.23,
0030             "volume_data_pc": 20.0,
0031             "cert_lifetime": {
0032                 "/data/atlpan/proxy/x509up_u25606_prod": 81,
0033                 "/data/atlpan/proxy/x509up_u25606_pilot": 81,
0034                 "/cephfs/atlpan/harvester/proxy/x509up_u25606_prod": 96,
0035                 "/cephfs/atlpan/harvester/proxy/x509up_u25606_pilot": 96,
0036             },
0037         }
0038
0039         # DBProxy expects the metrics in json format and stores them directly in the database
0040         metrics = [[creation_time, harvester_host, json.dumps(metric)]]
0041
0042         data = {"harvester_id": harvester_id, "metrics": metrics}
0043         status, output = self.http_client.post(url, data)
0044         print(output)
0045         expected_response = {"success": True, "message": "", "data": [True]}
0046         self.assertEqual(output, expected_response)
0047
0048     def test_add_dialogs(self):
0049         url = f"{api_url_ssl}/harvester/add_dialogs"
0050         print(f"Testing URL: {url}")
0051         harvester_id = HARVESTER_ID
0052
0053         dialog = {
0054             "diagID": 1,
0055             "moduleName": "test_module",
0056             "identifier": "test identifier",
0057             "creationTime": datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
0058             "messageLevel": "INFO",
0059             "diagMessage": "test message",
0060         }
0061
0062         dialogs = [dialog]
0063
0064         data = {"harvester_id": harvester_id, "dialogs": dialogs}
0065         status, output = self.http_client.post(url, data)
0066         print(output)
0067         expected_response = {"success": True, "message": "", "data": None}
0068         self.assertEqual(output, expected_response)
0069
0070     def test_heartbeat(self):
0071         url = f"{api_url_ssl}/harvester/heartbeat"
0072         print(f"Testing URL: {url}")
0073         harvester_id = HARVESTER_ID
0074         data = {"harvester_id": harvester_id, "data": []}
0075         status, output = self.http_client.post(url, data)
0076
0077         expected_response = {"success": True, "message": "", "data": None}
0078         self.assertEqual(output, expected_response)
0079
0080     def test_get_worker_statistics(self):
0081         url = f"{api_url_ssl}/harvester/get_worker_statistics"
0082         print(f"Testing URL: {url}")
0083         data = {}
0084         status, output = self.http_client.get(url, data)
0085         print(output)
0086         # the statistics can't be predicted, so we just check the type of the response
0087         self.assertEqual(True, output["success"])
0088         self.assertEqual(dict, type(output["data"]))
0089
0090     def test_get_current_worker_id(self):
0091         url = f"{api_url_ssl}/harvester/get_current_worker_id"
0092         print(f"Testing URL: {url}")
0093         data = {"harvester_id": HARVESTER_ID}
0094         status, output = self.http_client.get(url, data)
0095         print(output)
0096         # the current/max worker id can't be predicted, so we just check the type of the response
0097         self.assertEqual(True, output["success"])
0098         self.assertEqual(int, type(output["data"]))
0099
0100     def test_report_worker_statistics(self):
0101         url = f"{api_url_ssl}/harvester/report_worker_statistics"
0102         print(f"Testing URL: {url}")
0103         harvester_id = HARVESTER_ID
0104         panda_queue = PANDA_QUEUE
0105         # the json loads is done in DBProxy
0106         statistics = json.dumps({"user": {"SCORE": {"running": 1, "submitted": 1}}, "managed": {"MCORE": {"running": 1, "submitted": 1}}})
0107         data = {"harvester_id": harvester_id, "panda_queue": panda_queue, "statistics": statistics}
0108         status, output = self.http_client.post(url, data)
0109         print(output)
0110         expected_response = {"success": True, "message": "OK", "data": None}
0111         self.assertEqual(output, expected_response)
0112
0113     def test_update_workers(self):
0114         url = f"{api_url_ssl}/harvester/update_workers"
0115         print(f"Testing URL: {url}")
0116         worker = {
0117             "workerID": 1,
0118             "batchID": 1,
0119             "queueName": "queue1",
0120             "status": "running",
0121             "computingSite": "site1",
0122             "nCore": 1,
0123             "nodeID": None,
0124             "submitTime": "02-NOV-24 00:02:18",
0125             "startTime": "02-NOV-24 00:02:18",
0126             "endTime": None,
0127             "jobType": "managed",
0128             "resourceType": "SCORE",
0129             "nativeExitCode": None,
0130             "nativeStatus": None,
0131             "diagMessage": None,
0132             "nJobs": 1,
0133             "computingElement": "ce1",
0134             "syncLevel": 0,
0135             "submissionHost": "submissionhost1",
0136             "harvesterHost": "harvesterhost1",
0137             "errorCode": None,
0138             "minRamCount": 2000,
0139         }
0140         workers = [worker]
0141
0142         harvester_id = HARVESTER_ID
0143         data = {"harvester_id": harvester_id, "workers": workers}
0144
0145         status, output = self.http_client.post(url, data)
0146         print(output)
0147         expected_response = {"success": True, "message": "", "data": [True]}
0148
0149         self.assertEqual(output, expected_response)
0150
0151     def test_acquire_commands(self):
0152         url = f"{api_url_ssl}/harvester/acquire_commands"
0153         print(f"Testing URL: {url}")
0154
0155         harvester_id = HARVESTER_ID
0156         n_commands = 1
0157         data = {"harvester_id": harvester_id, "n_commands": n_commands}
0158         status, output = self.http_client.post(url, data)
0159         print(output)
0160         # the commands can't be predicted, so we just check the type of the response
0161         self.assertEqual(True, output["success"])
0162         self.assertEqual(list, type(output["data"]))
0163
0164     def test_acknowledge_commands(self):
0165         url = f"{api_url_ssl}/harvester/acknowledge_commands"
0166         print(f"Testing URL: {url}")
0167         command_ids = [1]
0168         data = {"command_ids": command_ids}
0169         status, output = self.http_client.post(url, data)
0170         print(output)
0171         expected_response = {"success": True, "message": "", "data": None}
0172         self.assertEqual(output, expected_response)
0173
0174     def test_add_target_slots(self):
0175         url = f"{api_url_ssl}/harvester/add_target_slots"
0176         print(f"Testing URL: {url}")
0177         panda_queue = PANDA_QUEUE
0178         slots = 0
0179         data = {"panda_queue": panda_queue, "slots": slots}
0180         status, output = self.http_client.post(url, data)
0181         print(output)
0182         self.assertEqual(True, output["success"])
0183         self.assertEqual(None, output["data"])
0184
0185
0186 # Run tests
0187 if __name__ == "__main__":
0188     unittest.main()