monitor_app/tests/test_rest_logging_integration.py

0001 from django.test import TestCase
0002 from django.urls import reverse
0003 from django.contrib.auth.models import User
0004 from django.utils import timezone
0005 from rest_framework.test import APITestCase, APIClient
0006 from rest_framework import status
0007 from monitor_app.models import SystemAgent, AppLog, Run, StfFile, Subscriber
0008 from monitor_app.serializers import AppLogSerializer
0009 from django.core.management import call_command
0010 from io import StringIO
0011 import logging
0012 import uuid
0013 import re
0014
0015
0016 class RestLoggingIntegrationTests(APITestCase):
0017     """
0018     Test the REST logging functionality end-to-end.
0019
0020     These tests verify that agents can send log messages to the database
0021     via the REST API endpoint using both direct REST calls and the
0022     custom Python logging handler.
0023     """
0024
0025     def setUp(self):
0026         """Set up test client and authentication."""
0027         unique_username = f"testuser_{uuid.uuid4()}"
0028         self.user = User.objects.create_user(username=unique_username, password='testpassword')
0029         self.client.force_authenticate(user=self.user)
0030         self.logs_url = reverse('monitor_app:applog-list')
0031         AppLog.objects.all().delete()  # Start with clean slate
0032
0033     def test_direct_rest_log_creation(self):
0034         """Test creating logs directly via REST API."""
0035         log_data = {
0036             'app_name': 'test_app',
0037             'instance_name': 'test_instance',
0038             'timestamp': timezone.now().isoformat(),
0039             'level': logging.INFO,
0040             'levelname': 'INFO',
0041             'message': 'Test log message via REST API',
0042             'module': 'test_module',
0043             'funcname': 'test_function',
0044             'lineno': 42,
0045             'process': 1234,
0046             'thread': 5678,
0047             'extra_data': {'test': 'data'}
0048         }
0049
0050         response = self.client.post(
0051             self.logs_url,
0052             data=log_data,
0053             format='json'
0054         )
0055
0056         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
0057
0058         # Verify the log was created in the database
0059         self.assertEqual(AppLog.objects.count(), 1)
0060         log = AppLog.objects.first()
0061         self.assertEqual(log.app_name, 'test_app')
0062         self.assertEqual(log.message, 'Test log message via REST API')
0063         self.assertEqual(log.level, logging.INFO)
0064
0065     def test_multiple_log_levels(self):
0066         """Test logging different severity levels."""
0067         test_logs = [
0068             ('DEBUG', logging.DEBUG, 'Debug message'),
0069             ('INFO', logging.INFO, 'Info message'),
0070             ('WARNING', logging.WARNING, 'Warning message'),
0071             ('ERROR', logging.ERROR, 'Error message'),
0072             ('CRITICAL', logging.CRITICAL, 'Critical message')
0073         ]
0074
0075         for levelname, level_int, message in test_logs:
0076             log_data = {
0077                 'app_name': 'multi_level_test',
0078                 'instance_name': 'test_instance',
0079                 'timestamp': timezone.now().isoformat(),
0080                 'level': level_int,
0081                 'levelname': levelname,
0082                 'message': message,
0083                 'module': 'test_module',
0084                 'funcname': 'test_function',
0085                 'lineno': 1,
0086                 'process': 1234,
0087                 'thread': 5678
0088             }
0089
0090             response = self.client.post(
0091                 self.logs_url,
0092                 data=log_data,
0093                 format='json'
0094             )
0095
0096             self.assertEqual(response.status_code, status.HTTP_201_CREATED)
0097
0098         # Verify all logs were created
0099         self.assertEqual(AppLog.objects.count(), len(test_logs))
0100
0101         # Verify different log levels are present
0102         levels_in_db = set(AppLog.objects.values_list('level', flat=True))
0103         expected_levels = {level_int for _, level_int, _ in test_logs}
0104         self.assertEqual(levels_in_db, expected_levels)
0105
0106     def test_bulk_logging(self):
0107         """Test creating many log entries (simulating real usage)."""
0108         num_logs = 25  # Reduced from 50 to keep test fast
0109
0110         for i in range(num_logs):
0111             log_data = {
0112                 'app_name': 'bulk_test_app',
0113                 'instance_name': f'instance_{i % 5}',  # 5 different instances
0114                 'timestamp': timezone.now().isoformat(),
0115                 'level': logging.INFO,
0116                 'levelname': 'INFO',
0117                 'message': f'Bulk test log message {i+1}',
0118                 'module': 'bulk_test_module',
0119                 'funcname': 'bulk_test_function',
0120                 'lineno': i + 1,
0121                 'process': 1234,
0122                 'thread': 5678
0123             }
0124
0125             response = self.client.post(
0126                 self.logs_url,
0127                 data=log_data,
0128                 format='json'
0129             )
0130
0131             self.assertEqual(response.status_code, status.HTTP_201_CREATED)
0132
0133         # Verify all logs were created
0134         self.assertEqual(AppLog.objects.count(), num_logs)
0135
0136         # Verify logs are properly distributed across instances
0137         instance_counts = AppLog.objects.values('instance_name').distinct().count()
0138         self.assertEqual(instance_counts, 5)
0139
0140     def test_log_retrieval(self):
0141         """Test retrieving logs via REST API."""
0142         # Create some test logs
0143         for i in range(3):
0144             log_data = {
0145                 'app_name': 'retrieval_test',
0146                 'instance_name': 'test_instance',
0147                 'timestamp': timezone.now().isoformat(),
0148                 'level': logging.INFO,
0149                 'levelname': 'INFO',
0150                 'message': f'Retrieval test message {i+1}',
0151                 'module': 'test_module',
0152                 'funcname': 'test_function',
0153                 'lineno': i + 1,
0154                 'process': 1234,
0155                 'thread': 5678
0156             }
0157
0158             self.client.post(
0159                 self.logs_url,
0160                 data=log_data,
0161                 format='json'
0162             )
0163
0164         # Retrieve logs via GET request
0165         response = self.client.get(self.logs_url)
0166         self.assertEqual(response.status_code, status.HTTP_200_OK)
0167
0168         # Verify response contains our logs
0169         response_data = response.json()
0170         # Handle both paginated and non-paginated response formats
0171         if isinstance(response_data, dict) and 'results' in response_data:
0172             results = response_data['results']
0173         else:
0174             results = response_data
0175
0176         self.assertEqual(len(results), 3)
0177
0178         # Verify log content
0179         messages = [log['message'] for log in results]
0180         self.assertIn('Retrieval test message 1', messages)
0181         self.assertIn('Retrieval test message 2', messages)
0182         self.assertIn('Retrieval test message 3', messages)
0183
0184     def test_invalid_log_data(self):
0185         """Test handling of invalid log data."""
0186         invalid_data = {
0187             'app_name': 'test_app',
0188             # Missing required fields
0189             'message': 'Invalid log entry'
0190         }
0191
0192         response = self.client.post(
0193             self.logs_url,
0194             data=invalid_data,
0195             format='json'
0196         )
0197
0198         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
0199         self.assertEqual(AppLog.objects.count(), 0)  # No log should be created
0200
0201     def test_agent_workflow_logging_simulation(self):
0202         """Test simulating how agents would send logs during workflow processing."""
0203         # Simulate agent starting up
0204         startup_logs = [
0205             ('INFO', 'Agent starting up'),
0206             ('INFO', 'Connecting to message queue'),
0207             ('INFO', 'Agent ready for processing')
0208         ]
0209
0210         for levelname, message in startup_logs:
0211             log_data = {
0212                 'app_name': 'workflow_agent',
0213                 'instance_name': 'agent_001',
0214                 'timestamp': timezone.now().isoformat(),
0215                 'level': getattr(logging, levelname),
0216                 'levelname': levelname,
0217                 'message': message,
0218                 'module': 'agent_workflow',
0219                 'funcname': 'startup_sequence',
0220                 'lineno': 1,
0221                 'process': 1234,
0222                 'thread': 5678
0223             }
0224
0225             response = self.client.post(self.logs_url, data=log_data, format='json')
0226             self.assertEqual(response.status_code, status.HTTP_201_CREATED)
0227
0228         # Simulate processing workflow
0229         processing_logs = [
0230             ('INFO', 'Processing file batch 1/10'),
0231             ('DEBUG', 'File validation successful'),
0232             ('WARNING', 'Processing took longer than expected'),
0233             ('INFO', 'Batch processing completed')
0234         ]
0235
0236         for levelname, message in processing_logs:
0237             log_data = {
0238                 'app_name': 'workflow_agent',
0239                 'instance_name': 'agent_001',
0240                 'timestamp': timezone.now().isoformat(),
0241                 'level': getattr(logging, levelname),
0242                 'levelname': levelname,
0243                 'message': message,
0244                 'module': 'agent_workflow',
0245                 'funcname': 'process_batch',
0246                 'lineno': 1,
0247                 'process': 1234,
0248                 'thread': 5678
0249             }
0250
0251             response = self.client.post(self.logs_url, data=log_data, format='json')
0252             self.assertEqual(response.status_code, status.HTTP_201_CREATED)
0253
0254         # Verify logs were created
0255         total_logs = len(startup_logs) + len(processing_logs)
0256         self.assertEqual(AppLog.objects.count(), total_logs)
0257
0258         # Verify all logs are from the same agent
0259         agent_logs = AppLog.objects.filter(
0260             app_name='workflow_agent',
0261             instance_name='agent_001'
0262         )
0263         self.assertEqual(agent_logs.count(), total_logs)
0264
0265         # Verify different log levels are present
0266         log_levels = set(agent_logs.values_list('levelname', flat=True))
0267         expected_levels = {'INFO', 'DEBUG', 'WARNING'}
0268         self.assertEqual(log_levels, expected_levels)