management/commands/load_computing_configs.py

0001 """
0002 Management command to load PanDA queue and Rucio endpoint configurations from JSON files.
0003 """
0004
0005 import json
0006 import os
0007 from pathlib import Path
0008 from django.core.management.base import BaseCommand
0009 from django.conf import settings
0010 from monitor_app.models import PandaQueue, RucioEndpoint
0011
0012
0013 class Command(BaseCommand):
0014     help = 'Load PanDA queue and Rucio endpoint configurations from JSON files'
0015
0016     def add_arguments(self, parser):
0017         parser.add_argument(
0018             '--config-dir',
0019             type=str,
0020             help='Path to config directory containing JSON files',
0021             default=None
0022         )
0023         parser.add_argument(
0024             '--clear',
0025             action='store_true',
0026             help='Clear existing data before loading',
0027         )
0028
0029     def handle(self, *args, **options):
0030         # Determine config directory
0031         if options['config_dir']:
0032             config_dir = Path(options['config_dir'])
0033         else:
0034             # Default to swf-testbed/config directory
0035             # Try to find it relative to the Django project
0036             testbed_root = Path(settings.BASE_DIR).parent.parent / 'swf-testbed'
0037             config_dir = testbed_root / 'config'
0038
0039             if not config_dir.exists():
0040                 # Try alternate path
0041                 config_dir = Path('/direct/eic+u/wenauseic/github/swf-testbed/config')
0042
0043         if not config_dir.exists():
0044             self.stdout.write(self.style.ERROR(f'Config directory not found: {config_dir}'))
0045             return
0046
0047         self.stdout.write(f'Loading configurations from: {config_dir}')
0048
0049         # Clear existing data if requested
0050         if options['clear']:
0051             self.stdout.write('Clearing existing data...')
0052             PandaQueue.objects.all().delete()
0053             RucioEndpoint.objects.all().delete()
0054             self.stdout.write(self.style.SUCCESS('Existing data cleared'))
0055
0056         # Load PanDA queues
0057         panda_file = config_dir / 'panda_queues.json'
0058         if panda_file.exists():
0059             self.load_panda_queues(panda_file)
0060         else:
0061             self.stdout.write(self.style.WARNING(f'PanDA queues file not found: {panda_file}'))
0062
0063         # Load Rucio endpoints
0064         rucio_file = config_dir / 'ddm_endpoints.json'
0065         if rucio_file.exists():
0066             self.load_rucio_endpoints(rucio_file)
0067         else:
0068             self.stdout.write(self.style.WARNING(f'Rucio endpoints file not found: {rucio_file}'))
0069
0070         self.stdout.write(self.style.SUCCESS('Configuration loading complete'))
0071
0072     def load_panda_queues(self, file_path):
0073         """Load PanDA queue configurations from JSON file."""
0074         self.stdout.write(f'Loading PanDA queues from {file_path}...')
0075
0076         try:
0077             with open(file_path, 'r') as f:
0078                 data = json.load(f)
0079
0080             created_count = 0
0081             updated_count = 0
0082
0083             for queue_name, config in data.items():
0084                 # Extract key fields from config
0085                 site = config.get('site', '')
0086                 queue_type = config.get('type', '')
0087
0088                 # Determine status based on config
0089                 status = 'active'  # Default to active
0090                 if config.get('status') == 'offline':
0091                     status = 'offline'
0092
0093                 # Create or update queue
0094                 queue, created = PandaQueue.objects.update_or_create(
0095                     queue_name=queue_name,
0096                     defaults={
0097                         'site': site,
0098                         'queue_type': queue_type,
0099                         'status': status,
0100                         'config_data': config,
0101                     }
0102                 )
0103
0104                 if created:
0105                     created_count += 1
0106                 else:
0107                     updated_count += 1
0108
0109             self.stdout.write(
0110                 self.style.SUCCESS(
0111                     f'PanDA queues: {created_count} created, {updated_count} updated'
0112                 )
0113             )
0114
0115         except Exception as e:
0116             self.stdout.write(
0117                 self.style.ERROR(f'Error loading PanDA queues: {str(e)}')
0118             )
0119
0120     def load_rucio_endpoints(self, file_path):
0121         """Load Rucio endpoint configurations from JSON file."""
0122         self.stdout.write(f'Loading Rucio endpoints from {file_path}...')
0123
0124         try:
0125             with open(file_path, 'r') as f:
0126                 data = json.load(f)
0127
0128             created_count = 0
0129             updated_count = 0
0130
0131             for endpoint_name, config in data.items():
0132                 # Extract key fields from config
0133                 site = config.get('rcsite', config.get('site', ''))
0134                 is_tape = config.get('is_tape', False)
0135
0136                 # Determine endpoint type
0137                 if is_tape:
0138                     endpoint_type = 'tape'
0139                 elif config.get('is_cache'):
0140                     endpoint_type = 'cache'
0141                 else:
0142                     endpoint_type = 'disk'
0143
0144                 # Check if active based on rc_site_state
0145                 is_active = config.get('rc_site_state') == 'ACTIVE'
0146
0147                 # Create or update endpoint
0148                 endpoint, created = RucioEndpoint.objects.update_or_create(
0149                     endpoint_name=endpoint_name,
0150                     defaults={
0151                         'site': site,
0152                         'endpoint_type': endpoint_type,
0153                         'is_tape': is_tape,
0154                         'is_active': is_active,
0155                         'config_data': config,
0156                     }
0157                 )
0158
0159                 if created:
0160                     created_count += 1
0161                 else:
0162                     updated_count += 1
0163
0164             self.stdout.write(
0165                 self.style.SUCCESS(
0166                     f'Rucio endpoints: {created_count} created, {updated_count} updated'
0167                 )
0168             )
0169
0170         except Exception as e:
0171             self.stdout.write(
0172                 self.style.ERROR(f'Error loading Rucio endpoints: {str(e)}')
0173             )