rest/v1/catalog.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2019 - 2024
0010
0011
0012 import traceback
0013
0014 from flask import Blueprint
0015
0016 from idds.common import exceptions
0017 from idds.common.constants import HTTP_STATUS_CODE, CollectionRelationType
0018 from idds.core.catalog import get_collections, get_contents, get_contents_ext, combine_contents_ext
0019 from idds.rest.v1.controller import IDDSController
0020
0021
0022 class Collections(IDDSController):
0023     """ Catalog """
0024
0025     def get(self, request_id, transform_id, workload_id, scope, name, relation_type):
0026         """ Get collections by request_id, transform_id, workload_id, scope and name
0027         HTTP Success:
0028             200 OK
0029         HTTP Error:
0030             404 Not Found
0031             500 InternalError
0032         :returns: dictionary of an request.
0033         """
0034
0035         try:
0036             if request_id in ['null', 'None']:
0037                 request_id = None
0038             else:
0039                 request_id = int(request_id)
0040             if transform_id in ['null', 'None']:
0041                 transform_id = None
0042             else:
0043                 transform_id = int(transform_id)
0044             if workload_id in ['null', 'None']:
0045                 workload_id = None
0046             else:
0047                 workload_id = int(workload_id)
0048             if scope in ['null', 'None']:
0049                 scope = None
0050             if name in ['null', 'None']:
0051                 name = None
0052             if relation_type in ['null', 'None']:
0053                 relation_type = None
0054             else:
0055                 relation_type = int(relation_type)
0056
0057             rets = get_collections(request_id=request_id, transform_id=transform_id, workload_id=workload_id,
0058                                    scope=scope, name=name, relation_type=relation_type, to_json=False)
0059         except exceptions.NoObject as error:
0060             return self.generate_http_response(HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error)
0061         except exceptions.IDDSException as error:
0062             return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error)
0063         except Exception as error:
0064             print(error)
0065             print(traceback.format_exc())
0066             return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
0067
0068         return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets)
0069
0070
0071 class Contents(IDDSController):
0072     """ Catalog """
0073
0074     def get(self, request_id, transform_id, workload_id, coll_scope, coll_name, relation_type, status):
0075         """ Get contents by request_id, transform_id, workload_id coll_scope, coll_name, relation_type and status.
0076         HTTP Success:
0077             200 OK
0078         HTTP Error:
0079             404 Not Found
0080             500 InternalError
0081         :returns: contents.
0082         """
0083
0084         try:
0085             if coll_scope in ['null', 'None']:
0086                 coll_scope = None
0087             if coll_name in ['null', 'None']:
0088                 coll_name = None
0089             if request_id in ['null', 'None']:
0090                 request_id = None
0091             else:
0092                 request_id = int(request_id)
0093             if transform_id in ['null', 'None']:
0094                 transform_id = None
0095             else:
0096                 transform_id = int(transform_id)
0097             if workload_id in ['null', 'None']:
0098                 workload_id = None
0099             else:
0100                 workload_id = int(workload_id)
0101             if relation_type in ['null', 'None']:
0102                 relation_type = None
0103             else:
0104                 relation_type = int(relation_type)
0105             if status in ['null', 'None']:
0106                 status = None
0107             else:
0108                 status = int(status)
0109
0110             rets = get_contents(request_id=request_id, transform_id=transform_id, workload_id=workload_id, coll_scope=coll_scope,
0111                                 coll_name=coll_name, relation_type=relation_type, status=status, to_json=False)
0112         except exceptions.NoObject as error:
0113             return self.generate_http_response(HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error)
0114         except exceptions.IDDSException as error:
0115             return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error)
0116         except Exception as error:
0117             print(error)
0118             print(traceback.format_exc())
0119             return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
0120
0121         return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets)
0122
0123
0124 class ContentsOutputExt(IDDSController):
0125     """ Catalog """
0126
0127     def get(self, request_id, workload_id, transform_id, group_by_jedi_task_id=False):
0128         """ Get contents by request_id, workload_id and transform_id.
0129         HTTP Success:
0130             200 OK
0131         HTTP Error:
0132             404 Not Found
0133             500 InternalError
0134         :returns: contents.
0135         """
0136
0137         try:
0138             if request_id in ['null', 'None']:
0139                 request_id = None
0140             else:
0141                 request_id = int(request_id)
0142             if workload_id in ['null', 'None']:
0143                 workload_id = None
0144             else:
0145                 workload_id = int(workload_id)
0146             if transform_id in ['null', 'None']:
0147                 transform_id = None
0148             else:
0149                 transform_id = int(transform_id)
0150             if group_by_jedi_task_id:
0151                 if type(group_by_jedi_task_id) in [bool]:
0152                     pass
0153                 else:
0154                     if type(group_by_jedi_task_id) in [str] and group_by_jedi_task_id.lower() in ['true']:
0155                         group_by_jedi_task_id = True
0156                     else:
0157                         group_by_jedi_task_id = False
0158             else:
0159                 group_by_jedi_task_id = False
0160
0161             if request_id is None:
0162                 return self.generate_http_response(HTTP_STATUS_CODE.BadRequest,
0163                                                    exc_cls=exceptions.BadRequest.__name__,
0164                                                    exc_msg="request_id must not be None")
0165
0166             else:
0167                 contents = get_contents(request_id=request_id, workload_id=workload_id, transform_id=transform_id,
0168                                         relation_type=CollectionRelationType.Output)
0169                 contents_ext = get_contents_ext(request_id=request_id, workload_id=workload_id, transform_id=transform_id)
0170
0171                 ret_contents = combine_contents_ext(contents, contents_ext, with_status_name=True)
0172                 rets = {}
0173                 for content in ret_contents:
0174                     if group_by_jedi_task_id:
0175                         jedi_task_id = content.get('jedi_task_id', 'None')
0176                         if jedi_task_id not in rets:
0177                             rets[jedi_task_id] = []
0178                         rets[jedi_task_id].append(content)
0179                     else:
0180                         transform_id = content.get('transform_id')
0181                         if transform_id not in rets:
0182                             rets[transform_id] = []
0183                         rets[transform_id].append(content)
0184         except exceptions.NoObject as error:
0185             return self.generate_http_response(HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error)
0186         except exceptions.IDDSException as error:
0187             return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error)
0188         except Exception as error:
0189             print(error)
0190             print(traceback.format_exc())
0191             return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
0192
0193         return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets)
0194
0195
0196 """----------------------
0197    Web service url maps
0198 ----------------------"""
0199
0200
0201 def get_blueprint():
0202     bp = Blueprint('catalog', __name__)
0203
0204     # catalog_view = Catalog.as_view('catalog')
0205
0206     collections_view = Collections.as_view('collections')
0207     bp.add_url_rule('/catalog/collections/<request_id>/<transform_id>/<workload_id>/<scope>/<name>/<relation_type>',
0208                     view_func=collections_view, methods=['get', ])  # get collections
0209
0210     contents_view = Contents.as_view('contents')
0211     bp.add_url_rule('/catalog/contents/<request_id>/<transform_id>/<workload_id>/<coll_scope>/<coll_name>/<relation_type>/<status>',
0212                     view_func=contents_view, methods=['get', ])  # get contents
0213
0214     contents_ext_view = ContentsOutputExt.as_view('contents_output_ext')
0215     bp.add_url_rule('/catalog/contents_output_ext/<request_id>/<workload_id>/<transform_id>/<group_by_jedi_task_id>',
0216                     view_func=contents_ext_view, methods=['get', ])
0217
0218     return bp