tests/CSGOptiXService_FastAPI_test/main.py

0001 """
0002 main.py
0003 ========
0004
0005 make_response
0006     returns fastapi.Response containing array and meta string
0007
0008 make_numpy_array_from_magic_bytes
0009     so far metadata not extracted
0010
0011 make_numpy_array_from_raw_bytes
0012
0013 parse_request
0014     extracting numpy array and values of some headers
0015
0016 simulate
0017     invokes C++ CSGOptiXService via _CSGOptiXService which straddles python/C++ barrier
0018
0019 array_create
0020     test endpoint
0021
0022 """
0023
0024
0025 import io
0026 import numpy as np
0027 from typing import Annotated, Optional
0028 #from pydantic import BaseModel
0029 from fastapi import FastAPI, Request, Response, Header, Depends, HTTPException
0030
0031 import opticks_CSGOptiX as cx
0032
0033 _svc = cx._CSGOptiXService()
0034
0035 app = FastAPI()
0036
0037
0038 def make_response( arr:np.ndarray, meta:str="", magic:bool=False, index:int=-1, level:int = 0):
0039     """
0040     https://stackoverflow.com/questions/15879315/what-is-the-difference-between-ndarray-and-array-in-numpy
0041
0042     numpy.array is just a convenience function to create an ndarray; it is not a class itself.
0043     """
0044     if level > 0:
0045         print("make_response arr:%s meta:%s magic:%s index:%s level:%s  " % (arr, meta,magic,index,level))
0046     pass
0047     headers = {}
0048     headers["x-opticks-index"] = str(index)
0049     headers["x-opticks-level"] = str(level)
0050     media_type = "application/octet-stream"
0051
0052     data:bytes = b''
0053
0054     if magic:
0055         buffer = io.BytesIO()
0056         np.save(buffer, arr)
0057         if len(meta) > 0:
0058             buffer.write(meta.encode("utf-8"))
0059         pass
0060         data = buffer.getvalue()
0061     else:
0062         headers["x-opticks-dtype"] = arr.dtype.name
0063         headers["x-opticks-shape"] = str(arr.shape)
0064         data = arr.tobytes('C')
0065     pass
0066     return Response(data, headers=headers, media_type=media_type )
0067
0068
0069
0070
0071 def make_numpy_array_from_magic_bytes(data:bytes, level:int=0):
0072     """
0073     :param data: bytes which are assumed to include the numpy magic header
0074     :return arr: numpy.ndarray constructed from the bytes
0075
0076     NB any metadata concatenated following array data is currently ignored
0077
0078     TODO: follow tests/NP_nanobind_test/meta_check.py to get meta from the bytes too
0079
0080     Used from parse_request when magic enabled
0081     """
0082     buffer = io.BytesIO(data)
0083     buffer.seek(0)
0084     arr = np.load(buffer)
0085     return arr
0086
0087 def make_numpy_array_from_magic_bytes_with_meta(data:bytes, level:int=0):
0088     """
0089     Test for this in ~/np/tests/NP_nanobind_test/meta_check.py
0090     """
0091
0092     if level > 0:print(f"[make_numpy_array_from_magic_bytes_with_meta")
0093
0094     buffer = io.BytesIO(data)
0095     buffer.seek(0)
0096     arr = np.load(buffer)
0097
0098     buf_nbytes = len(data) # buffer.getbuffer().nbytes
0099     hdr_nbytes = data.find(b'\n') + 1  # 1 + index of first newline, typically 128 but can be more for arrays with many dimensions
0100     arr_nbytes = arr.nbytes
0101     meta_nbytes = buf_nbytes - hdr_nbytes - arr_nbytes
0102
0103     buffer.seek( hdr_nbytes + arr_nbytes )
0104     _meta:Optional[bytes] = buffer.read(meta_nbytes) if meta_nbytes>0 else None
0105     meta = _meta.decode("utf-8")
0106
0107     if level > 0:print(f"-make_numpy_array_from_magic_bytes_with_meta buf_nbytes:{buf_nbytes} hdr_nbytes:{hdr_nbytes} arr_nbytes:{arr_nbytes} meta_nbytes:{meta_nbytes} meta:{meta} ")
0108     if level > 0:print(f"]make_numpy_array_from_magic_bytes_with_meta")
0109
0110     return arr, meta
0111
0112
0113
0114
0115
0116 def make_numpy_array_from_raw_bytes(data:bytes, dtype_:str, shape_:str, level:int=0 ):
0117     """
0118     :param data: bytes which are assumed to just carry array data, no header or metadata
0119     :param dtype_: str
0120     :param shape_: str
0121
0122     Used from parse_request when magic not enabled
0123
0124     raw bytes require dtype and shape metadata strings
0125     """
0126     dtype = getattr(np, dtype_, None)
0127     shape = tuple(map(int,filter(None,map(str.strip,shape_.replace("(","").replace(")","").split(",")))))
0128     a0 = np.frombuffer(data, dtype=dtype ).reshape(*shape)
0129     return a0
0130
0131
0132
0133 async def parse_request(request: Request):
0134     """
0135     :param request: FastAPI Request
0136     :return arr: NumPy array
0137
0138     Uses request body and headers with array dtype and shape to reconstruct the uploaded NumPy array
0139
0140     TODO: handle metadata concatenated after the array data
0141     """
0142     token_ = request.headers.get('x-opticks-token')
0143     if token_ != "secret":
0144         raise HTTPException(status_code=401, detail="x-opticks-token invalid")
0145     pass
0146
0147     level_ = request.headers.get('x-opticks-level','0')
0148     level = int(level_)
0149     index_ = request.headers.get('x-opticks-index','0')
0150     index = int(index_)
0151     type_ = request.headers.get('content-type')
0152
0153     if level > 0:
0154         print("[parse_request")
0155         print("request\n", request)
0156         print("request.url\n",request.url)
0157         print("request.headers\n",request.headers)
0158     pass
0159
0160     data:bytes = b''
0161     if type_.startswith("multipart/form-data"):
0162         field = "upload"  # needs to match field name from client
0163         form = await request.form()
0164         filename = form[field].filename
0165         data = await form[field].read()
0166     else:
0167         filename = None
0168         data = await request.body()
0169     pass
0170
0171     numpy_magic = b'\x93NUMPY'
0172     has_numpy_magic = data.startswith(numpy_magic)
0173     if has_numpy_magic:
0174         dtype_ = None
0175         shape_ = None
0176         #arr0 = make_numpy_array_from_magic_bytes(data, level)
0177         arr0, meta = make_numpy_array_from_magic_bytes_with_meta(data, level)
0178     else:
0179         dtype_ = request.headers.get('x-opticks-dtype','')
0180         shape_ = request.headers.get('x-opticks-shape','')
0181         arr0 = make_numpy_array_from_raw_bytes(data, dtype_, shape_, level )
0182         meta = None
0183     pass
0184     arr = arr0 if level == 10 else arr0.copy()
0185     ## without the copy get runtime type error in the nanobind call across the C++ python barrier
0186
0187     if level > 0:
0188         # https://numpy.org/doc/stable/reference/generated/numpy.ndarray.flags.html
0189         print("has_numpy_magic:%s" % has_numpy_magic )
0190         print("arr0.data.c_contiguous:%s" % arr0.data.c_contiguous )
0191         print("arr.data.c_contiguous:%s" % arr.data.c_contiguous )
0192         #print("arr0.flags:\n", arr0.flags)
0193         #print("arr.flags:\n", arr.flags)
0194         print("content-type:%s" % type_ )
0195         print("filename:%s" % filename )
0196         print("token_[%s]" % token_ )
0197         print("level[%d]" % level )
0198         print("index[%d]" % index )
0199         print("dtype_[%s]" % str(dtype_) )
0200         print("shape_[%s]" % str(shape_) )
0201         print("type(arr0)\n", type(arr0))
0202         print("type(arr)\n", type(arr))
0203         print("arr[%s]" % arr )
0204         print("meta[%s]" % meta )
0205         print("]parse_request")
0206     pass
0207     request.state.array = arr
0208     request.state.meta = meta
0209     request.state.index = index
0210     request.state.level = level
0211     request.state.magic = has_numpy_magic
0212
0213
0214
0215
0216 @app.post('/simulate', response_class=Response, dependencies=[Depends(parse_request)])
0217 async def simulate(request: Request):
0218     """
0219     :param request: Request
0220     :return response: Response
0221
0222     1. parse_request dependency sets request.state values
0223     2. call _svc with *gs* to give *ht*
0224     3. return *ht* as FastAPI Response
0225
0226     Test this with ~/np/tests/np_curl_test/np_curl_test.sh
0227     """
0228     gs = request.state.array
0229     gs_meta = request.state.meta
0230     index = request.state.index
0231     level = request.state.level
0232     magic = request.state.magic
0233
0234     if level > 0: print("main.py:simulate index %d gs %s gs_meta[%s] " % ( index, repr(gs), gs_meta ))
0235
0236     #ht = _svc.simulate(gs, index)   ## NB this wrapper from CSGOptiX/opticks_CSGOptiX handles numpy<=>NP conversion
0237     ht, ht_meta = _svc.simulate_with_meta(gs, gs_meta, index)
0238
0239     response = make_response(ht, ht_meta, magic, index, level)
0240
0241     return response
0242
0243
0244
0245
0246 @app.get('/array_create', response_class=Response)
0247 def array_create():
0248     """
0249     ::
0250
0251        curl -s -D /dev/stdout http://127.0.0.1:8000/array_create  --output arr
0252
0253     """
0254     arr = np.arange(32, dtype=np.float32).reshape(2,4,4)
0255     meta:str = ""
0256     magic:bool = True
0257     index:int = 1
0258     level:int = 1
0259
0260     response = make_response( arr, meta, magic, index, level )
0261     return response
0262
0263
0264
0265