Merge pull request #56 from duggalsu/cpu_profiling

Add cpu profiling and optimize operator

Author: rorybee

src/api/.gitignore

@@ -1,3 +1,8 @@
 credentials.json
+
+# memray
 *.bin
 *.html
+
+# pyinstrument
+*.json

src/api/core/operators/vid_vec_rep_resnet.py

@@ -3,8 +3,8 @@
 def initialize(param):
     print("Installing packages for vid_vec_rep_resnet")
 
-    global TextBlob, os, np, cv2, qr, torch, data, models, transforms, Image, FFmpeg, wget
-    global imagenet_transform, ImageListDataset, VideoAnalyzer, gendata, compress_video
+    global TextBlob, os, np, cv2, qr, torch, data, models, transforms, Image, wget #, FFmpeg
+    global imagenet_transform, ImageListDataset, VideoAnalyzer, gendata #, compress_video
 
     import os
     import numpy as np
@@ -15,7 +15,7 @@ def initialize(param):
     import torchvision.models as models
     import torchvision.transforms as transforms
     from PIL import Image
-    from ffmpy import FFmpeg
+    # from ffmpy import FFmpeg
     import wget
 
     imagenet_transform = transforms.Compose(
@@ -26,6 +26,7 @@ def initialize(param):
         ]
     )
 
+    '''
     def compress_video(fname):
         newname = "/tmp/compressed.mp4"
         FNULL = open(os.devnull, "w")
@@ -38,6 +39,8 @@ def compress_video(fname):
         ff.run(stdout=FNULL, stderr=FNULL)
         # os.remove(fname)
         return newname
+    '''
+
 
     def gendata(vid_analyzer):
         # average vector
@@ -205,11 +208,11 @@ def run(file):
     fname = file["path"]
     fsize = os.path.getsize(fname) / 1e6
     print("original size: ", fsize)
-    if fsize < 10:
-        print("compressing video")
-        fname = compress_video(fname)
-        fsize = os.path.getsize(fname) / 1e6
-        print("compressed video size: ", fsize)
+    # if fsize < 10:
+    #     print("compressing video")
+    #     fname = compress_video(fname)
+    #     fsize = os.path.getsize(fname) / 1e6
+    #     print("compressed video size: ", fsize)
     if fsize > 10:
         raise Exception("Video too large")
 
@@ -219,7 +222,7 @@ def run(file):
 
     doable, error_msg = vid_analyzer.check_constraints()
 
-    os.remove(fname)
+    # os.remove(fname)
 
     if not doable:
         raise "Unsupported Video. Cannot index video."

src/api/core/operators/vid_vec_rep_resnet_requirements.in

@@ -2,7 +2,6 @@ torch==2.1.2+cpu
 torchvision==0.16.2+cpu
 numpy==1.26.3
 wget==3.2
-ffmpy==0.3.1
 Pillow==10.2.0
 scipy==1.11.4
 opencv-python-headless==4.9.0.80

src/api/core/operators/vid_vec_rep_resnet_requirements.txt

@@ -10,8 +10,6 @@ certifi==2023.11.17
     # via requests
 charset-normalizer==3.3.2
     # via requests
-ffmpy==0.3.1
-    # via -r vid_vec_rep_resnet_requirements.in
 filelock==3.13.1
     # via torch
 fsspec==2023.12.2

src/api/requirements.in

@@ -22,3 +22,4 @@ pytest==7.4.4
 PyYAML==6.0.1
 dacite==1.8.1
 memray==1.11.0  # dev
+pyinstrument==4.6.2

src/api/requirements.txt

@@ -171,6 +171,8 @@ pyasn1-modules==0.2.8
     # via google-auth
 pygments==2.17.2
     # via rich
+pyinstrument==4.6.2
+    # via -r requirements.in
 pytest==7.4.4
     # via -r requirements.in
 python-dateutil==2.8.1