Skip to content
18 changes: 9 additions & 9 deletions src/modules/ObjectDetectionCoral/modulesettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"RuntimeLocation": "Local", // Can be Local, Shared or System
"PostStartPauseSecs": 1, // Generally 1 if using GPU, 0 for CPU
"Queue": "objectdetection_queue", // We make all Object detectors use the same queue.
"Parallelism": 1 // 0 = Default = number of CPUs / 2
"Parallelism": 16 // Should probably be TPU count * 2; I don't see harm in overprovisioning threads
},

"ModelRequirements" : [{
Expand All @@ -36,7 +36,7 @@

"EnvironmentVariables": {
"CPAI_CORAL_MULTI_TPU": "true",
"CPAI_CORAL_MODEL_NAME": "MobileNet SSD", // "MobileNet SSD", "EfficientDet-Lite", "YOLOv5"
"CPAI_CORAL_MODEL_NAME": "MobileNet SSD", // "MobileNet SSD", "EfficientDet-Lite", "YOLOv5", "YOLOv8"

"MODELS_DIR": "%CURRENT_MODULE_PATH%/assets",
"MODEL_SIZE": "Small"
Expand Down Expand Up @@ -85,16 +85,16 @@
],
"DownloadableModels":[

{ "Name": "EfficientDet Large", "Filename": "objectdetection-efficientdet-large-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Large", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "EfficientDet Medium", "Filename": "objectdetection-efficientdet-medium-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "EfficientDet Small", "Filename": "objectdetection-efficientdet-small-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Small", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "EfficientDet Tiny", "Filename": "objectdetection-efficientdet-tiny-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Tiny", "FileSizeKb": 275800, "PreInstall": false },

{ "Name": "MobileNet Large", "Filename": "objectdetection-mobilenet-large-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Large", "FileSizeKb": 275800, "PreInstall": true },
{ "Name": "MobileNet Medium", "Filename": "objectdetection-mobilenet-medium-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Medium", "FileSizeKb": 275800, "PreInstall": true },
{ "Name": "MobileNet Small", "Filename": "objectdetection-mobilenet-small-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Small", "FileSizeKb": 275800, "PreInstall": true },
{ "Name": "MobileNet Tiny", "Filename": "objectdetection-mobilenet-tiny-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Tiny", "FileSizeKb": 275800, "PreInstall": true },

{ "Name": "EfficientDet Large", "Filename": "objectdetection-efficientdet-large-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Large", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "EfficientDet Medium", "Filename": "objectdetection-efficientdet-medium-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "EfficientDet Small", "Filename": "objectdetection-efficientdet-small-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Small", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "EfficientDet Tiny", "Filename": "objectdetection-efficientdet-tiny-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Tiny", "FileSizeKb": 275800, "PreInstall": false },

{ "Name": "YOLOv5 Large", "Filename": "objectdetection-yolov5-large-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Large", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "YOLOv5 Medium", "Filename": "objectdetection-yolov5-medium-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
{ "Name": "YOLOv5 Small", "Filename": "objectdetection-yolov5-small-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Small", "FileSizeKb": 275800, "PreInstall": false },
Expand All @@ -112,9 +112,9 @@
"Label": "Model",
"Options": [
{ "Label": "MobileNet SSD", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "MobileNet SSD" },
{ "Label": "EfficientDet Lite", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "EfficientDet-Lite" },
{ "Label": "YOLOv5", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv5" },
{ "Label": "YOLOv8", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv8" },
{ "Label": "EfficientDet Lite", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "EfficientDet-Lite" }
{ "Label": "YOLOv8", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv8" }
]
},
{
Expand Down
30 changes: 20 additions & 10 deletions src/modules/ObjectDetectionCoral/objectdetection_coral_multitpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,20 +231,28 @@ def main():

thread_cnt = 16
tot_infr_time = 0
half_wall_start = None
half_infr_count = 0
if args.count > 1:
with concurrent.futures.ThreadPoolExecutor(max_workers=thread_cnt) as executor:
start = time.perf_counter()
for chunk_i in range(0, args.count-1, thread_cnt*8):
fs = [executor.submit(_tpu_runner.process_image, options, copy.copy(image), args.threshold)
for i in range(min(thread_cnt*8, args.count-1 - chunk_i))]
for f in concurrent.futures.as_completed(fs):
_, infr_time = f.result()
_, infr_time, _ = f.result()
tot_infr_time += infr_time

# Start a timer for the last ~half of the run for more accurate benchmark
if chunk_i > (args.count-1) / 3.0:
half_infr_count += 1
if half_wall_start is None:
half_wall_start = time.perf_counter()

# Uncomment for testing
# import random
# logging.info("Pause")
# time.sleep(random.randint(0,INTERPRETER_LIFESPAN_SECONDS*2))
# time.sleep(random.randint(0,INTERPRETER_LIFESPAN_SECONDS*3))
else:
start = time.perf_counter()

Expand All @@ -254,20 +262,22 @@ def main():
# print(stat)

start_one = time.perf_counter()
objs, infr_time = _tpu_runner.process_image(options, image, args.threshold)
objs, infr_time, _ = _tpu_runner.process_image(options, copy.copy(image), args.threshold)
tot_infr_time += infr_time
half_infr_count += 1
wall_time = time.perf_counter() - start
print('completed one run every %.2f ms for %d runs; %.2f ms wall time for a single run' %

half_wall_time = 0.0
if half_wall_start is not None:
half_wall_time = time.perf_counter() - half_wall_start

print('completed one run every %.2fms for %d runs; %.2fms wall time for a single run' %
(wall_time * 1000 / args.count, args.count,
(time.perf_counter() - start_one) * 1000))

# Optimizing the number of segments used for a model would result in the
# lowest average time spent adjusted for number of TPUs used. At some point,
# adding additional segments just removes from the pool of TPUs you can use
# for parallelism.
print('%.2f ms avg time blocked across %d threads; %.2f avg TPU * ms / run' %
print('%.2fms avg time blocked across %d threads; %.2fms ea for final %d inferences' %
(tot_infr_time / args.count, thread_cnt,
len(_tpu_runner.pipe.tpu_list) * wall_time * 1000 / args.count))
half_wall_time * 1000 / half_infr_count, half_infr_count))

print('-------RESULTS--------')
if not objs:
Expand Down
4 changes: 2 additions & 2 deletions src/modules/ObjectDetectionCoral/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
print("Unable to import ModuleOptions, running with defaults")
class ModuleOptions:
module_path = '.'
def getEnvVariable(self, a, b):
def getEnvVariable(a, b):
return b

class Settings:
Expand Down Expand Up @@ -163,7 +163,7 @@ def __init__(self):

self.MAX_PIPELINE_QUEUE_LEN = 1000 # Multi-only
self.TILE_OVERLAP = 15 # Multi-only.
self.DOWNSAMPLE_BY = 5.2 # Multi-only. Smaller number results in more tiles generated
self.DOWNSAMPLE_BY = 6.0 # Multi-only. Smaller number results in more tiles generated
self.IOU_THRESHOLD = 0.1 # Multi-only

# ----------------------------------------------------------------------
Expand Down
210 changes: 0 additions & 210 deletions src/modules/ObjectDetectionCoral/pipelined_model_runner.py

This file was deleted.

20 changes: 9 additions & 11 deletions src/modules/ObjectDetectionCoral/segment_and_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import subprocess
import time
import shutil
import re

#'''
fn_list = [
Expand All @@ -11,12 +12,12 @@
#'ssd_mobilenet_v1_coco_quant_postprocess',
'tf2_ssd_mobilenet_v1_fpn_640x640_coco17_ptq',
#'efficientdet_lite0_320_ptq',
'efficientdet_lite1_384_ptq',
#'efficientdet_lite1_384_ptq',
'efficientdet_lite2_448_ptq',
'efficientdet_lite3_512_ptq',
'efficientdet_lite3x_640_ptq',
#'yolov5n-int8',
'yolov5s-int8',
#'yolov5s-int8',
'yolov5m-int8',
'yolov5l-int8',
#'yolov8n_416_640px', # lg 1st seg
Expand Down Expand Up @@ -192,7 +193,7 @@ def seg_exists(filename, segment_type, segment_count):

MAX_TPU_COUNT = 8

#'''
'''
# Generate segment files
for sn in range(1,MAX_TPU_COUNT+1):
for fn in fn_list:
Expand Down Expand Up @@ -313,15 +314,12 @@ def seg_exists(filename, segment_type, segment_count):
seg_list + ["--labels","coral/pycoral/test_data/coco_labels.txt","--input","/home/seth/coral/pycoral/test_data/grace_hopper.bmp",
"--count","1000","--num-tpus",str(num_tpus)]
print(cmd)
start_time = time.perf_counter()
subprocess.run(cmd)
timings.append(((time.perf_counter() - start_time), num_tpus, fn, seg_type, sn))
c = subprocess.run(cmd, capture_output=True)
print(c.stdout)
print(c.stderr)
ms_time = float(re.compile(r'threads; ([\d\.]+)ms ea').findall(c.stdout)[0])
timings.append((ms_time, num_tpus, fn, seg_type, sn))

# Find segment pipeline efficencies
#cmd = ["/home/seth/libcoral/out/k8/tools/model_pipelining_performance_analysis","--data_dir",seg_dir+seg_type,
# "--model_list",fn,"--num_segments_list",','.join([str(i) for i in range(1,max_seg+1)])]
#print(cmd)
#subprocess.run(cmd)
timings = sorted(timings, key=lambda t: t[0])

# Print the top three
Expand Down
Loading