fix chronological consistency in compute queue with explicit wait

Author: mikex86

driverapi/include/librecuda.h

@@ -82,7 +82,8 @@ LIBRECUDA_EXPORT libreCudaStatus_t libreCuLaunchKernel(LibreCUFunction function,
                                                        uint32_t blockDimZ,
                                                        uint32_t sharedMemBytes, LibreCUstream stream,
                                                        void **kernelParams, size_t numParams,
-                                                       void **extra);
+                                                       void **extra,
+                                                       bool async=false);
 /**
  * Submits the built up command buffer to the gpu.
  * Operations performed on streams fall into two types: "compute" (eg. launch kernel) and "dma".

driverapi/internal/cmdqueue.h

@@ -173,7 +173,7 @@ class NvCommandQueue {
     /**
      * Waits for the pending operations in the currently executing command queue to complete
      */
-    libreCudaStatus_t awaitExecution(QueueType queueType);
+    libreCudaStatus_t awaitExecution();
 
     ~NvCommandQueue();
 
@@ -184,7 +184,7 @@ class NvCommandQueue {
                    uint32_t gridDimX, uint32_t gridDimY, uint32_t gridDimZ,
                    uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ,
                    uint32_t sharedMemBytes,
-                   void **params, size_t numParams);
+                   void **params, size_t numParams, bool async);
 
     libreCudaStatus_t gpuMemcpy(void *dst, void *src, size_t numBytes);
 
@@ -200,6 +200,14 @@ class NvCommandQueue {
 
     libreCudaStatus_t signalWaitCpu(NvSignal *pSignal, NvU32 signalTarget);
 
+    /**
+     * waits on the gpu compute queue until the signal reaches the specified target
+     * @param pSignal the signal
+     * @param signalTarget the target value
+     * @return status
+     */
+    libreCudaStatus_t signalWaitGpu(NvSignal *pSignal, NvU32 signalTarget);
+
     libreCudaStatus_t submitToFifo(QueueType type);
 
     libreCudaStatus_t allocKernArgs(NvU64 *pMemOut, size_t size);

driverapi/src/cmdqueue.cpp

@@ -131,20 +131,22 @@ libreCudaStatus_t NvCommandQueue::initializeQueue() {
                 )
         );
         timelineCtr++;
+        LIBRECUDA_ERR_PROPAGATE(signalNotify(timelineSignal, timelineCtr, COMPUTE));
     }
     LIBRECUDA_ERR_PROPAGATE(startExecution(COMPUTE));
-    LIBRECUDA_ERR_PROPAGATE(awaitExecution(COMPUTE));
+    LIBRECUDA_ERR_PROPAGATE(awaitExecution());
 
     // setup copy queue
     {
         LIBRECUDA_ERR_PROPAGATE(
                 enqueue(makeNvMethod(4, NVC6C0_SET_OBJECT, 1), {AMPERE_DMA_COPY_B}, DMA)
         );
         timelineCtr++;
+        LIBRECUDA_ERR_PROPAGATE(signalNotify(timelineSignal, timelineCtr, DMA));
     }
 
     LIBRECUDA_ERR_PROPAGATE(startExecution(DMA));
-    LIBRECUDA_ERR_PROPAGATE(awaitExecution(DMA));
+    LIBRECUDA_ERR_PROPAGATE(awaitExecution());
 
     // allocate kernargs page
     {
@@ -250,9 +252,10 @@ NvCommandQueue::~NvCommandQueue() {
     gpuFree(ctx, reinterpret_cast<NvU64>(kernArgsPageVa));
 }
 
-libreCudaStatus_t NvCommandQueue::awaitExecution(QueueType queueType) {
+libreCudaStatus_t NvCommandQueue::awaitExecution() {
     LIBRECUDA_VALIDATE(timelineSignal != nullptr, LIBRECUDA_ERROR_NOT_INITIALIZED);
     LIBRECUDA_ERR_PROPAGATE(signalWaitCpu(timelineSignal, timelineCtr));
+    kernArgsWriteIdx = 0; // reset kern args write idx
     LIBRECUDA_SUCCEED();
 }
 
@@ -339,12 +342,9 @@ libreCudaStatus_t NvCommandQueue::signalWaitCpu(NvSignal *pSignal, NvU32 signalT
 }
 
 libreCudaStatus_t NvCommandQueue::startExecution(QueueType queueType) {
-    LIBRECUDA_ERR_PROPAGATE(signalNotify(timelineSignal, timelineCtr, queueType));
     LIBRECUDA_ERR_PROPAGATE(submitToFifo(
             queueType
     ));
-
-    kernArgsWriteIdx = 0; // reset kern args write idx
     LIBRECUDA_SUCCEED();
 }
 
@@ -437,6 +437,8 @@ libreCudaStatus_t NvCommandQueue::ensureEnoughLocalMem(NvU32 localMemReq) {
         timelineCtr++;
     }
 
+    //LIBRECUDA_ERR_PROPAGATE(signalWaitGpu(timelineSignal, timelineCtr));
+
     LIBRECUDA_SUCCEED();
 }
 
@@ -448,10 +450,13 @@ NvCommandQueue::launchFunction(LibreCUFunction function,
                                uint32_t gridDimX, uint32_t gridDimY, uint32_t gridDimZ,
                                uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ,
                                uint32_t sharedMemBytes,
-                               void **params, size_t numParams) {
+                               void **params, size_t numParams,
+                               bool async) {
     LIBRECUDA_VALIDATE(function != nullptr, LIBRECUDA_ERROR_INVALID_VALUE);
     LIBRECUDA_VALIDATE(numParams == function->param_info.size(), LIBRECUDA_ERROR_INVALID_VALUE);
-
+    if (!async) {
+        LIBRECUDA_ERR_PROPAGATE(signalWaitGpu(timelineSignal, timelineCtr));
+    }
     LIBRECUDA_ERR_PROPAGATE(ensureEnoughLocalMem(function->local_mem_req));
 
     if (dmaCommandBuffer.empty()) {
@@ -673,6 +678,7 @@ NvCommandQueue::launchFunction(LibreCUFunction function,
         ));
     }
     timelineCtr++;
+    LIBRECUDA_ERR_PROPAGATE(signalNotify(timelineSignal, timelineCtr, COMPUTE));
     LIBRECUDA_SUCCEED();
 }
 
@@ -722,7 +728,7 @@ libreCudaStatus_t NvCommandQueue::gpuMemcpy(void *dst, void *src, size_t numByte
             DMA
     ));
     timelineCtr++;
-
+    LIBRECUDA_ERR_PROPAGATE(signalNotify(timelineSignal, timelineCtr, DMA));
     LIBRECUDA_SUCCEED();
 }
 
@@ -772,21 +778,42 @@ libreCudaStatus_t NvCommandQueue::startExecution() {
                     break;
                 }
             }
-            LIBRECUDA_ERR_PROPAGATE(signalNotify(timelineSignal, backlog_entry.timelineCtr, backlog_entry.queueType));
             LIBRECUDA_ERR_PROPAGATE(submitToFifo(backlog_entry.queueType));
             LIBRECUDA_ERR_PROPAGATE(signalWaitCpu(timelineSignal, backlog_entry.timelineCtr));
         }
         commandBufBacklog.clear();
     } else {
         if (!computeCommandBuffer.empty()) {
             LIBRECUDA_ERR_PROPAGATE(startExecution(COMPUTE));
-            LIBRECUDA_ERR_PROPAGATE(awaitExecution(COMPUTE));
         }
         if (!dmaCommandBuffer.empty()) {
             LIBRECUDA_ERR_PROPAGATE(startExecution(DMA));
-            LIBRECUDA_ERR_PROPAGATE(awaitExecution(DMA));
         }
     }
-    kernArgsWriteIdx = 0; // reset kern args write idx
+    LIBRECUDA_SUCCEED();
+}
+
+libreCudaStatus_t NvCommandQueue::signalWaitGpu(NvSignal *pSignal, NvU32 signalTarget) {
+    if (pSignal->value == signalTarget) {
+        // no need to wait, if cpu can confirm.
+
+        return LIBRECUDA_SUCCESS;
+    }
+    LIBRECUDA_ERR_PROPAGATE(enqueue(
+            makeNvMethod(0, NVC56F_SEM_ADDR_LO, 5),
+            {
+                    // little endian
+                    U64_LO_32_BITS(pSignal),
+                    U64_HI_32_BITS(pSignal),
+
+                    // little endian
+                    signalTarget,
+                    0,
+
+                    (NVC56F_SEM_EXECUTE_OPERATION_ACQUIRE) |
+                    (NVC56F_SEM_EXECUTE_PAYLOAD_SIZE_64BIT << 24)
+            },
+            COMPUTE
+    ));
     LIBRECUDA_SUCCEED();
 }

driverapi/src/librecuda.cpp

@@ -1336,16 +1336,21 @@ libreCudaStatus_t libreCuLaunchKernel(LibreCUFunction function,
                                       uint32_t sharedMemBytes,
                                       LibreCUstream stream,
                                       void **kernelParams, size_t numParams,
-                                      void **extra) {
+                                      void **extra,
+                                      bool async) {
     LIBRECUDA_VALIDATE(function != nullptr, LIBRECUDA_ERROR_INVALID_VALUE);
     LIBRECUDA_VALIDATE(stream != nullptr, LIBRECUDA_ERROR_INVALID_VALUE);
-    LIBRECUDA_ERR_PROPAGATE(stream->command_queue->launchFunction(function,
-                                                                  gridDimX, gridDimY, gridDimZ,
-                                                                  blockDimX, blockDimY, blockDimZ,
-                                                                  sharedMemBytes,
-                                                                  kernelParams,
-                                                                  numParams
-    ));
+    LIBRECUDA_ERR_PROPAGATE(
+            stream->command_queue->launchFunction(
+                    function,
+                    gridDimX, gridDimY, gridDimZ,
+                    blockDimX, blockDimY, blockDimZ,
+                    sharedMemBytes,
+                    kernelParams,
+                    numParams,
+                    async
+            )
+    );
     LIBRECUDA_SUCCEED();
 }
 
@@ -1385,8 +1390,7 @@ libreCudaStatus_t libreCuStreamCommence(LibreCUstream stream) {
 
 libreCudaStatus_t libreCuStreamAwait(LibreCUstream stream) {
     LIBRECUDA_VALIDATE(stream != nullptr, LIBRECUDA_ERROR_INVALID_VALUE);
-    LIBRECUDA_ERR_PROPAGATE(stream->command_queue->awaitExecution(COMPUTE));
-    LIBRECUDA_ERR_PROPAGATE(stream->command_queue->awaitExecution(DMA));
+    LIBRECUDA_ERR_PROPAGATE(stream->command_queue->awaitExecution());
     LIBRECUDA_SUCCEED();
 }
 

tests/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_subdirectory(write_float)
 add_subdirectory(memcopy)
 add_subdirectory(dynamic_shared_mem)
-add_subdirectory(compute_dma_iterleave)
+add_subdirectory(compute_chronological_consistency)

tests/compute_dma_iterleave/CMakeLists.txt renamed to tests/compute_chronological_consistency/CMakeLists.txt

@@ -1,9 +1,9 @@
 add_executable(
-        test_compute_dma_interleave
+        test_chronological_consistency
         main.cpp
 )
 target_link_libraries(
-        test_compute_dma_interleave
+        test_chronological_consistency
         PRIVATE
         driverapi
 )