Fix yolov2 support and other bugs found in deep review

.github/workflows/RunTests.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        julia-version: ['lts', '1', 'pre']
+        julia-version: ['lts', '1', 'pre', '1.13-nightly', 'nightly']
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     steps:

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## Unreleased
+
+### Bugfixes
+- Fix the reorg (passthrough) layer and the batchnorm read order for pre-0.2 darknet weight
+  headers, and apply softmax (not sigmoid) to region-layer class scores when the cfg enables
+  `softmax=1` (as the bundled v2 cfgs do; with `softmax=0` class scores are now left linear,
+  matching darknet). Together these fix `v2_COCO` (previously failed to load) and
+  `v2_tiny_COCO` (previously deviated from darknet); both now match AlexeyAB darknet output
+  and are enabled in the test suite.
+- Fix `overridecfg!`/`cfgchanges` targeting any layer other than `:net` (e.g.
+  `(:yolo, 1, :classes, n)`), which previously errored or edited the wrong block.
+- Fix soft-NMS (`nms_kind=soft`): decayed scores are now written back into the results,
+  duplicate keeps are avoided, and boxes whose decayed score falls below `detect_thresh`
+  are pruned from the returned detections.
+- Fix the class-score maximum (CPU and CUDA) scanning one row past the last class into a
+  zero-filled scratch attribute; with non-positive class scores (e.g. region `softmax=0`)
+  the reported class index could point one past the last real class.
+- Fix `prepare_image!` returning a bare 1-channel array (no padding tuple) for matching-size
+  2D `Float32` inputs.
+- CUDA: fix the last detection being dropped in `keepdetections`, an invalid class index
+  when all class scores are non-positive, and a `>`/`>=` threshold boundary mismatch vs CPU.
+
 ## v0.2
 
 ### Breaking changes

Project.toml

@@ -1,7 +1,10 @@
 name = "ObjectDetector"
 uuid = "3dfc1049-5314-49cf-8447-288dfd02f9fb"
-authors = ["Robert Luciani"]
 version = "1.1.2"
+authors = ["Robert Luciani"]
+
+[workspace]
+projects = ["test"]
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
@@ -10,6 +13,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Cairo = "159f3aea-2a34-519c-b102-8c37f9878175"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
 ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534"
 ImageDraw = "4381153b-2b60-58ae-a1ba-fd683676385f"
 ImageFiltering = "6a3955dd-da59-5b1f-98d4-e7296123deb5"
@@ -24,6 +28,9 @@ UnsafeArrays = "c4a57d5a-5b31-53a6-b365-19f8c011fbd6"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
 
+[sources]
+Functors = {rev = "master", url = "https://github.com/FluxML/Functors.jl"}
+
 [extensions]
 CUDAExt = ["CUDA", "cuDNN"]
 
@@ -34,33 +41,16 @@ BenchmarkTools = "0.4, 0.5, 0.6, 0.7, 1.0"
 CUDA = "4, 5, 6"
 Cairo = "1.1.1"
 Colors = "0.13.0"
-Darknet = "0.5.0"
-FileIO = "1"
 Flux = "0.12, 0.13, 0.14.1, 0.15, 0.16"
+Functors = "0.5"
 ImageCore = "0.8, 0.9, 0.10"
 ImageDraw = "0.2"
 ImageFiltering = "0.6, 0.7"
-ImageIO = "0.6"
 ImageTransformations = "0.8, 0.9, 0.10"
 LazyArtifacts = "1.3"
 PrecompileTools = "1"
 PrettyTables = "2.0, 3"
-ReferenceTests = "0.10"
 TimerOutputs = "0.5.28, 1"
 UnsafeArrays = "1.0.6"
 cuDNN = "1, 2, 3, 4, 5, 6"
 julia = "1.10"
-
-[extras]
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-Darknet = "e2912957-7d06-5673-a7d6-96d153624877"
-FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
-ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19"
-OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-ReferenceTests = "324d217c-45ce-50fc-942e-d289b448e8cf"
-Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
-
-[targets]
-test = ["CUDA", "cuDNN", "Darknet", "FileIO", "ImageIO", "OrderedCollections", "ReferenceTests", "Test", "Suppressor"]

README.md

@@ -6,7 +6,7 @@ Supported YOLO models are: `v2`, `v2-tiny`, `v3`, `v3-spp`, `v3-tiny`, `v4`, `v4
 
 Other less standard models may work also.
 
-Note that v3+ models have result parity with [AlexeyAB/darknet](https://github.com/AlexeyAB/darknet), and are directly tested against [Darknet.jl](https://github.com/IanButterworth/Darknet.jl) (see tests)
+Note that all supported models have result parity with [AlexeyAB/darknet](https://github.com/AlexeyAB/darknet), and are directly tested against [Darknet.jl](https://github.com/IanButterworth/Darknet.jl) (see tests)
 
 Training using ObjectDetector is currently unproven/untested.
 
@@ -18,7 +18,7 @@ Requires julia v1.10+. From the Julia REPL, type `]` to enter the Pkg REPL mode
 pkg> add ObjectDetector
 ```
 
-As of ObjectDetector v0.3, if you want to use CUDA accelleration you will also need to
+As of ObjectDetector v0.3, if you want to use CUDA acceleration you will also need to
 add `CUDA` and `cuDNN` to your project and load both packages.
 
 ## Usage
@@ -29,7 +29,7 @@ add `CUDA` and `cuDNN` to your project and load both packages.
 ```julia
 using ObjectDetector, FileIO, ImageIO
 
-yolomod = YOLO.v3_608_COCO(batch=1, silent=true) # Load the YOLOv3-tiny model pretrained on COCO, with a batch size of 1
+yolomod = YOLO.v3_608_COCO(batch=1, silent=true) # Load the 608x608 YOLOv3 model pretrained on COCO, with a batch size of 1
 
 batch = emptybatch(yolomod) # Create a batch object. Automatically uses the GPU if available
 
@@ -62,10 +62,10 @@ dimension is an integer multiple of the filter size of the first conv layer (typ
 
 On CPU an `AllocArrays` & `Adapt` - based allocator is used to reduce allocations.
 
-To opt out of the allocator use `disable_bumper=true`.
+To opt out of the allocator use `disallow_bumper=true`.
 i.e.
 ```julia
-yolomod = YOLO.v3_608_COCO(batch=1, disable_bumper=true)
+yolomod = YOLO.v3_608_COCO(batch=1, disallow_bumper=true)
 ```
 
 ### Visualizing the result
@@ -80,7 +80,7 @@ save("result.png", imgBoxes)
 The darknet YOLO models from https://pjreddie.com/darknet/yolo/ that are pretrained on the COCO dataset are available:
 
 ```julia
-YOLO.v2_COCO() #Currently broken (weights seem bad, model may work with custom weights)
+YOLO.v2_COCO()
 YOLO.v2_tiny_COCO()
 
 YOLO.v3_COCO()

ext/CUDAExt.jl

@@ -23,23 +23,28 @@ function kern_clipdetect(input::CuDeviceArray, conf::Float32)
     idx = (blockIdx().x-1) * blockDim().x + threadIdx().x
     cols = gridDim().x
     if idx <= cols
-        @inbounds input[end-2, idx] = ifelse(input[end-2, idx] > conf, input[end-2, idx], Float32(0.0))
+        # keep values >= conf, matching the CPU clipdetect! boundary behavior
+        @inbounds input[end-2, idx] = ifelse(input[end-2, idx] >= conf, input[end-2, idx], Float32(0.0))
     end
     return
 end
 
 
 function findmax!(input::CuArray)
     rows, cols = size(input)
-    idst, idend = 6, rows - 3
+    # class scores live in rows 6:end-4; rows end-3:end are the appended
+    # scratch attributes and must not participate in the max
+    idst, idend = 6, rows - 4
     @cuda blocks=cols threads=rows kern_findmax!(input, idst, idend)
 end
 function kern_findmax!(input::CuDeviceMatrix{T}, idst::Integer, idend::Integer) where {T}
     if threadIdx().x == idend
         j = blockIdx().x
-        val = zero(T)
-        idx = zero(T)
-        for i in idst:idend
+        # initialize with the first candidate so the first maximum wins,
+        # matching CPU findmax semantics even when all scores are <= 0
+        val = input[idst, j]
+        idx = idst
+        for i in (idst+1):idend
             if input[i, j] > val
                 val = input[i, j]
                 idx = i
@@ -80,7 +85,7 @@ end
 function kern_genbools(input::CuDeviceArray, output::CuDeviceArray)
     col = (blockIdx().x-1) * blockDim().x + threadIdx().x
     cols = gridDim().x
-    if col < cols && input[end-2, col] > Float32(0)
+    if col <= cols && input[end-2, col] > Float32(0)
         @inbounds output[col] = Int32(1)
     end
     return

src/prepareimage.jl

@@ -124,7 +124,7 @@ function prepare_image!(dest_arr::AbstractArray{Float32}, img::AbstractArray{Flo
     elseif ndims(img) == 3 && size(img)[[2,1,3]] == size(dest_arr)
         return (maybe_gpu(PermutedDimsArray(img, [2,1,3])), [0,0,0,0])
     elseif ndims(img) == 2 && size(img)[[2,1]] == size(dest_arr)[1:2]
-        return (maybe_gpu(reshape(PermutedDimsArray(img, [2,1]), size(img,2), size(img, 1), 1)))
+        return (maybe_gpu(repeat(reshape(PermutedDimsArray(img, [2,1]), size(img,2), size(img,1), 1), outer=[1,1,size(dest_arr,3)])), [0,0,0,0])
     elseif ndims(img) == 2
         return prepare_image!(dest_arr, colorview(Gray, img), kern; use_gpu)
     elseif size(img, 1) == 1

src/yolo/nms.jl

@@ -66,10 +66,11 @@ function bboxiou!(out::AbstractArray{T}, box1, box2; distance::Bool=false, beta
 end
 
 """
-    nms(dets, iou_thresh; kind=:default, beta=0.6f0)
+    nms!(dets, iou_thresh; kind=:default, beta=0.6f0)
 
 Performs Non-Maximum Suppression (NMS) on a set of detection boxes `dets`, returning the indices
-of boxes to keep. This function supports multiple NMS strategies:
+of boxes to keep. For `kind = :soft`, `dets` is mutated: decayed scores are written back into
+row `end-2`. This function supports multiple NMS strategies:
 
 Arguments:
 - `dets`: A matrix of shape (≥5, N), where each column represents a detection.
@@ -82,15 +83,18 @@ Keyword Arguments:
     - `:default` (default): traditional hard-threshold NMS
     - `:greedynms` score-decay using IoU penalty (`score *= 1 - IoU`) with fixed beta of 0.6
     - `:diounms`: score-decay using IoU penalty (`score *= 1 - IoU`)
-    - `:soft`: Soft-NMS using exponential decay (`score *= exp(-IoU^2 / beta)`)
+    - `:soft`: Soft-NMS using exponential decay (`score *= exp(-IoU^2 / beta)`).
+      Keeps all boxes; decayed scores are written back into row `end-2` of
+      `dets` (i.e. `dets` is mutated), and pruning is left to the caller's
+      score threshold.
 - `beta` (`Float32`): smoothing factor for soft-NMS (default `0.6`)
 
 Returns:
 - `keep`: a vector of column indices in `dets` to retain
 
 See https://github.com/AlexeyAB/darknet/blob/9d40b619756be9521bc2ccd81808f502daaa3e9a/src/box.c#L195
 """
-function nms(dets::AbstractArray{T}, iou_thresh; kind::Symbol = :default, beta::T = T(0.6)) where T
+function nms!(dets::AbstractArray{T}, iou_thresh; kind::Symbol = :default, beta::T = T(0.6)) where T
     N = size(dets, 2)
     idxs = similar(dets, Int, N)
     @inbounds for j in 1:N
@@ -128,21 +132,29 @@ function nms(dets::AbstractArray{T}, iou_thresh; kind::Symbol = :default, beta::
                     idxs[write_idx] = idxs[j+1]
                 end
             end
-        elseif kind === :soft # untested
+        elseif kind === :soft
+            # Soft-NMS (Bodla et al. 2017), gaussian variant: no box is removed;
+            # overlapping boxes have their scores decayed (written back into
+            # `dets`) and final pruning is left to the caller's score threshold.
             @inbounds for j in 1:b2_len
+                col = idxs[j+1]
                 decay = exp(-(ious[j]^2) / beta)
-                scores[idxs[j+1]] *= decay
+                scores[col] *= decay
+                dets[end-2, col] = scores[col]
             end
-            @inbounds for j in 2:idx_len
-                key = idxs[j]
-                k = j - 1
-                while k >= 1 && scores[idxs[k]] < scores[key]
-                    idxs[k + 1] = idxs[k]
-                    k -= 1
+            # compact survivors down one slot and swap the top decayed score to
+            # the front; only the per-round argmax matters for the keep order
+            best = 1
+            @inbounds for j in 1:b2_len
+                idxs[j] = idxs[j+1]
+                if scores[idxs[j]] > scores[idxs[best]]
+                    best = j
                 end
-                idxs[k + 1] = key
             end
-            write_idx = idx_len - 1
+            if best != 1
+                @inbounds idxs[1], idxs[best] = idxs[best], idxs[1]
+            end
+            write_idx = b2_len
         else
             error("Unknown NMS kind: $kind")
         end
@@ -152,12 +164,17 @@ function nms(dets::AbstractArray{T}, iou_thresh; kind::Symbol = :default, beta::
 end
 
 """
-    perform_detection_nms(batchout, overlap_thresh, batchsize)
+    perform_detection_nms(batchout, overlap_thresh, batchsize; kind, beta, detect_thresh)
 
 For each batch `b` in `1:batchsize`, extract the detections from `batchout`,
 group them by class, sort each group by the end-2 column (class confidence score) descending, and
 run NMS to remove duplicates using bboxiou and overlap_thresh.
 
+`detect_thresh` re-applies the caller's score threshold to the kept boxes.
+This only matters for `kind = :soft`, where scores are decayed during NMS and
+boxes that fall below the original detection threshold must be pruned (the
+other kinds only ever return boxes that already passed the threshold).
+
 Returns a Vector of detection matrices, each of size (num_fields, kept_boxes).
 
 The input `batchout` is a 2D array of shape (num_fields, N), where `N` is the
@@ -171,7 +188,7 @@ batchout rows:
 - end-1: the class index
 - The last row is the batch index
 """
-function perform_detection_nms(batchout, overlap_thresh, batchsize::Int; kind::Symbol=:default, beta::Float32=0.6f0)
+function perform_detection_nms(batchout, overlap_thresh, batchsize::Int; kind::Symbol=:default, beta::Float32=0.6f0, detect_thresh::Float32=0f0)
     output = similar(batchout)
     i = 1  # index for writing into `output`
 
@@ -204,9 +221,10 @@ function perform_detection_nms(batchout, overlap_thresh, batchsize::Int; kind::S
             # nms takes views of sorted_dets and copying here results in lower allocs and faster nms
             sorted_dets = dets[:, sorted_idx]
 
-            keep = nms(sorted_dets, overlap_thresh; kind, beta)
+            keep = nms!(sorted_dets, overlap_thresh; kind, beta)
 
             @inbounds for k in keep
+                sorted_dets[end-2, k] < detect_thresh && continue # soft-NMS may have decayed the score below threshold
                 output[:, i] = sorted_dets[:, k]
                 i += 1
             end