feat: use deno to download models

This commit introduces a new step in the CI pipeline that downloads necessary models for testing. A new script `download_models.ts` is added which uses Deno to download the models. The CI workflow is updated to run this script before building and testing. Deno is added as a dependency in `devbox.json` and `devbox.lock`.
ryoppippi · May 11, 2024 · afb6e67 · afb6e67
1 parent 67ea8ef
commit afb6e67
Show file tree

Hide file tree

Showing 8 changed files with 98 additions and 124 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,14 +20,24 @@ jobs:
       - run: devbox run fmt-check
 
   test:
-    runs-on: macos-14
+    strategy:
+      matrix:
+        os: 
+          - macos-14
+          # - ubuntu-latest
+    runs-on: ${{matrix.os}}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Install devbox
         uses: jetify-com/[email protected]
-      - run: devbox run test
+      - name: Download models
+        run: devbox run download-models
+      - name: Build
+        run: devbox run build
+      - name: Test
+        run: devbox run test
 
   # deploy:
   #   permissions:

diff --git a/devbox.json b/devbox.json
@@ -5,16 +5,17 @@
     "[email protected]",
     "[email protected]",
     "pkg-config@latest",
-    "curl@latest",
-    "unzip@latest"
+    "unzip@latest",
+    "[email protected]"
   ],
   "shell": {
     "scripts": {
-      "version": "zig version",
-      "build": "zig build --verbose",
-      "test": "zig build test --verbose",
-      "fmt": "zig fmt ./**/*.zig",
-      "fmt-check": "zig fmt --check ./**/*.zig"
+      "version":         "zig version",
+      "download-models": "deno run -A ./scripts/download_models.ts",
+      "build":           "zig build --verbose",
+      "test":            "zig build test --verbose",
+      "fmt":             "zig fmt ./**/*.zig",
+      "fmt-check":       "zig fmt --check ./**/*.zig"
     }
   }
 }
diff --git a/devbox.lock b/devbox.lock
@@ -1,127 +1,51 @@
 {
   "lockfile_version": "1",
   "packages": {
-    "curl@latest": {
-      "last_modified": "2024-05-06T15:07:38Z",
-      "resolved": "github:NixOS/nixpkgs/860e65d27036476edfb85dd847d982277880b143#curl",
+    "[email protected]": {
+      "last_modified": "2024-05-06T06:27:37Z",
+      "resolved": "github:NixOS/nixpkgs/dd1290b0f857782a60b251f89651c831cd3eef9d#deno",
       "source": "devbox-search",
-      "version": "8.7.1",
+      "version": "1.43.1",
       "systems": {
         "aarch64-darwin": {
           "outputs": [
-            {
-              "name": "bin",
-              "path": "/nix/store/fk1nbg9csmdskv4w466d779n87bvc3n0-curl-8.7.1-bin",
-              "default": true
-            },
-            {
-              "name": "man",
-              "path": "/nix/store/kl3ds7m52n7wgz75qcbfinrz1r7cm1i0-curl-8.7.1-man",
-              "default": true
-            },
-            {
-              "name": "dev",
-              "path": "/nix/store/dzwxhz2kl4asqsqpdzz5yf89j25igwk6-curl-8.7.1-dev"
-            },
-            {
-              "name": "devdoc",
-              "path": "/nix/store/2rhikjyxizidbd4p1hk479vmr9rc8m5h-curl-8.7.1-devdoc"
-            },
             {
               "name": "out",
-              "path": "/nix/store/9x1dp01k13riyvk54ciwjf5za8idwmhi-curl-8.7.1"
+              "path": "/nix/store/kc5aglf91zj78jw4837nazlgr6ys33cp-deno-1.43.1",
+              "default": true
             }
           ],
-          "store_path": "/nix/store/fk1nbg9csmdskv4w466d779n87bvc3n0-curl-8.7.1-bin"
+          "store_path": "/nix/store/kc5aglf91zj78jw4837nazlgr6ys33cp-deno-1.43.1"
         },
         "aarch64-linux": {
           "outputs": [
-            {
-              "name": "bin",
-              "path": "/nix/store/i5l8l75nywb6hkd4g2i332xwm4djjvbh-curl-8.7.1-bin",
-              "default": true
-            },
-            {
-              "name": "man",
-              "path": "/nix/store/1kdrycxjg247g313pql8lg8dk0bl89lw-curl-8.7.1-man",
-              "default": true
-            },
-            {
-              "name": "debug",
-              "path": "/nix/store/cdf1fylklg2vbafari42yqp44bmmk5cn-curl-8.7.1-debug"
-            },
-            {
-              "name": "dev",
-              "path": "/nix/store/q91fvqihpnirbm3pz2sq11kjdpj71zl5-curl-8.7.1-dev"
-            },
-            {
-              "name": "devdoc",
-              "path": "/nix/store/9yr2nxivsgd8k8cqabpikbiin1hpqq7r-curl-8.7.1-devdoc"
-            },
             {
               "name": "out",
-              "path": "/nix/store/x2ngx6la8fix8iz7zhyi7gkgsszy9kkx-curl-8.7.1"
+              "path": "/nix/store/c0bg2vwn4mlx0dvaysl67rn5yf73hi40-deno-1.43.1",
+              "default": true
             }
           ],
-          "store_path": "/nix/store/i5l8l75nywb6hkd4g2i332xwm4djjvbh-curl-8.7.1-bin"
+          "store_path": "/nix/store/c0bg2vwn4mlx0dvaysl67rn5yf73hi40-deno-1.43.1"
         },
         "x86_64-darwin": {
           "outputs": [
-            {
-              "name": "bin",
-              "path": "/nix/store/jlhz559xd973by93qj83izdgm1bkwyiw-curl-8.7.1-bin",
-              "default": true
-            },
-            {
-              "name": "man",
-              "path": "/nix/store/xvrgskrlc8y23kb83adykc79n3m3221f-curl-8.7.1-man",
-              "default": true
-            },
-            {
-              "name": "dev",
-              "path": "/nix/store/x2vxybfyjv3ga3wdifn5m9xjmn7wjjk9-curl-8.7.1-dev"
-            },
-            {
-              "name": "devdoc",
-              "path": "/nix/store/cayjhlcrvyq27rzk0p6fj3f8ik5757h5-curl-8.7.1-devdoc"
-            },
             {
               "name": "out",
-              "path": "/nix/store/zjjazsh938488vqj83b73wi0dg8xw9p3-curl-8.7.1"
+              "path": "/nix/store/jzhp7f068y0ph8vyhrwvy86y4lfh969s-deno-1.43.1",
+              "default": true
             }
           ],
-          "store_path": "/nix/store/jlhz559xd973by93qj83izdgm1bkwyiw-curl-8.7.1-bin"
+          "store_path": "/nix/store/jzhp7f068y0ph8vyhrwvy86y4lfh969s-deno-1.43.1"
         },
         "x86_64-linux": {
           "outputs": [
-            {
-              "name": "bin",
-              "path": "/nix/store/rzqab54yp9a21sii4q59v9mlvn15nxxd-curl-8.7.1-bin",
-              "default": true
-            },
-            {
-              "name": "man",
-              "path": "/nix/store/gx188rl2rmj99k6s65ylvcpavxrwqla9-curl-8.7.1-man",
-              "default": true
-            },
-            {
-              "name": "debug",
-              "path": "/nix/store/51lgfz0ghqn4pp7wg0wpr9lk62sxpsgx-curl-8.7.1-debug"
-            },
-            {
-              "name": "dev",
-              "path": "/nix/store/n6q80yww217mly3ac6k2vdrd2zvrjk2c-curl-8.7.1-dev"
-            },
-            {
-              "name": "devdoc",
-              "path": "/nix/store/vm8qcnr0xw33aqxl53shcgrk4cqdq1hb-curl-8.7.1-devdoc"
-            },
             {
               "name": "out",
-              "path": "/nix/store/qv1qp23si0prgzcx6b8522f0g7fipaph-curl-8.7.1"
+              "path": "/nix/store/vvq3r8xjg85892anwj74n154ail9bfs6-deno-1.43.1",
+              "default": true
             }
           ],
-          "store_path": "/nix/store/rzqab54yp9a21sii4q59v9mlvn15nxxd-curl-8.7.1-bin"
+          "store_path": "/nix/store/vvq3r8xjg85892anwj74n154ail9bfs6-deno-1.43.1"
         }
       }
     },

diff --git a/examples/dnndetection/main.zig b/examples/dnndetection/main.zig
@@ -21,7 +21,7 @@
 
 const std = @import("std");
 const cv = @import("zigcv");
-const downloadFile = cv.utils.downloadFile;
+// const downloadFile = cv.utils.downloadFile;
 const Mat = cv.Mat;
 const Size = cv.Size;
 const c_api = cv.c_api;
@@ -59,8 +59,8 @@ pub fn main() anyerror!void {
     defer img.deinit();
 
     // open DNN object tracking model
-    try downloadFile(model_url, cache_dir, allocator);
-    try downloadFile(config_url, cache_dir, allocator);
+    // try downloadFile(model_url, cache_dir, allocator);
+    // try downloadFile(config_url, cache_dir, allocator);
     var net = cv.Net.readNet(model_path, config_path) catch |err| {
         std.debug.print("Error: {}\n", .{err});
         std.os.exit(1);

diff --git a/scripts/download_models.ts b/scripts/download_models.ts
@@ -0,0 +1,42 @@
+#!/usr/bin/env -S deno run -A
+import { $ } from "jsr:@david/[email protected]";
+
+$.setPrintCommand(true);
+
+const MODEL_URLS = [
+  "http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel",
+  "https://raw.githubusercontent.com/opencv/opencv_extra/20d18acad1bcb312045ea64a239ebe68c8728b88/testdata/dnn/bvlc_googlenet.prototxt",
+  "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip",
+  "https://github.com/onnx/models/raw/4eff8f9b9189672de28d087684e7085ad977747c/vision/classification/inception_and_googlenet/googlenet/model/googlenet-9.onnx",
+  "https://github.com/opencv/opencv_3rdparty/raw/dnn_samples_face_detector_20170830/res10_300x300_ssd_iter_140000.caffemodel",
+  "https://raw.githubusercontent.com/opencv/opencv/master/samples/dnn/face_detector/deploy.prototxt",
+] as const;
+
+/** get the root of the git repository */
+const git_root = await $`git rev-parse --show-toplevel`.text();
+
+/** save the models in the zig-cache/tmp/ directory */
+const saveDir = $.path(git_root).join("./zig-cache/tmp/");
+
+/** create the directory if it does not exist */
+await $`mkdir -p ${saveDir.toString()}`;
+
+$.cd(saveDir);
+
+/** list the files in the directory */
+await $`ls -lh ${saveDir}`;
+
+/** download the models */
+const downloadedPath = await Promise.all(
+  MODEL_URLS.map((url) => $.request(url).showProgress().pipeToPath()),
+);
+
+/** decompress the zip files */
+await Promise.all(
+  downloadedPath
+    .filter((path) => path.extname() === ".zip")
+    .map((path) => $`unzip -o ${path}`),
+);
+
+/** list the files in the directory */
+await $`ls -lh ${saveDir}`;
diff --git a/src/dnn/test.zig b/src/dnn/test.zig
@@ -26,9 +26,9 @@ const tensorflow_model_file = cache_dir ++ tensorflow_model_filename;
 const onnx_model_url = "https://github.com/onnx/models/raw/4eff8f9b9189672de28d087684e7085ad977747c/vision/classification/inception_and_googlenet/googlenet/model/googlenet-9.onnx";
 const onnx_model_file = cache_dir ++ "googlenet-9.onnx";
 
-pub fn downloadModel(url: []const u8, allocator_: std.mem.Allocator) !void {
-    try utils.downloadFile(url, cache_dir, allocator_);
-}
+// pub fn downloadModel(url: []const u8, allocator_: std.mem.Allocator) !void {
+//     try utils.downloadFile(url, cache_dir, allocator_);
+// }
 
 fn checkNet(net: *Net, allocator: std.mem.Allocator) !void {
     net.setPreferableBackend(.default);
@@ -95,8 +95,8 @@ fn checkNet(net: *Net, allocator: std.mem.Allocator) !void {
 }
 
 test "dnn read net from disk" {
-    try downloadModel(caffe_model_url, test_allocator);
-    try downloadModel(caffe_prototext_url, test_allocator);
+    // try downloadModel(caffe_model_url, test_allocator);
+    // try downloadModel(caffe_prototext_url, test_allocator);
     var net = try Net.readNet(
         caffe_model_file,
         caffe_prototext_file,
@@ -108,8 +108,8 @@ test "dnn read net from disk" {
 }
 
 test "dnn read net from memory" {
-    try downloadModel(caffe_model_url, test_allocator);
-    try downloadModel(caffe_prototext_url, test_allocator);
+    // try downloadModel(caffe_model_url, test_allocator);
+    // try downloadModel(caffe_prototext_url, test_allocator);
 
     var model_file = try std.fs.cwd().openFile(caffe_model_file, .{});
     const m_stat = try std.fs.cwd().statFile(caffe_model_file);
@@ -169,8 +169,8 @@ fn checkCaffeNet(net: *Net) !void {
 }
 
 test "dnn read caffe disk" {
-    try downloadModel(caffe_model_url, test_allocator);
-    try downloadModel(caffe_prototext_url, test_allocator);
+    // try downloadModel(caffe_model_url, test_allocator);
+    // try downloadModel(caffe_prototext_url, test_allocator);
     var net = try Net.readNetFromCaffe(
         caffe_prototext_file,
         caffe_model_file,
@@ -181,8 +181,8 @@ test "dnn read caffe disk" {
 }
 
 test "dnn read caffe memory" {
-    try downloadModel(caffe_model_url, test_allocator);
-    try downloadModel(caffe_prototext_url, test_allocator);
+    // try downloadModel(caffe_model_url, test_allocator);
+    // try downloadModel(caffe_prototext_url, test_allocator);
 
     var model_file = try std.fs.cwd().openFile(caffe_model_file, .{});
     const m_stat = try std.fs.cwd().statFile(caffe_model_file);
@@ -240,7 +240,7 @@ fn checkTensorflow(net: *Net) !void {
 }
 
 fn downloadTFModel() !void {
-    try downloadModel(tensorflow_model_zip_url, test_allocator);
+    // try downloadModel(tensorflow_model_zip_url, test_allocator);
     var arena = std.heap.ArenaAllocator.init(test_allocator);
     defer arena.deinit();
     const arena_allocator = arena.allocator();
@@ -322,15 +322,15 @@ fn checkONNX(net: *Net) !void {
 }
 
 test "dnn read onnx disk" {
-    try downloadModel(onnx_model_url, test_allocator);
+    // try downloadModel(onnx_model_url, test_allocator);
     var net = try Net.readNetFromONNX(onnx_model_file);
     defer net.deinit();
 
     try checkONNX(&net);
 }
 
 test "dnn read onnx memory" {
-    try downloadModel(onnx_model_url, test_allocator);
+    // try downloadModel(onnx_model_url, test_allocator);
     var model_file = try std.fs.cwd().openFile(onnx_model_file, .{});
     const m_stat = try std.fs.cwd().statFile(onnx_model_file);
     defer model_file.close();

diff --git a/src/imgcodecs.zig b/src/imgcodecs.zig
@@ -269,8 +269,7 @@ test "imgcodecs imread" {
 }
 
 test "imgcodecs imread not found error" {
-    var e = imRead(face_detect_img_path ++ "aaa", .color);
-    std.debug.print("error: '{s}' is expected\n", .{"File not found: ./libs/gocv/images/face-detect.jpgaaa"});
+    var e = imRead("not-exist-path/" ++ face_detect_img_path, .color);
     try testing.expectError(error.FileNotFound, e);
 }
 

diff --git a/src/videoio.zig b/src/videoio.zig
@@ -446,17 +446,15 @@ test "videoio VideoCapture captureFileWithAPI" {
 test "videoio VideoCapture captureFile invalid file" {
     var vc = try VideoCapture.init();
     defer vc.deinit();
-    var e = vc.captureFile(video_path ++ "4");
+    var e = vc.captureFile("not-exist-path/" ++ video_path);
     try testing.expectError(error.VideoCaptureOpenFileError, e);
-    std.debug.print("error '{s}' is expected\n", .{"OpenCV: Couldn't read video stream from file libs/gocv/images/small.mp44"});
 }
 
 test "videoio VideoCapture captureFileWithAPI invalid file" {
     var vc = try VideoCapture.init();
     defer vc.deinit();
-    var e = vc.captureFileWithAPI(video_path ++ "4", .any);
+    var e = vc.captureFileWithAPI("not-exist-path/" ++ video_path, .any);
     try testing.expectError(error.VideoCaptureOpenFileError, e);
-    std.debug.print("error '{s}' is expected\n", .{"OpenCV: Couldn't read video stream from file libs/gocv/images/small.mp44"});
 }
 
 test "videoio VideoCapture openDevice unknown error" {