WasmEdge FFmpeg Plugin to Process Video and Audio Files

.cargo/config.toml

@@ -3,16 +3,4 @@
 target = "wasm32-wasi"
 
 [target.wasm32-wasi]
-runner = "./scripts/wasmedge-runner.sh"
-
-[env]
-# ffmpeg install path (default is ```assets/ffmpeg-lib```)
-FFMPEG_DIR = { value = "assets/ffmpeg-lib", relative = true }
-# wasi sysroot path (default is ```assets/wasi-sysroot```)
-WASI_SYSROOT = { value = "assets/wasi-sysroot", relative = true }
-# lib clang rt path (default is ```assets/clang-rt```)
-CLANG_RT = { value = "assets/clang-rt", relative = true }
-
-# config wasi-sysroot path (default is ```./asset/wasi-sysroot```)
-#BINDGEN_EXTRA_CLANG_ARGS = "--sysroot=./asset/wasi-sysroot --target=wasm32-wasi -fvisibility=default"
-BINDGEN_EXTRA_CLANG_ARGS = "--sysroot=/opt/wasi-sdk/share/wasi-sysroot --target=wasm32-wasi -fvisibility=default"
+runner = "./scripts/wasmedge-runner.sh"

Cargo.toml

@@ -39,10 +39,9 @@ lazy_static = { version = "^1", optional = true }
 symphonia-core = { version = "^0", optional = true }
 
 [dependencies.ffmpeg-next]
-features = ["static"]
 optional = true
-git = "https://github.com/yanghaku/rust-ffmpeg.git"
-branch = "wasm32-wasi"
+git = "https://github.com/Hrushi20/rust-ffmpeg"
+branch = "master"
 
 
 [features]

README.md

@@ -269,7 +269,7 @@ from https://storage.googleapis.com/mediapipe-tasks/gesture_recognizer/victory.j
 Example output in console:
 
 ```console
-$ cargo run --release --example gesture_recognition -- ./assets/models/gesture_recognition/gesture_recognizer.task ./assets/testdata/img/gesture_recognition_google_samples/victory.jpg
+$ cargo run --release --example gesture_recognition -- ./assets/models/gesture_recognition/gesture_recognizer.task ./assets/testdata/img/victory.jpg
     Finished release [optimized] target(s) in 0.02s
      Running `/mediapipe-rs/./scripts/wasmedge-runner.sh target/wasm32-wasi/release/examples/gesture_recognition.wasm ./assets/models/gesture_recognition/gesture_recognizer.task ./assets/testdata/img/gesture_recognition_google_samples/victory.jpg`
       Category name: "Victory"
@@ -389,21 +389,10 @@ fn inference(
 
 ## Use the FFMPEG feature to process video and audio.
 
-When building the library with ```ffmpeg``` feature using cargo, users must set the following environment variables:
+When building the library with ```ffmpeg``` feature using cargo, users must build WasmEdge by enabling FFmpeg Plugin:
 
-* ```FFMPEG_DIR```: the pre-built FFmpeg library path. You can download it from
-  https://github.com/yanghaku/ffmpeg-wasm32-wasi/releases.
-* ```WASI_SDK``` or (```WASI_SYSROOT``` and ```CLANG_RT```), You can download it from
-  https://github.com/WebAssembly/wasi-sdk/releases
-* ```BINDGEN_EXTRA_CLANG_ARGS```: set **sysroot** and **target** and **function visibility** for libclang.
-  (The sysroot must be **absolute path**).
-
-Example:
-
-```shell
-export FFMPEG_DIR=/path/to/ffmpeg/library
-export WASI_SDK=/opt/wasi-sdk
-export BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/opt/wasi-sdk/share/wasi-sysroot --target=wasm32-wasi -fvisibility=default"
+```console
+$ cmake -DCMAKE_BUILD_TYPE=Release -GNinja -DWASMEDGE_PLUGIN_WASI_NN_BACKEND="TensorflowLite" -DWASMEDGE_PLUGIN_FFMPEG=ON ..
 
 # Then run cargo
 ```

examples/audio_classification.rs

@@ -32,7 +32,7 @@ fn read_audio_using_symphonia(audio_path: String) -> SymphoniaAudioData {
 }
 
 #[cfg(feature = "ffmpeg")]
-fn read_video_using_ffmpeg(audio_path: String) -> FFMpegAudioData {
+fn read_audio_using_ffmpeg(audio_path: String) -> FFMpegAudioData {
     ffmpeg_next::init().unwrap();
     FFMpegAudioData::new(ffmpeg_next::format::input(&audio_path.as_str()).unwrap()).unwrap()
 }
@@ -43,7 +43,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(not(feature = "ffmpeg"))]
     let audio = read_audio_using_symphonia(audio_path);
     #[cfg(feature = "ffmpeg")]
-    let audio = read_video_using_ffmpeg(audio_path);
+    let audio = read_audio_using_ffmpeg(audio_path);
 
     let classification_results = AudioClassifierBuilder::new()
         .max_results(3) // set max result

src/preprocess/vision/ffmpeg.rs

@@ -2,6 +2,7 @@ use super::*;
 use std::cell::RefCell;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use ffmpeg_next::avUtilTypes::AVPixelFormat;
 
 type FFMpegVideoInput =
     common::ffmpeg_input::FFMpegInput<ffmpeg_next::decoder::Video, ffmpeg_next::frame::Video>;
@@ -33,11 +34,12 @@ impl FFMpegVideoData {
         } else {
             source.decoder.time_base().numerator()
         };
+
         let filter_desc_in = format!(
             "buffer=video_size={}x{}:pix_fmt={}:time_base={}/{}:pixel_aspect={}/{}",
             source.decoder.width(),
             source.decoder.height(),
-            ffmpeg_next::ffi::AVPixelFormat::from(source.decoder.format()) as u32,
+            ffmpeg_next::util::format::Pixel::from(source.decoder.format() as AVPixelFormat).mask(),
             time_base_num,
             source.decoder.time_base().denominator(),
             source.decoder.aspect_ratio().numerator(),
@@ -105,16 +107,16 @@ impl<'a> ImageToTensor for FFMpegFrame<'a> {
             }
             let out_format = match to_tensor_info.color_space {
                 ImageColorSpaceType::GRAYSCALE => {
-                    ffmpeg_next::ffi::AVPixelFormat::AV_PIX_FMT_GRAY8 as u32
+                    ffmpeg_next::format::Pixel::GRAY8
                 }
-                _ => ffmpeg_next::ffi::AVPixelFormat::AV_PIX_FMT_RGB24 as u32,
+                _ => ffmpeg_next::format::Pixel::RGB24,
             };
             desc.extend(
                 format!(
                     "[s_in];[s_in]scale={}:{}[f];[f]format=pix_fmts={}[out];[out]buffersink",
                     to_tensor_info.width(),
                     to_tensor_info.height(),
-                    out_format
+                    out_format.mask()
                 )
                 .chars(),
             );
@@ -158,7 +160,7 @@ impl<'a> ImageToTensor for FFMpegFrame<'a> {
                 let img = image::ImageBuffer::<image::Rgb<u8>, &[u8]>::from_raw(
                     to_tensor_info.width(),
                     to_tensor_info.height(),
-                    data,
+                    &data,
                 )
                 .unwrap();
                 image::rgb8_image_buffer_to_tensor(&img, to_tensor_info, output_buffer)?;