From c3f5f11d11c3e793d1059d0dbe4f6b1e24d11ff3 Mon Sep 17 00:00:00 2001 From: Benjamin Ruesink Date: Tue, 8 Oct 2024 09:15:04 +0200 Subject: [PATCH] Update README, update language standard to C++20, update cmake to include uWebSockets library --- CMakeLists.txt | 10 +- README.MD | 30 ++++ samples/AiMaskStreamApp/AiMaskStreamApp.cpp | 156 ++++++++++---------- samples/AiMaskStreamApp/CMakeLists.txt | 1 + samples/external/CMakeLists.txt | 3 + samples/external/uSockets/CMakeLists.txt | 18 +++ samples/external/uWebSockets/CMakeLists.txt | 14 ++ 7 files changed, 155 insertions(+), 77 deletions(-) create mode 100644 samples/external/uSockets/CMakeLists.txt create mode 100644 samples/external/uWebSockets/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ebe6b8..d5f0d91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,15 +4,19 @@ cmake_minimum_required(VERSION 3.10) set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR} CACHE PATH "Path to where the samples will be installed") option(INSTALL_SDK "Install binaries into the samples folder" OFF) -project(NvVideoEffects_SDK CXX) +project(NvVideoEffects_SDK C CXX) set(CMAKE_CONFIGURATION_TYPES "Release") -# Require C++11 and disable non-standard extensions -set(CMAKE_CXX_STANDARD 11) +# Require C++20 and disable non-standard extensions +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +set(VCPKG_TARGET_TRIPLET "x64-windows") +set(CMAKE_TOOLCHAIN_FILE "C:/vcpkg/scripts/buildsystems/vcpkg.cmake") + + add_definitions(-DNOMINMAX -DWIN32_LEAN_AND_MEAN) # Set common build path for all targets diff --git a/README.MD b/README.MD index 7a8ffce..92d3b53 100644 --- a/README.MD +++ b/README.MD @@ -1,4 +1,34 @@ # README +## AiMaskStreamApp requirements + +### Setup +- Install [vcpkg](https://github.com/microsoft/vcpkg) to e.g. C:/vcpkg +```bash +vcpkg integrate install +``` + +- Install required packages +```bash +vcpkg install libuv:x64-windows uwebsockets-json:x64-windows zlib:x64-windows usockets:x64-windows +``` + +- Configure and generate the Visual Studio solution file +- If using CMake GUI, set the source folder to the root of the repository and the build folder to `build`, and set the toolchain file (-T) to `C:/vcpkg/scripts/buildsystems/vcpkg.cmake` +```bash +mkdir build +``` +- else: +```bash +cd build +cmake -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake .. +``` + +### Build +- Open the generated Visual Studio solution file and build the solution +- Built binaries will be in `build/Release` +- You might need to copy `opencv_world346.dll` from `samples/external/opencv/bin` to the binary directory to run the application + + ## NVIDIA MAXINE VideoEffects SDK: API Source Code and Sample Applications NVIDIA MAXINE Video Effects SDK enables AI-based visual effects that run with standard webcam input and can easily be integrated into video conference and content creation pipelines. The underlying deep learning models are optimized with NVIDIA AI using NVIDIA® TensorRT™ for high-performance inference, making it possible for developers to apply multiple effects in real-time applications. diff --git a/samples/AiMaskStreamApp/AiMaskStreamApp.cpp b/samples/AiMaskStreamApp/AiMaskStreamApp.cpp index 8d0cff4..2104a97 100644 --- a/samples/AiMaskStreamApp/AiMaskStreamApp.cpp +++ b/samples/AiMaskStreamApp/AiMaskStreamApp.cpp @@ -37,16 +37,17 @@ #include "nvVideoEffects.h" #include "opencv2/opencv.hpp" +#define UWS_NO_ZLIB + #include "httplib.h" -#include "mjpeg_streamer.hpp" +#include "App.h" //#include #include #include #pragma comment(lib, "Ws2_32.lib") - -using MJPEGStreamer = nadjieb::MJPEGStreamer; +//#pragma comment(lib, "uSockets.lib") enum StreamType { ST_ORIGINAL, @@ -777,7 +778,6 @@ bail: return (FXApp::Err)vfxErr; } -MJPEGStreamer streamer; std::mutex clients_mutex; struct PerSocketData { }; @@ -1120,40 +1120,9 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { static std::vector params = { cv::IMWRITE_JPEG_QUALITY, 90 }; - /*std::vector buff_bgr; - cv::imencode(".jpg", originalImg, buff_bgr, params); - streamer.publish("/original", std::string(buff_bgr.begin(), buff_bgr.end()));*/ - - //cv::Mat hsv; - //cv::cvtColor(result, hsv, cv::COLOR_BGR2HSV); - - // http://localhost:8080/hsv - /*std::vector buff_hsv; - cv::imencode(".jpg", result, buff_hsv, params); - streamer.publish("/masked", std::string(buff_hsv.begin(), buff_hsv.end()));*/ - - pushToFrameQueue(ST_ORIGINAL, originalImg); pushToFrameQueue(ST_MASKED, result); - /* std::vector buf; - cv::imencode(".jpg", originalImg, buf); - std::string encoded(buf.begin(), buf.end()); - - std::lock_guard lock(clients_mutex);*/ - /*for (auto ws : clients) { - ws->send(encoded, uWS::OpCode::BINARY); - }*/ - - //std::this_thread::sleep_for(std::chrono::milliseconds(33)); // ~30 fps - - //pushToFrameQueue(ST_PROCESSED, outlineResult); - - // Display the results - //cv::imshow("Original", originalImg); - //cv::imshow("Overlay", result); - //cv::imshow("OutlineTest", outlineResult); - int key = cv::waitKey(1); if (key > 0) { appErr = processKey(key); @@ -1242,6 +1211,83 @@ void startHttpServer() { svr.listen("0.0.0.0", 8090); } +void testWS() { + /* ws->getUserData returns one of these */ + struct PerSocketData { + /* Fill with user data */ + std::vector topics; + int nr = 0; + }; + + { + /* Keep in mind that uWS::SSLApp({options}) is the same as uWS::App() when compiled without SSL support. + * You may swap to using uWS:App() if you don't need SSL */ + uWS::SSLApp* app = new uWS::SSLApp({ + /* There are example certificates in uWebSockets.js repo */ + .key_file_name = "misc/key.pem", + .cert_file_name = "misc/cert.pem", + .passphrase = "1234" + }); + + app->ws("/*", { + /* Settings */ + .compression = uWS::DISABLED, + .maxPayloadLength = 16 * 1024 * 1024, + .idleTimeout = 60, + .maxBackpressure = 16 * 1024 * 1024, + .closeOnBackpressureLimit = false, + .resetIdleTimeoutOnSend = true, + .sendPingsAutomatically = false, + /* Handlers */ + .upgrade = nullptr, + .open = [](auto* ws) { + /* Open event here, you may access ws->getUserData() which points to a PerSocketData struct */ + + printf("opened\n"); + PerSocketData* perSocketData = (PerSocketData*)ws->getUserData(); + + for (int i = 0; i < 32; i++) { + std::string topic = std::to_string((uintptr_t)ws) + "-" + std::to_string(i); + perSocketData->topics.push_back(topic); + ws->subscribe(topic); + } + }, + .message = [&app](auto* ws, std::string_view message, uWS::OpCode opCode) { + PerSocketData* perSocketData = (PerSocketData*)ws->getUserData(); + + printf("Message: %s\n", message.data()); + + app->publish(perSocketData->topics[(size_t)(++perSocketData->nr % 32)], message, opCode); + ws->publish(perSocketData->topics[(size_t)(++perSocketData->nr % 32)], message, opCode); + }, + .drain = [](auto*/*ws*/) { + /* Check ws->getBufferedAmount() here */ + //std::cout << "drain" << std::endl; + }, + .ping = [](auto*/*ws*/, std::string_view) { + /* Not implemented yet */ + }, + .pong = [](auto*/*ws*/, std::string_view) { + /* Not implemented yet */ + }, + .close = [](auto*/*ws*/, int /*code*/, std::string_view /*message*/) { + /* You may access ws->getUserData() here */ + } + }).listen(9001, [](auto* listen_s) { + if (listen_s) { + std::cout << "Listening on port " << 9001 << std::endl; + //listen_socket = listen_s; + } + }); + + app->run(); + + delete app; + + uWS::Loop::get()->free(); + } +} + int main(int argc, char **argv) { int nErrs = 0; nErrs = ParseMyArgs(argc, argv); @@ -1250,49 +1296,11 @@ int main(int argc, char **argv) { return nErrs; } - /*std::thread serverThread(startHttpServer); - streamer.start(8001);*/ + std::thread testws(&testWS); TcpServer server(8093); std::thread serverThread2(&TcpServer::start, &server); - //httplib::Server svr; - - //svr.Get("/video", [](const httplib::Request&, httplib::Response& res) { - // cv::VideoCapture cap(0); // Open webcam - // if (!cap.isOpened()) { - // std::cerr << "Error: Cannot open webcam\n"; - // return; - // } - // // Content provider for streaming - // res.set_content_provider( - // "multipart/x-mixed-replace; boundary=frame", - // [cap](size_t offset, httplib::DataSink& sink) mutable { - // cv::Mat frame; - // cap >> frame; - - // if (!frame.empty()) { - // std::vector bytes = matToBytes(frame); - - // // Write multipart content - // std::string header = "--frame\r\nContent-Type: image/jpeg\r\n\r\n"; - // sink.write(header.data(), header.size()); - // sink.write(reinterpret_cast(bytes.data()), bytes.size()); - // sink.write("\r\n", 2); - // } - - // return true; // Continue streaming - // }, - // [](bool success) { - // // Completion handler when stream ends - // } - // ); - // }); - - //svr.listen("0.0.0.0", 8080); - - - FXApp::Err fxErr = FXApp::errNone; FXApp app; diff --git a/samples/AiMaskStreamApp/CMakeLists.txt b/samples/AiMaskStreamApp/CMakeLists.txt index 1b7e47b..748a69c 100644 --- a/samples/AiMaskStreamApp/CMakeLists.txt +++ b/samples/AiMaskStreamApp/CMakeLists.txt @@ -20,6 +20,7 @@ if(MSVC) target_link_libraries(AiMaskStreamApp PUBLIC opencv346 NVVideoEffects + uWebSockets ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib ) diff --git a/samples/external/CMakeLists.txt b/samples/external/CMakeLists.txt index 0be0af0..eb0bb8a 100644 --- a/samples/external/CMakeLists.txt +++ b/samples/external/CMakeLists.txt @@ -1,3 +1,6 @@ +add_subdirectory(uSockets) +add_subdirectory(uWebSockets) + ####################### # Interface to OpenCV # ####################### diff --git a/samples/external/uSockets/CMakeLists.txt b/samples/external/uSockets/CMakeLists.txt new file mode 100644 index 0000000..7fe9f37 --- /dev/null +++ b/samples/external/uSockets/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.24) +# set(CMAKE_C_STANDARD 11) +include(FetchContent) +FetchContent_Declare( + uSockets_content + GIT_REPOSITORY https://github.com/uNetworking/uSockets + GIT_TAG v0.8.5 + GIT_SHALLOW ON + GIT_SUBMODULES "" +) +FetchContent_MakeAvailable(uSockets_content) +file(GLOB_RECURSE SOURCES ${usockets_content_SOURCE_DIR}/src/*.c) +add_library(uSockets ${SOURCES}) +target_include_directories(uSockets PUBLIC ${usockets_content_SOURCE_DIR}/src) +target_compile_definitions(uSockets PRIVATE LIBUS_NO_SSL) + +find_package(libuv CONFIG REQUIRED) +target_link_libraries(uSockets PRIVATE $,libuv::uv_a,libuv::uv>) \ No newline at end of file diff --git a/samples/external/uWebSockets/CMakeLists.txt b/samples/external/uWebSockets/CMakeLists.txt new file mode 100644 index 0000000..c8235ed --- /dev/null +++ b/samples/external/uWebSockets/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.24) +include(FetchContent) +FetchContent_Declare( + uWebSockets_content + GIT_REPOSITORY https://github.com/uNetworking/uWebSockets + GIT_TAG v20.37.0 + GIT_SHALLOW ON + GIT_SUBMODULES "" +) +FetchContent_MakeAvailable(uWebSockets_content) +find_package(ZLIB REQUIRED) +add_library(uWebSockets INTERFACE) +target_include_directories(uWebSockets INTERFACE ${uwebsockets_content_SOURCE_DIR}/src/) +target_link_libraries(uWebSockets INTERFACE uSockets ${ZLIB_LIBRARIES}) \ No newline at end of file