b-strike
/
nvidia-vfx-prototyping
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add httplib, prototype masking and web streaming

Browse Source
master
Benjamin Ruesink 2 months ago
parent 17054e59c6
commit 2690a8b0a7
2 changed files with 10501 additions and 5 deletions
Show Stats Download Patch File Download Diff File

221
samples/AiMaskStreamApp/AiMaskStreamApp.cpp
Unescape View File

@ -29,11 +29,42 @@
#include <string>
#include <iostream>
#include <vector>
#include <queue>
#include <mutex>
#include <thread>
#include "nvCVOpenCV.h"
#include "nvVideoEffects.h"
#include "opencv2/opencv.hpp"
#include "../../../utils/httplib.h"
enum FrameType {
TYPE_ORIGINAL,
TYPE_OUTLINE,
TYPE_PROCESSED,
TYPE_MAX
};
struct FrameQueue {
std::queue<cv::Mat> frameQueue;
std::mutex frameMutex;
};
std::vector<FrameQueue> frameQueues(TYPE_MAX);
// Convert cv::Mat to JPEG buffer
std::vector<uchar> matToBytes(const cv::Mat& img) {
std::vector<uchar> buf;
cv::imencode(".jpg", img, buf);
return buf;
}
void pushToFrameQueue(int type, const cv::Mat& frame) {
std::lock_guard<std::mutex> lock(frameQueues[type].frameMutex);
frameQueues[type].frameQueue.push(frame.clone());
}
#ifdef _MSC_VER
#define strcasecmp _stricmp
#include <Windows.h>
@ -77,6 +108,7 @@ std::string FLAG_modelDir;
std::string FLAG_outDir;
std::string FLAG_outFile;
std::string FLAG_bgFile;
int FLAG_camIndex = 0;
static bool GetFlagArgVal(const char *flag, const char *arg, const char **val) {
if (*arg != '-') return false;
@ -176,7 +208,7 @@ static int ParseMyArgs(int argc, char **argv) {
GetFlagArgVal("cam_res", arg, &FLAG_camRes) || GetFlagArgVal("mode", arg, &FLAG_mode) ||
GetFlagArgVal("progress", arg, &FLAG_progress) || GetFlagArgVal("show", arg, &FLAG_show) ||
GetFlagArgVal("comp_mode", arg, &FLAG_compMode) || GetFlagArgVal("blur_strength", arg, &FLAG_blurStrength) ||
GetFlagArgVal("cuda_graph", arg, &FLAG_cudaGraph) )) {
GetFlagArgVal("cuda_graph", arg, &FLAG_cudaGraph) || GetFlagArgVal("cam_index", arg, &FLAG_camIndex))) {
continue;
} else if (GetFlagArgVal("help", arg, &help)) {
return NVCV_ERR_HELP;
@ -709,7 +741,6 @@ bail:
FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) {
float ms = 0.0f;
FXApp::Err appErr = errNone;
const int camIndex = 0;
NvCV_Status vfxErr = NVCV_SUCCESS;
bool ok;
cv::Mat result;
@ -725,7 +756,7 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) {
if (inFile) {
reader.open(inFile);
} else {
reader.open(camIndex);
reader.open(FLAG_camIndex);
if (!FLAG_camRes.empty()) {
int camWidth, camHeight, n;
n = sscanf(FLAG_camRes.c_str(), "%d%*[xX]%d", &camWidth, &camHeight);
@ -847,6 +878,10 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) {
CV_8UC3); // Make sure the result is allocated. TODO: allocate outsifde of the loop?
BAIL_IF_NULL(result.data, vfxErr, NVCV_ERR_MEMORY);
result.setTo(cv::Scalar::all(0)); // TODO: This may no longer be necessary since we no longer coerce to 16:9
cv::Mat originalImg;
_srcImg.copyTo(originalImg);
switch (_compMode) {
case compNone:
_srcImg.copyTo(result);
@ -876,7 +911,7 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) {
} else { // If the webcam was cropped, also crop the compositing
cv::Rect rect(0, (_srcImg.rows - _srcVFX.height) / 2, _srcVFX.width, _srcVFX.height);
cv::Mat subResult = result(rect);
overlay(_srcImg(rect), _dstImg(rect), 0.5, subResult);
overlay(_srcImg(rect), _dstImg(rect), 3, subResult);
}
break;
case compGreen: {
@ -918,7 +953,92 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) {
}
if (_show) {
drawFrameRate(result);
cv::imshow("Output", result);
// Ensure _dstImg is grayscale
if (_dstImg.channels() == 3) {
cv::cvtColor(_dstImg, _dstImg, cv::COLOR_BGR2GRAY);
}
// Find edges
cv::Mat edges;
cv::Canny(_dstImg, edges, 100, 200);
// Dilate the edges
cv::Mat dilatedEdges;
const int iterations = 3; // higher = less detail but less errors
cv::dilate(edges, dilatedEdges, cv::Mat(), cv::Point(-1, -1), iterations);
// Ensure _srcImg and outlineResult are the same size and type
cv::Mat outlineResult = cv::Mat::zeros(_srcImg.size(), _srcImg.type());
cv::Mat dilatedMask;
cv::dilate(_dstImg, dilatedMask, cv::Mat(), cv::Point(-1, -1), iterations);
// Convert dilatedMask to three channels if needed
if (dilatedMask.channels() == 1) {
cv::cvtColor(dilatedMask, dilatedMask, cv::COLOR_GRAY2BGR);
}
// Find contours
std::vector<std::vector<cv::Point>> contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(dilatedEdges, contours, hierarchy, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
// Find the largest contour
cv::Mat contourTest = cv::Mat::zeros(_srcImg.size(), _srcImg.type());
double maxArea = 0;
int largestContourIndex = -1;
for (size_t i = 0; i < contours.size(); ++i) {
double area = cv::contourArea(contours[i]);
if (area > maxArea) {
maxArea = area;
largestContourIndex = i;
}
}
if (largestContourIndex >= 0) {
// Fill the largest contour
cv::drawContours(contourTest, contours, largestContourIndex, cv::Scalar(255, 255, 0), cv::FILLED);
}
// Convert contourTest to three channels if needed
if (contourTest.channels() == 1) {
cv::cvtColor(contourTest, contourTest, cv::COLOR_GRAY2BGR);
}
// Combine the dilated mask and filled contour
cv::Mat filledRegion;
cv::bitwise_or(dilatedMask, contourTest, filledRegion);
// Apply the outline color to the filled region
cv::Vec3b outlineColor(0, 0, 255);
for (int y = 0; y < filledRegion.rows; ++y) {
for (int x = 0; x < filledRegion.cols; ++x) {
if (filledRegion.at<cv::Vec3b>(y, x) != cv::Vec3b(0, 0, 0)) {
outlineResult.at<cv::Vec3b>(y, x) = outlineColor;
}
}
}
// Regular outeline
cv::Vec3b outlineColor2(255, 0, 255);
for (int y = 0; y < dilatedEdges.rows; ++y) {
for (int x = 0; x < dilatedEdges.cols; ++x) {
if (dilatedEdges.at<uchar>(y, x) > 0) {
outlineResult.at<cv::Vec3b>(y, x) = outlineColor2;
}
}
}
pushToFrameQueue(0, originalImg);
pushToFrameQueue(1, result);
pushToFrameQueue(2, outlineResult);
// Display the results
cv::imshow("Original", originalImg);
cv::imshow("Overlay", result);
cv::imshow("OutlineTest", outlineResult);
int key = cv::waitKey(1);
if (key > 0) {
appErr = processKey(key);
@ -970,6 +1090,58 @@ bool isCompModeEnumValid(const FXApp::CompMode& mode)
return true;
}
void startHttpServer() {
httplib::Server svr;
auto streamHandler = [](FrameType frameType) {
return [frameType](const httplib::Request&, httplib::Response& res) {
res.set_content_provider(
"multipart/x-mixed-replace; boundary=frame",
[frameType](size_t offset, httplib::DataSink& sink) {
while (true) {
cv::Mat frame;
{
std::lock_guard<std::mutex> lock(frameQueues[frameType].frameMutex);
if (!frameQueues[frameType].frameQueue.empty()) {
frame = frameQueues[frameType].frameQueue.front();
frameQueues[frameType].frameQueue.pop();
}
}
if (!frame.empty()) {
std::vector<uchar> bytes = matToBytes(frame);
// Write multipart frame
std::string header = "--frame\r\nContent-Type: image/jpeg\r\n\r\n";
sink.write(header.data(), header.size());
sink.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
sink.write("\r\n", 2);
}
else {
std::this_thread::sleep_for(std::chrono::milliseconds(30)); // Wait to avoid busy-waiting
}
}
return true; // Continue streaming
}
);
};
};
// Array of route names corresponding to FrameType enum
const std::array<std::string, TYPE_MAX> routeNames = {
"original", "outline", "processed"
};
// Set up routes for each frame type
for (int i = 0; i < TYPE_MAX; ++i) {
std::string route = "/video_" + routeNames[i];
svr.Get(route, streamHandler(static_cast<FrameType>(i)));
}
svr.listen("0.0.0.0", 8080); // Start server on port 8080
}
int main(int argc, char **argv) {
int nErrs = 0;
nErrs = ParseMyArgs(argc, argv);
@ -978,6 +1150,45 @@ int main(int argc, char **argv) {
return nErrs;
}
std::thread serverThread(startHttpServer);
//httplib::Server svr;
//svr.Get("/video", [](const httplib::Request&, httplib::Response& res) {
// cv::VideoCapture cap(0); // Open webcam
// if (!cap.isOpened()) {
// std::cerr << "Error: Cannot open webcam\n";
// return;
// }
// // Content provider for streaming
// res.set_content_provider(
// "multipart/x-mixed-replace; boundary=frame",
// [cap](size_t offset, httplib::DataSink& sink) mutable {
// cv::Mat frame;
// cap >> frame;
// if (!frame.empty()) {
// std::vector<uchar> bytes = matToBytes(frame);
// // Write multipart content
// std::string header = "--frame\r\nContent-Type: image/jpeg\r\n\r\n";
// sink.write(header.data(), header.size());
// sink.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
// sink.write("\r\n", 2);
// }
// return true; // Continue streaming
// },
// [](bool success) {
// // Completion handler when stream ends
// }
// );
// });
//svr.listen("0.0.0.0", 8080);
FXApp::Err fxErr = FXApp::errNone;
FXApp app;

10285
samples/utils/httplib.h
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save