diff --git a/src/PointCloudManager.cpp b/src/PointCloudManager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4127df3abf45185c8fef551d29eb6f0bf7467b66 --- /dev/null +++ b/src/PointCloudManager.cpp @@ -0,0 +1,90 @@ +#include "PointCloudManager.h" + +void PointCloudManager::draw() +{ + //ofLog(OF_LOG_NOTICE) << "Drawing cloud point (" << pointCloud.getVertices().size() << ") points)"; + ofPushMatrix(); + ofNoFill(); + pointCloud.draw(); + auto vs = pointCloud.getVertices(); + if (vs.size() > 0) { + auto v = vs[12000]; + ofLog(OF_LOG_NOTICE) << "Color at vertex 12000 (of " << vs.size() << "): \n" + << "R " << v.r << "\n" + << "G " << v.g << "\n" + << "B " << v.b; + } + ofPopMatrix(); +} + +void PointCloudManager::updateRGB(RGBFrame::Ptr data) +{ + rgbFrameSize.x = data->getCols(); + rgbFrameSize.y = data->getRows(); + + const Color3* rgbData = data->getData(); + uint8_t* colorDataPtr = (uint8_t*)rgbData; + ofPixels rgbPix; + // Assuming the data comes from the live feed (hence BGR) + rgbPix.setFromPixels(colorDataPtr, rgbFrameSize.x, rgbFrameSize.y, OF_PIXELS_BGR); + rgbTex.loadData(rgbPix); + + createPointCloudIfNotExist(depthFrameSize); + + for (int y = 0; y < depthFrameSize.y; y += skip) { + for (int x = 0; x < depthFrameSize.x; x += skip) { + const int index = y * depthFrameSize.x + x; + const int skippedIndex = (y / skip) * (depthFrameSize.x / skip) + (x / skip); + + pointCloud.setColor(skippedIndex, 10 * rgbPix.getColor(index)); + } + } +} + +void PointCloudManager::updateDepth(DepthFrame::Ptr data) +{ + depthFrameSize.x = data->getCols(); + depthFrameSize.y = data->getRows(); + + const unsigned short* depthData = data->getData(); + ofShortPixels pix; + pix.setFromPixels(depthData, depthFrameSize.x, depthFrameSize.y, OF_PIXELS_GRAY); + depthTex.loadData(pix); + + createPointCloudIfNotExist(depthFrameSize); + + for (int y = 0; y < depthFrameSize.y; y += skip) { + for (int x = 0; x < depthFrameSize.x; x += skip) { + const int index = y * depthFrameSize.x + x; + const int skippedIndex = (y / skip) * (depthFrameSize.x / skip) + (x / skip); + + const unsigned short d = depthData[index]; + Vector3 v = depthSensor->convertProjToRealCoords(x, y, depthData[index]); + pointCloud.setVertex(skippedIndex, ofxnui::Tracker::fromVector3(v) * 0.001); + } + } +} + +void PointCloudManager::createPointCloudIfNotExist(glm::vec2 dim) +{ + if (pointCloud.getVertices().size() == 0) { + ofLog(OF_LOG_NOTICE) << "YOU ONLY SEE THIS ONCE"; + + pointCloud.setMode(OF_PRIMITIVE_POINTS); + // Allocate + pointCloud.clear(); + int size = ceil(((float)dim.x / skip) * ((float)dim.y / skip)); + + vector p; + p.assign(size, glm::vec3(0, 0, 0)); + pointCloud.addVertices(p); + + vector c; + c.assign(size, ofFloatColor(0, 0, 0, 0.9)); + pointCloud.addColors(c); + } +} + +void PointCloudManager::setDepthSensor(DepthSensor::Ptr sensor) { + depthSensor = sensor; +} diff --git a/src/PointCloudManager.h b/src/PointCloudManager.h new file mode 100644 index 0000000000000000000000000000000000000000..e441e066d36d3b6db945c274fe6d16ca2ccd0b90 --- /dev/null +++ b/src/PointCloudManager.h @@ -0,0 +1,33 @@ +#pragma once + +#include "ofMain.h" +#include +#include "ofxNuitrack.h" + +using namespace tdv::nuitrack; + +class PointCloudManager +{ +private: + DepthSensor::Ptr depthSensor; + + ofTexture rgbTex; + glm::vec2 rgbFrameSize; + + ofTexture depthTex; + glm::vec2 depthFrameSize; + + ofVboMesh pointCloud; + + const int skip = 2; + + void createPointCloudIfNotExist(glm::vec2 dim); + +public: + void draw(); + void updateRGB(RGBFrame::Ptr data); + void updateDepth(DepthFrame::Ptr data); + + void setDepthSensor(DepthSensor::Ptr sensor); +}; + diff --git a/src/SkeletonFinder.cpp b/src/SkeletonFinder.cpp index d87ad833f9b343cde8891b0ec7eb22b1965145ee..8c1a78a27f1c48b3a6c639013858c8b5f186f512 100644 --- a/src/SkeletonFinder.cpp +++ b/src/SkeletonFinder.cpp @@ -7,426 +7,6 @@ #include "SkeletonFinder.h" -void SkeletonFinder::initNuitrack() { - tracker = ofxnui::Tracker::create(); - tracker->init(""); - - // depth feed settings - tracker->setConfigValue("Realsense2Module.Depth.FPS", "30"); - tracker->setConfigValue("Realsense2Module.Depth.RawWidth", "1280"); - tracker->setConfigValue("Realsense2Module.Depth.RawHeight", "720"); - tracker->setConfigValue("Realsense2Module.Depth.ProcessWidth", "1280"); - tracker->setConfigValue("Realsense2Module.Depth.ProcessHeight", "720"); - tracker->setConfigValue("Realsense2Module.Depth.ProcessMaxDepth", "7000"); - - // rgb feed settings - tracker->setConfigValue("Realsense2Module.RGB.FPS", "30"); - tracker->setConfigValue("Realsense2Module.RGB.RawWidth", "1280"); - tracker->setConfigValue("Realsense2Module.RGB.RawHeight", "720"); - tracker->setConfigValue("Realsense2Module.RGB.ProcessWidth", "1280"); - tracker->setConfigValue("Realsense2Module.RGB.ProcessHeight", "720"); - - // post processing settings - tracker->setConfigValue("Realsense2Module.Depth.PostProcessing.SpatialFilter.spatial_alpha", "0.1"); - tracker->setConfigValue("Realsense2Module.Depth.PostProcessing.SpatialFilter.spatial_delta", "50"); - - // distance settings - tracker->setConfigValue("Segmentation.MAX_DISTANCE", "7000"); - tracker->setConfigValue("Skeletonization.MaxDistance", "7000"); - - tracker->setIssuesCallback([this](nuitrack::IssuesData::Ptr data) { - auto issue = data->getIssue(); - if (issue) { - ofLogNotice() << "Issue detected! " << issue->getName() << " [" << issue->getId() << "]"; - } - }); - //tracker->setRGBCallback([](nuitrack::RGBFrame::Ptr) {ofLog(OF_LOG_NOTICE) << "Skeleton found ! None position : "; }); - //tracker->setDepthCallback([](nuitrack::DepthFrame::Ptr) {}); - //// skeleton detection callback - //tracker->setSkeletonCallback([this](nuitrack::SkeletonData::Ptr data) { - // update(data); - // }); -} - -void SkeletonFinder::setup(ofMatrix4x4* transformMatrix, ofxGui& gui) { - this->transformMatrix = transformMatrix; - initNuitrack(); - initGUI(gui); -} - -void SkeletonFinder::update(nuitrack::SkeletonData::Ptr data) { - skeletons.clear(); - // TODO: filter using the capture bounds - for (nuitrack::Skeleton skel : data->getSkeletons()) { - /*vector joints; - for (nuitrack::Joint joint : skel.joints) { - ofVec3f pos = ofxnui::Tracker::fromVector3(joint.real); - pos = *transformMatrix * pos; - - joints.push_back(Joint(joint.type, joint.confidence, pos)); - } - - skeletons.push_back(Skeleton(skel.id, joints));*/ - ofLog(OF_LOG_NOTICE) << "Skeleton found !"; - /*ofLog(OF_LOG_NOTICE) << "Skeleton found ! None position : " << joints[0].pos; - ofLog(OF_LOG_NOTICE) << "Skeleton found ! Head position : " << joints[0].pos;*/ - } -} - -void SkeletonFinder::run() { - tracker->run(); -} - - -// TODO: remove once it is safe to do so -// There could be something of interest here -void /*SkeletonFinder::*/update() { - return; - //ofColor white = ofColor::white; - //ofColor black = ofColor::black; - - ///**************************************************************** - // PREPARE NEW FRAME - //*****************************************************************/ - - //int minID = 0; - - ////tells all the blobs that the next frame is comming - //for (int e = 0; e < skeletonEvents.size(); e++) { - // skeletonEvents[e].updatePrepare(); - // minID = (skeletonEvents[e].mID >= minID) ? skeletonEvents[e].mID + 1 : minID; - //} - - //if (useMask.get()) { - // fbo.begin(); - // // Cleaning everthing with alpha mask on 0 in order to make it transparent for default - // ofClear(0, 0, 0, 0); - - // maskShader.begin(); - // maskShader.setUniformTexture("maskTex", maskImg.getTexture(), 1); - - // captureFBO.draw(0, 0); - - // maskShader.end(); - // fbo.end(); - - // fbo.readToPixels(fbopixels); - //} - //else { - // captureFBO.readToPixels(fbopixels); - //} - - // colorImg.setFromPixels(fbopixels); - - // // load grayscale captured depth image from the color source - // grayImage.setFromColorImage(colorImg); - // - // //grayImage.blurHeavily(); - // - //ofPixelsRef blobRefPixls = skeletonRef.getPixels(); - // - // ofPixelsRef greyref = grayImage.getPixels(); - //ofPixelsRef eyeRef = grayEyeLevel.getPixels(); - // - //eyeRef.setColor(black); - - // float sensorFieldFront = sensorBoxFront.get() * SCALE; - // float sensorFieldBack = sensorBoxBack.get() * SCALE; - // float sensorFieldLeft = sensorBoxLeft.get() * SCALE; - // float sensorFieldRight = sensorBoxRight.get() * SCALE; - // float sensorFieldTop = sensorBoxTop .get() * SCALE; - // float sensorFieldBottom = sensorBoxBottom.get() * SCALE; - // float sensorFieldWidth = sensorFieldRight - sensorFieldLeft; - // float sensorFieldHeigth = sensorFieldTop - sensorFieldBottom; - // float sensorFieldDepth = sensorFieldBack - sensorFieldFront; - // - // int eyeLevelColor = eyeLevel.get() / sensorFieldHeigth * 255; - // - // int headTopThreshold = eyeLevelColor / 4; - // - // //ofLog(OF_LOG_NOTICE, "eyeLevelColor = " + ofToString(eyeLevelColor)); - // - // //ofLog(OF_LOG_NOTICE, "eyref size : " + ofToString(eyeRef.size())); - // - ///**************************************************************** - // FIND BODY CONTOURS - //*****************************************************************/ - - //int minBlobSize = pow(blobAreaMinStp1.get() * sensorFboSize.get(), 2); - //int maxBlobSize = pow(blobAreaMax.get() * sensorFboSize.get(), 2); - - - //detectedHeads.clear(); - - //contourFinder.findContours(grayImage, minBlobSize, maxBlobSize, countBlob.get(), false); - - // for (int i = 0; i < contourFinder.nBlobs; i++){ - // ofRectangle bounds = contourFinder.blobs[i].boundingRect; - // float pixelBrightness = 0; - // float brightness = 0; - // - // // find the brightest pixel within the blob. this defines the height of the blob - // for(int x = bounds.x; x < bounds.x + bounds.width; x++){ - // for(int y = bounds.y; y < bounds.y + bounds.height; y++){ - // pixelBrightness = greyref.getColor(x, y).getBrightness(); - // brightness = (pixelBrightness > brightness)?pixelBrightness: brightness; - // } - // } - - // /* - // float averageBrightness = 0; - // int averageCounter = 0; - - // // go through the pixels again and get the average brightness for the headTopThreshold - // for(int x = bounds.x; x < bounds.x + bounds.width; x++){ - // for(int y = bounds.y; y < bounds.y + bounds.height; y++){ - // pixelBrightness = greyref.getColor(x, y).getBrightness(); - // if(pixelBrightness > brightness - headTopThreshold){ - // averageBrightness += pixelBrightness; - // averageCounter++; - // } - // } - // } - // - // brightness = averageBrightness / averageCounter; - // */ - // - // // find all the pixels down to the eyelevel threshold. this yealds an image with blobs that mark the size of the head at eyelevel. - // ofVec2f headtop2d = ofVec2f(); - // int brighCounter = 0; - // for(int x = bounds.x; x < bounds.x + bounds.width; x++){ - // for(int y = bounds.y; y < bounds.y + bounds.height; y++){ - // pixelBrightness = greyref.getColor(x, y).getBrightness(); - // if(pixelBrightness > (brightness - eyeLevelColor)){ - // //writes the pixels above the eyelevel into the eyeRef image - // eyeRef.setColor(x, y, brightness); - // }else{ - // eyeRef.setColor(x, y, black); - // } - // if(pixelBrightness >= brightness - (eyeLevelColor / 4)){ - // headtop2d += ofVec2f(x, y); - // brighCounter++; - // } - // } - // } - // headtop2d /= brighCounter; - // - // //ofLog(OF_LOG_NOTICE, "headtop2d = " + ofToString(headtop2d)); - // - // ofVec3f headTop = ofVec3f((headtop2d.x / captureScreenSize.x) * sensorFieldWidth + sensorFieldLeft, sensorFieldBack - (headtop2d.y / captureScreenSize.y ) * sensorFieldDepth, (brightness / 255.0) * sensorFieldHeigth + sensorFieldBottom); - - // //ofVec3f headCenter = ofVec3f(headTop.x, headTop.y, headTop.z - eyeLevel.get()); - - // DetectedSkeleton newBodyBlob; - // newBodyBlob.bound = bounds; - // newBodyBlob.headTop = headTop; - // newBodyBlob.hasBeenTaken = false; - - // detectedHeads.push_back(newBodyBlob); - - //} - - ///**************************************************************** - // UPDATE BODY EVENTS - //*****************************************************************/ - - //int matchBlobID = -1; - //int matchEventID = -1; - //float minDistance = 1000000; - - //do { - // matchBlobID = -1; - // matchEventID = -1; - // minDistance = 1000000; - // for (int i = 0; i < skeletonEvents.size(); i++) { // iterate through all the current events - // if (skeletonEvents[i].isAlive() && !skeletonEvents[i].hasBeenUpdated()) { // all those that haven't been updated yet - // for (int j = 0; j < detectedHeads.size(); j++) { // now we go through all the new blobs - // if (!detectedHeads[j].hasBeenTaken) { // but only if the blobs havent been taken before - // float dist = skeletonEvents[i].getDistance(detectedHeads[j].headTop); - // if (dist < eventMaxSize.get()*SCALE) { // and test if the distance is within the threshold - // if (minDistance > dist) { - // minDistance = dist; - // matchBlobID = j; - // matchEventID = i; - // } - // } - // } - // } - // } - // } - // if (matchEventID >= 0) { - // ofRectangle bounds = detectedHeads[matchBlobID].bound; - // ofVec3f headTop = detectedHeads[matchBlobID].headTop; - - // //calculate the blob pos in worldspace - // ofVec3f blobPos = ofVec3f(((float)bounds.getCenter().x / captureScreenSize.x) * sensorFieldWidth + sensorFieldLeft, sensorFieldBack - ((float)bounds.getCenter().y / captureScreenSize.y) * sensorFieldDepth, headTop.z); - - // //calculate the blob size in worldspace - // ofVec2f blobSize = ofVec2f(((float)bounds.getWidth() / captureScreenSize.x) * sensorFieldWidth, ((float)bounds.getHeight() / captureScreenSize.y) * sensorFieldDepth); - - // //ofLogVerbose("Updating old event with ID: " + ofToString(blobEvents[matchEventID].mID)); - - // skeletonEvents[matchEventID].update(bounds, blobPos, blobSize, headTop, smoothFactor.get()); - // detectedHeads[matchBlobID].hasBeenTaken = true; - // } - //} while (matchEventID >= 0); - - ///**************************************************************** - // CREATE NEW BODY EVENTS - //*****************************************************************/ - //for (int j = 0; j < detectedHeads.size(); j++) { - // if (!detectedHeads[j].hasBeenTaken) { - // ofRectangle bounds = detectedHeads[j].bound; - // ofVec3f headTop = detectedHeads[j].headTop; - - // //calculate the blob pos in worldspace - // ofVec3f blobPos = ofVec3f(((float)bounds.getCenter().x / captureScreenSize.x) * sensorFieldWidth + sensorFieldLeft, sensorFieldBack - ((float)bounds.getCenter().y / captureScreenSize.y) * sensorFieldDepth, headTop.z); - - // //calculate the blob size in worldspace - // ofVec2f blobSize = ofVec2f(((float)bounds.getWidth() / captureScreenSize.x) * sensorFieldWidth, ((float)bounds.getHeight() / captureScreenSize.y) * sensorFieldDepth); - - // //ofLogVerbose("Creating new event with ID: " + ofToString(minID)); - - // skeletonEvents.push_back(SkeletonTracker(minID++, eventBreathSize.get(), bounds, blobPos, blobSize, headTop)); - // } - //} - - - ///**************************************************************** - // FIND HEAD CONTOURS - //*****************************************************************/ - ///** - - // //preprocesses the eyeRef image - // //grayEyeLevel.setFromPixels(eyeRef.getPixels(), eyeRef.getWidth(), eyeRef.getHeight()); - // //grayEyeLevel.invert(); - // //grayEyeLevel.threshold(20); - // //grayEyeLevel.invert(); - // grayEyeLevel.blurGaussian(); - - // //ofLog(OF_LOG_NOTICE, "contourEyeFinder nBlobs : " + ofToString(contourEyeFinder.nBlobs)); - - //int minBlobSize2 = pow(blobAreaMinStp2.get() * sensorFboSize.get(), 2); - - // //find head shape on eye height contours - // contourEyeFinder.findContours(grayEyeLevel, minBlobSize2, maxBlobSize, countBlob.get(), false); - // for(int i = 0; i < contourEyeFinder.nBlobs; i++){ - - // ofRectangle bounds = contourEyeFinder.blobs[i].boundingRect; - - // int brightness = eyeRef.getColor((float)bounds.getCenter().x, (float)bounds.getCenter().y).getBrightness(); - // float height = (brightness / 255.0) * sensorFieldHeigth + sensorFieldBottom; - - // //calculate the blob pos in worldspace - // ofVec3f headBlobCenter = ofVec3f(((float)bounds.getCenter().x / captureScreenSize.x) * sensorFieldWidth + sensorFieldLeft, sensorFieldBack - ((float)bounds.getCenter().y / captureScreenSize.y) * sensorFieldDepth, height); - - // //calculate the blob size in worldspace - // ofVec2f headBlobSize = ofVec2f(((float)bounds.getWidth() / captureScreenSize.x) * sensorFieldWidth, ((float)bounds.getHeight() / captureScreenSize.y) * sensorFieldDepth); - - // //calculate the gazeVector - // //ofVec3f gaze = blobEvents[bid].getCurrentHeadCenter() - gazePoint.get(); - - // //gaze.z = 0; - - // float smalestAngle = 180; - // ofVec3f eyePoint = headBlobCenter; - - // //clears the contour storage - // blobEvents[bid].countour.clear(); - - // // findes the closest contour point to the eyegave-vector, takes its distance to the headCenter and calculated - // // the eye - center - point - // for (int v = 0; v < contourEyeFinder.blobs[i].pts.size(); v++) { - // ofVec3f headPoint = ofVec3f(((float)contourEyeFinder.blobs[i].pts[v].x / captureScreenSize.x) * sensorFieldWidth + sensorFieldLeft, sensorFieldBack - ((float)contourEyeFinder.blobs[i].pts[v].y / captureScreenSize.y) * sensorFieldDepth, blobEvents[bid].headCenter.z); - - // blobEvents[bid].countour.push_back(headPoint); - - // ofVec3f gaze2 = blobEvents[bid].getCurrentHeadCenter() - headPoint; - - // float angle = gaze.angle(gaze2); - - // if (smalestAngle > angle) { - // smalestAngle = angle; - // eyePoint = blobEvents[bid].getCurrentHeadCenter() - gaze.normalize().scale(gaze2.length() * eyeInset.get()); - // } - // } - - // bool foundBlob = false; - - // for(int bid = 0; bid < blobEvents.size(); bid++){ - // // find the blob - // if(!blobEvents[bid].hasBeenUpdated() && blobEvents[bid].isMatching(bounds, eventMaxSize.get())){ - // - // ///////////////////////////////////////////////////////////////// - // UPDATE HEAD EVENTS - // ///////////////////////////////////////////////////////////////// - - // ofLogVerbose("Updating old event with ID: " + ofToString(blobEvents[bid].mID) + " headHight = " + ofToString(headBlobCenter.z)); - - // blobEvents[bid].update(bounds, headBlobCenter, headBlobSize, eyePoint, smoothFactor.get()); - // foundBlob = true; - // break; - // } - // } - // // if none is found, create a new one. - // if (!foundBlob) { - // ofLogVerbose("Creating new event with ID: " + ofToString(minID)); - // blobEvents.push_back(BlobTracker(minID++, bounds, eventBreathSize.get())); - // blobEvents.back().update(bounds, headBlobCenter, headBlobSize, eyePoint, smoothFactor.get()); - // } - - // } - //*/ - - ///**************************************************************** - // SORT EVENTS - //*****************************************************************/ - - ///* - ////sets the sort value to the current index. - //int sortPos = 0; - - //for (int i = 0; i < blobEvents.size(); i++) { - // blobEvents[i].sortPos = sortPos++; - //} - - //// if we are using the gaze point - //if (useGazePoint.get()) { - // if (blobEvents.size() > 0) { - // for (int i = 0; i < (blobEvents.size() - 1); i++) { - // for (int j = 1; j < blobEvents.size(); j++) { - // if ((blobEvents[i].headCenter - gazePoint.get()).length() < (blobEvents[j].headCenter - gazePoint.get()).length()) { - // if (blobEvents[i].sortPos > blobEvents[j].sortPos) { - // int savepos = blobEvents[j].sortPos; - // blobEvents[j].sortPos = blobEvents[i].sortPos; - // blobEvents[i].sortPos = savepos; - // } - // } - // else { - // if (blobEvents[i].sortPos < blobEvents[j].sortPos) { - // int savepos = blobEvents[j].sortPos; - // blobEvents[j].sortPos = blobEvents[i].sortPos; - // blobEvents[i].sortPos = savepos; - // } - // } - // } - // } - // } - //} - //*/ - - // - ///**************************************************************** - // CLEANUP EVENTS - //*****************************************************************/ - - // //updates all alive blobs and removes all the blobs that havent had an update for a specific number of frames or have been killed - // for(int e = skeletonEvents.size() - 1; e >= 0; e--){ - // if(skeletonEvents[e].isDead()){ - // skeletonEvents.erase(skeletonEvents.begin() + e); - // } - // } -} void SkeletonFinder::initGUI(ofxGui& gui) { panel = gui.addPanel(); @@ -454,6 +34,31 @@ void SkeletonFinder::initGUI(ofxGui& gui) { panel->loadFromFile("trackings.xml"); } +void SkeletonFinder::setTransformMatrix(ofMatrix4x4* mat) { + transformMatrix = mat; +} + +void SkeletonFinder::update(nuitrack::SkeletonData::Ptr data) { + skeletons.clear(); + // TODO: filter using the capture bounds + for (nuitrack::Skeleton skel : data->getSkeletons()) { + vector joints; + for (nuitrack::Joint joint : skel.joints) { + glm::vec3 pos = ofxnui::Tracker::fromVector3(joint.real); + // pos = *transformMatrix * pos; + pos = 0.001 * pos; + + joints.push_back(Joint(joint.type, joint.confidence, pos)); + } + + skeletons.push_back(Skeleton(skel.id, joints)); + } +} + +vector SkeletonFinder::getSkeletons() { + return skeletons; +} + void SkeletonFinder::drawSensorBox() { sensorBox.draw(); @@ -475,7 +80,7 @@ void SkeletonFinder::drawSkeletons2d(ofRectangle _rect) { }*/ } -// adapted from example +// adapted from ofxNuitrack example void SkeletonFinder::drawSkeletons() { static vector bones = { Bone(nuitrack::JOINT_WAIST, nuitrack::JOINT_TORSO, glm::vec3(0, 1, 0)), @@ -501,6 +106,7 @@ void SkeletonFinder::drawSkeletons() { Bone(nuitrack::JOINT_RIGHT_KNEE, nuitrack::JOINT_RIGHT_ANKLE, glm::vec3(0, -1, 0)), }; + ofSetColor(0, 255, 0); for (Skeleton skel : skeletons) { for (Bone bone : bones) { auto j1 = skel.joints[bone.from]; @@ -510,7 +116,6 @@ void SkeletonFinder::drawSkeletons() { continue; } - ofSetColor(ofColor(0, 1, 0)); ofDrawLine(j1.pos, j2.pos); } } diff --git a/src/SkeletonFinder.h b/src/SkeletonFinder.h index 25066437939878f2cc034986e32650fb34433fae..ca3889621f6bfa504ffeed11d783f5935d1bffa5 100644 --- a/src/SkeletonFinder.h +++ b/src/SkeletonFinder.h @@ -26,41 +26,12 @@ using namespace tdv; -// TODO: This is the file which will change the most -/* included in ofApp and TrackingNetworkManager -methods and fields called in ofApp : - setup(gui) - panel->set{Width,Position} - { - //update - skeletonFinder.captureMaskBegin(); - drawCapturePointCloud(true); - skeletonFinder.captureMaskEnd(); - } - update - { - //draw - skeletonFinder.grayImage.draw(viewGrid[2]); - skeletonFinder.contourFinder.draw(viewGrid[3]); - skeletonFinder.maskFbo.draw(viewGrid[4]); - } - drawSkel2d - drawSkel - sensorBoxXXX - clearMask - saveMask - loadMask -methods and fields called in TrackingetworkManager : - sensorBoxXXX - blobEvents // should be bypassed via nuitrack itself -*/ - struct Joint { nuitrack::JointType type; float confidence; - ofVec3f pos; + glm::vec3 pos; - Joint(nuitrack::JointType type, float confidence, ofVec3f pos) : + Joint(nuitrack::JointType type, float confidence, glm::vec3 pos) : type(type), confidence(confidence), pos(pos) {} }; @@ -89,8 +60,9 @@ class SkeletonFinder { public: SkeletonFinder() {} - void setup(ofMatrix4x4* transformMatrix, ofxGui &gui); - void run(); + void initGUI(ofxGui& gui); + void setTransformMatrix(ofMatrix4x4* mat); + void update(nuitrack::SkeletonData::Ptr data); void updateSensorBox(int & value); @@ -98,18 +70,14 @@ public: void drawSkeletons2d(ofRectangle _rect); void drawSkeletons(); - //vector getSkeletons(); + vector getSkeletons(); private: - void initNuitrack(); - void initGUI(ofxGui& gui); - ofxnui::TrackerRef tracker; vector skeletons; ofMatrix4x4* transformMatrix; - void update(nuitrack::SkeletonData::Ptr data); public: ofxGuiPanel *panel; diff --git a/src/ofApp.cpp b/src/ofApp.cpp index 55a9b120d1ddc095b147b9567a47b4bd3a7a109b..2fe01eab3f34d9e58abf794f17853fb5c8b71091 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -7,41 +7,59 @@ //-------------------------------------------------------------- -void ofApp::setup(){ -#ifdef TARGET_OPENGLES - shader.load("shadersES2/shader"); -#else - if (ofIsGLProgrammableRenderer()) { - shader.load("shadersGL3/shader"); - } - else { - shader.load("shadersGL2/shader"); - } -#endif - - ofLog(OF_LOG_NOTICE, "MainAPP: looking for RealSense Device..."); - - ofSetLogLevel(OF_LOG_VERBOSE); - - // crashes here - realSense = RSDevice::createUniquePtr(); - - realSense->checkConnectedDialog(); - - //realSense->hardwareReset(); - - realSense->setVideoSize(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT); - - ofLog(OF_LOG_NOTICE, "... RealSense Device found."); - - // we don't want to be running to fast - //ofSetVerticalSync(true); - //ofSetFrameRate(30); - - ///////////////////////////// - // DEFINE VIEWPORTS // - ///////////////////////////// +void ofApp::initNuitrack() { + tracker = ofxnui::Tracker::create(); + tracker->init(""); + + // depth feed settings + tracker->setConfigValue("Realsense2Module.Depth.FPS", "30"); + tracker->setConfigValue("Realsense2Module.Depth.RawWidth", "1280"); + tracker->setConfigValue("Realsense2Module.Depth.RawHeight", "720"); + tracker->setConfigValue("Realsense2Module.Depth.ProcessWidth", "1280"); + tracker->setConfigValue("Realsense2Module.Depth.ProcessHeight", "720"); + tracker->setConfigValue("Realsense2Module.Depth.ProcessMaxDepth", "7000"); + + // rgb feed settings + tracker->setConfigValue("Realsense2Module.RGB.FPS", "30"); + tracker->setConfigValue("Realsense2Module.RGB.RawWidth", "1280"); + tracker->setConfigValue("Realsense2Module.RGB.RawHeight", "720"); + tracker->setConfigValue("Realsense2Module.RGB.ProcessWidth", "1280"); + tracker->setConfigValue("Realsense2Module.RGB.ProcessHeight", "720"); + + // feeds alignement + tracker->setConfigValue("DepthProvider.Depth2ColorRegistration", "true"); + + // post processing settings + tracker->setConfigValue("Realsense2Module.Depth.PostProcessing.SpatialFilter.spatial_alpha", "0.1"); + tracker->setConfigValue("Realsense2Module.Depth.PostProcessing.SpatialFilter.spatial_delta", "50"); + + // distance settings + tracker->setConfigValue("Segmentation.MAX_DISTANCE", "7000"); + tracker->setConfigValue("Skeletonization.MaxDistance", "7000"); + + tracker->setIssuesCallback([this](nuitrack::IssuesData::Ptr data) { + auto issue = data->getIssue(); + if (issue) { + ofLogNotice() << "Issue detected! " << issue->getName() << " [" << issue->getId() << "]"; + } + }); + tracker->setRGBCallback([this](nuitrack::RGBFrame::Ptr data) { + pointCloudManager.updateRGB(data); + }); + tracker->setDepthCallback([this](nuitrack::DepthFrame::Ptr data) { + pointCloudManager.updateDepth(data); + }); + tracker->setSkeletonCallback([this](nuitrack::SkeletonData::Ptr data) { + skeletonFinder.update(data); + }); + + pointCloudManager.setDepthSensor(tracker->depthTracker); + + // TODO: maybe account for this, that used to be in old code : + // realSense->setVideoSize(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT); +} +void ofApp::initViewports() { float xOffset = VIEWGRID_WIDTH; //ofGetWidth() / 3; float yOffset = VIEWPORT_HEIGHT / N_CAMERAS; @@ -58,76 +76,41 @@ void ofApp::setup(){ viewGrid[i].height = yOffset; } - iMainCamera = 0; - - previewCam.setUpAxis(glm::vec3(0, 0, 1)); - previewCam.setTranslationSensitivity(2., 2., 2.); + iMainCamera = 0; + + previewCam.setUpAxis(glm::vec3(0, 0, 1)); + previewCam.setTranslationSensitivity(2., 2., 2.); previewCam.setNearClip(0.001f); +} - //////////////////// - // BLOBFINDER // - //////////////////// - ofLog(OF_LOG_NOTICE, "MainAPP: setting up blobfinder"); +void ofApp::setupCalibGui() { + // TODO: remove what is useless - skeletonFinder.setup(&deviceTransform, gui); + setupCalib = gui.addPanel(); - ///////////////////////////// - // REALSENSE GUI SETUP // - ///////////////////////////// - ofLog(OF_LOG_NOTICE, "MainAPP: loading postprocessing GUI"); + setupCalib->loadTheme("theme/theme_light.json"); + setupCalib->setName("Calibration Panel"); - post = gui.addPanel(); - post->loadTheme("theme/theme_light.json"); - post->setName("PostProcessing"); - post->add(realSense->param_usePostProcessing); - post->add(realSense->param_filterDecimation); - post->add(realSense->param_filterDecimation_mag); - post->add(realSense->param_filterDisparities); - post->add(realSense->param_filterSpatial); - post->add(realSense->param_filterSpatial_smoothAlpha); - post->add(realSense->param_filterSpatial_smoothDelta); - post->add(realSense->param_filterSpatial_mag); - post->add(realSense->param_filterTemporal); - post->add(realSense->param_filterTemporal_smoothAlpha); - post->add(realSense->param_filterTemporal_smoothDelta); - post->add(realSense->param_filterTemporal_persistency); + setupCalib->add(blobGrain.set("Grain", 2, 1, 10)); - post->loadFromFile("postprocessing.xml"); + setupCalib->add(calibPoint_X.set("calibrationPoint_X", ofVec2f(REALSENSE_VIDEO_WIDTH / 2, REALSENSE_VIDEO_HEIGHT / 2), ofVec2f(0, 0), ofVec2f(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT))); + setupCalib->add(calibPoint_Y.set("calibrationPoint_Y", ofVec2f(REALSENSE_VIDEO_WIDTH / 2, REALSENSE_VIDEO_HEIGHT / 2), ofVec2f(0, 0), ofVec2f(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT))); + setupCalib->add(calibPoint_Z.set("calibrationPoint_Z", ofVec2f(REALSENSE_VIDEO_WIDTH / 2, REALSENSE_VIDEO_HEIGHT / 2), ofVec2f(0, 0), ofVec2f(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT))); + nearFrustum.addListener(this, &ofApp::updateFrustumCone); + farFrustum.addListener(this, &ofApp::updateFrustumCone); - ///////////////////////////// - //CALIBRATION GUI SETUP // - //////////////////////////// - ofLog(OF_LOG_NOTICE, "MainAPP: loading calibration settings"); + frustumGuiGroup.setName("frustumField"); + frustumGuiGroup.add(nearFrustum.set("nearFrustum", 400, 200, 2000)); + frustumGuiGroup.add(farFrustum.set("farFrustum", 4000, 2000, 10000)); + setupCalib->addGroup(frustumGuiGroup); - setupCalib = gui.addPanel(); - - setupCalib->loadTheme("theme/theme_light.json"); - setupCalib->setName("Calibration Panel"); - - setupCalib->add(blobGrain.set("Grain", 2, 1, 10)); - - setupCalib->add(calibPoint_X.set("calibrationPoint_X", ofVec2f(REALSENSE_VIDEO_WIDTH / 2, REALSENSE_VIDEO_HEIGHT / 2), ofVec2f(0, 0), ofVec2f(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT))); - setupCalib->add(calibPoint_Y.set("calibrationPoint_Y", ofVec2f(REALSENSE_VIDEO_WIDTH / 2, REALSENSE_VIDEO_HEIGHT / 2), ofVec2f(0, 0), ofVec2f(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT))); - setupCalib->add(calibPoint_Z.set("calibrationPoint_Z", ofVec2f(REALSENSE_VIDEO_WIDTH / 2, REALSENSE_VIDEO_HEIGHT / 2), ofVec2f(0, 0), ofVec2f(REALSENSE_VIDEO_WIDTH, REALSENSE_VIDEO_HEIGHT))); - - nearFrustum.addListener(this, &ofApp::updateFrustumCone); - farFrustum.addListener(this, &ofApp::updateFrustumCone); - - frustumGuiGroup.setName("frustumField"); - frustumGuiGroup.add(nearFrustum.set("nearFrustum", 400, 200, 2000)); - frustumGuiGroup.add(farFrustum.set("farFrustum", 4000, 2000, 10000)); - setupCalib->addGroup(frustumGuiGroup); - - //setupCalib->add(transformation.set("matrix rx ry tz", ofVec3f(0, 0, 0), ofVec3f(-90, -90, -6000), ofVec3f(90, 90, 6000))); - - setupCalib->loadFromFile("settings.xml"); + //setupCalib->add(transformation.set("matrix rx ry tz", ofVec3f(0, 0, 0), ofVec3f(-90, -90, -6000), ofVec3f(90, 90, 6000))); - //////////////////////////// - // GUI Transfromation // - //////////////////////////// - ofLog(OF_LOG_NOTICE, "MainAPP: loading transformation matrix"); + setupCalib->loadFromFile("settings.xml"); +} +void ofApp::setupTransformGui() { guitransform = gui.addPanel(); guitransform->loadTheme("theme/theme_light.json"); @@ -143,7 +126,64 @@ void ofApp::setup(){ bool invisible = false; guitransform->setVisible(invisible); +} + +void ofApp::setup(){ + //ofLog(OF_LOG_NOTICE) << "Entered setup"; +#ifdef TARGET_OPENGLES + shader.load("shadersES2/shader"); +#else + if (ofIsGLProgrammableRenderer()) { + shader.load("shadersGL3/shader"); + } + else { + shader.load("shadersGL2/shader"); + } +#endif + + ofLog(OF_LOG_NOTICE) << "Nuitrack setup started"; + initNuitrack(); + skeletonFinder.setTransformMatrix(&deviceTransform); + ofLog(OF_LOG_NOTICE) << "Nuitrack setup ended"; + + // we don't want to be running too fast + //ofSetVerticalSync(true); + //ofSetFrameRate(30); + + ofLog(OF_LOG_NOTICE) << "Viewport setup started"; + initViewports(); + ofLog(OF_LOG_NOTICE) << "Viewport setup ended"; + + ///////////////////////////// + // REALSENSE GUI SETUP // + ///////////////////////////// + /* + ofLog(OF_LOG_NOTICE, "MainAPP: loading postprocessing GUI"); + + post = gui.addPanel(); + post->loadTheme("theme/theme_light.json"); + post->setName("PostProcessing"); + post->add(realSense->param_usePostProcessing); + post->add(realSense->param_filterDecimation); + post->add(realSense->param_filterDecimation_mag); + post->add(realSense->param_filterDisparities); + post->add(realSense->param_filterSpatial); + post->add(realSense->param_filterSpatial_smoothAlpha); + post->add(realSense->param_filterSpatial_smoothDelta); + post->add(realSense->param_filterSpatial_mag); + post->add(realSense->param_filterTemporal); + post->add(realSense->param_filterTemporal_smoothAlpha); + post->add(realSense->param_filterTemporal_smoothDelta); + post->add(realSense->param_filterTemporal_persistency); + + post->loadFromFile("postprocessing.xml"); + */ + ofLog(OF_LOG_NOTICE, "Loading clibration GUI"); + setupCalibGui(); + ofLog(OF_LOG_NOTICE, "Loading transformation GUI"); + setupTransformGui(); + updateMatrix(); ///////////////////////////// @@ -158,21 +198,21 @@ void ofApp::setup(){ // OPERATING GUI // ///////////////////////////// - operating = gui.addPanel(); - operating->loadTheme("theme/theme_light.json"); - operating->setName("Operating"); + //operating = gui.addPanel(); + //operating->loadTheme("theme/theme_light.json"); + //operating->setName("Operating"); - operatingModes.setName("Modes"); - operatingModes.add(mode0Capture.set("normal", false)); - operatingModes.add(mode1Record.set("recording", false)); - operatingModes.add(mode2Playback.set("playback", false)); + //operatingModes.setName("Modes"); + //operatingModes.add(mode0Capture.set("normal", false)); + //operatingModes.add(mode1Record.set("recording", false)); + //operatingModes.add(mode2Playback.set("playback", false)); - operatingToggles = operating->addGroup(operatingModes); - operatingToggles->setExclusiveToggles(true); - operatingToggles->setConfig(ofJson({ {"type", "radio"} })); + //operatingToggles = operating->addGroup(operatingModes); + //operatingToggles->setExclusiveToggles(true); + //operatingToggles->setConfig(ofJson({ {"type", "radio"} })); - operatingToggles->setActiveToggle(0); - operatingToggles->getActiveToggleIndex().addListener(this, &ofApp::changeOperation); + //operatingToggles->setActiveToggle(0); + //operatingToggles->getActiveToggleIndex().addListener(this, &ofApp::changeOperation); //////////////////////// // RealSense // @@ -181,9 +221,8 @@ void ofApp::setup(){ ofLog(OF_LOG_NOTICE, "MainAPP: starting attached Device..."); // firing up the device, creating the GUI and loading the device parameters - if (realSense->capture()) { - createGUIDeviceParams(); - } + tracker->run(); + createGUIDeviceParams(); ofLog(OF_LOG_NOTICE, "...starting attached Device done."); @@ -192,130 +231,101 @@ void ofApp::setup(){ bPreviewPointCloud = false; ofLog(OF_LOG_NOTICE, "MainAPP: setting up networking..."); - - networkMng.setup(gui, realSense->getSerialNumber(-1)); - + // TODO: remove useless arg + networkMng.setup(gui, "sauce"); ofLog(OF_LOG_NOTICE, "...networking done."); - int * val = 0; - updateFrustumCone(*val); - + int val = 0; + updateFrustumCone(val); + //crashes here setupViewports(); - - skeletonFinder.run(); - createHelp(); + + tracker->run(); capMesh.reSize(4); - ofSetLogLevel(OF_LOG_NOTICE); - ofSetLogLevel(OF_LOG_VERBOSE); //ofLogToFile("myLogFile.txt", true); + ofLog(OF_LOG_NOTICE) << "Setup over"; if (ofIsGLProgrammableRenderer()) { ofLog(OF_LOG_NOTICE, "ofIsGLProgrammableRenderer() = " + ofToString(ofIsGLProgrammableRenderer())); } } -void ofApp::changeOperation(int& _index) { - - switch (_index) { - case 0: - if (realSense->capture()) { - createGUIDeviceParams(); - setupViewports(); - } - break; - case 1: - if (realSense->record()) { - createGUIDeviceParams(); - setupViewports(); - } - break; - case 2: - if (realSense->playback()) { - } - break; - } -} +void ofApp::changeOperation(int& _index) {} void ofApp::createGUIDeviceParams() { + //ofLog(OF_LOG_NOTICE) << "Entered createGUIDeviceParams"; device->clear(); device->loadTheme("theme/theme_light.json"); device->setName("RealSense Device"); - device->add(realSense->getSerialNumber(-1)); intrinsicGuiGroup.clear(); intrinsicGuiGroup.setName("Settings"); - intrinsicGuiGroup.add(realSense->param_deviceLaser); - intrinsicGuiGroup.add(realSense->param_deviceLaser_mag); - intrinsicGuiGroup.add(realSense->param_deviceAutoExposure); - intrinsicGuiGroup.add(realSense->param_deviceExposure_mag); - intrinsicGuiGroup.add(realSense->param_deviceGain_mag); - intrinsicGuiGroup.add(realSense->param_deviceFrameQueSize_mag); - intrinsicGuiGroup.add(realSense->param_deviceAsicTemparature); - intrinsicGuiGroup.add(realSense->param_deviceProjectorTemparature); + //intrinsicGuiGroup.add(realSense->param_deviceLaser); + //intrinsicGuiGroup.add(realSense->param_deviceLaser_mag); + //intrinsicGuiGroup.add(realSense->param_deviceAutoExposure); + //intrinsicGuiGroup.add(realSense->param_deviceExposure_mag); + //intrinsicGuiGroup.add(realSense->param_deviceGain_mag); + //intrinsicGuiGroup.add(realSense->param_deviceFrameQueSize_mag); + //intrinsicGuiGroup.add(realSense->param_deviceAsicTemparature); + //intrinsicGuiGroup.add(realSense->param_deviceProjectorTemparature); device->addGroup(intrinsicGuiGroup); - device->loadFromFile(realSense->getSerialNumber(-1) + ".xml"); + device->loadFromFile("device.xml"); } //-------------------------------------------------------------- void ofApp::setupViewports(){ + //ofLog(OF_LOG_NOTICE) << "Entered setupViewports"; //call here whenever we resize the window - device->setWidth(MENU_WIDTH / 4); - post->setWidth(MENU_WIDTH / 4); + //post->setWidth(MENU_WIDTH / 4); setupCalib->setWidth(MENU_WIDTH / 4); - skeletonFinder.panel->setWidth(MENU_WIDTH / 4); + //skeletonFinder.panel->setWidth(MENU_WIDTH / 4); networkMng.panel->setWidth(MENU_WIDTH / 4); - operating->setWidth(MENU_WIDTH / 4); + //operating->setWidth(MENU_WIDTH / 4); device->setPosition(ofGetWidth() - MENU_WIDTH, 20); - post->setPosition(ofGetWidth() - MENU_WIDTH, 400); - operating->setPosition(ofGetWidth() - MENU_WIDTH, 800); + //post->setPosition(ofGetWidth() - MENU_WIDTH, 400); + //operating->setPosition(ofGetWidth() - MENU_WIDTH, 800); setupCalib->setPosition(ofGetWidth() - MENU_WIDTH / 4 * 3, 20); - skeletonFinder.panel->setPosition(ofGetWidth() - MENU_WIDTH / 4 * 2, 20); + //skeletonFinder.panel->setPosition(ofGetWidth() - MENU_WIDTH / 4 * 2, 20); networkMng.panel->setPosition(ofGetWidth() - MENU_WIDTH / 4, 20); - //ofLog(OF_LOG_NOTICE, "ofGetWidth()" + ofToString(ofGetWidth())); - - - // - //-- } -void ofApp::updateFrustumCone(int & value){ - if(realSense != NULL && realSense->isRunning()){ - double ref_pix_size = 1;// kinect.getZeroPlanePixelSize(); - double ref_distance = 1;// kinect.getZeroPlaneDistance(); +void ofApp::updateFrustumCone(int& val){ + //ofLog(OF_LOG_NOTICE) << "Entered updateFrustumCone"; + double ref_pix_size = 1;// kinect.getZeroPlanePixelSize(); + double ref_distance = 1;// kinect.getZeroPlaneDistance(); - realSenseFrustum.near1 = nearFrustum.get(); - realSenseFrustum.far1 = farFrustum.get(); + realSenseFrustum.near1 = nearFrustum.get(); + realSenseFrustum.far1 = farFrustum.get(); - double factorNear = 2 * ref_pix_size * realSenseFrustum.near1 / ref_distance; - double factorFar = 2 * ref_pix_size * realSenseFrustum.far1 / ref_distance; + double factorNear = 2 * ref_pix_size * realSenseFrustum.near1 / ref_distance; + double factorFar = 2 * ref_pix_size * realSenseFrustum.far1 / ref_distance; - //ofVec3f((x - DEPTH_X_RES/2) *factor, (y - DEPTH_Y_RES/2) *factor, raw[y * w + x])); + //ofVec3f((x - DEPTH_X_RES/2) *factor, (y - DEPTH_Y_RES/2) *factor, raw[y * w + x])); - realSenseFrustum.left = (0 - DEPTH_X_RES/2) *factorNear; - realSenseFrustum.right = (640 - DEPTH_X_RES/2) *factorNear; - realSenseFrustum.top = (0 - DEPTH_Y_RES/2) *factorNear; - realSenseFrustum.bottom = (480 - DEPTH_Y_RES/2) *factorNear; + realSenseFrustum.left = (0 - DEPTH_X_RES/2) *factorNear; + realSenseFrustum.right = (640 - DEPTH_X_RES/2) *factorNear; + realSenseFrustum.top = (0 - DEPTH_Y_RES/2) *factorNear; + realSenseFrustum.bottom = (480 - DEPTH_Y_RES/2) *factorNear; - realSenseFrustum.leftFar = (0 - DEPTH_X_RES/2) *factorFar; - realSenseFrustum.rightFar = (640 - DEPTH_X_RES/2) *factorFar; - realSenseFrustum.topFar = (0 - DEPTH_Y_RES/2) *factorFar; - realSenseFrustum.bottomFar = (480 - DEPTH_Y_RES/2) *factorFar; + realSenseFrustum.leftFar = (0 - DEPTH_X_RES/2) *factorFar; + realSenseFrustum.rightFar = (640 - DEPTH_X_RES/2) *factorFar; + realSenseFrustum.topFar = (0 - DEPTH_Y_RES/2) *factorFar; + realSenseFrustum.bottomFar = (480 - DEPTH_Y_RES/2) *factorFar; - realSenseFrustum.update(); - //createFrustumCone(); - } - + realSenseFrustum.update(); + //createFrustumCone(); } void ofApp::measurementCycleRaw(){ + //ofLog(OF_LOG_NOTICE) << "Entered measurementCycleRaw"; if(cycleCounter < N_MEASURMENT_CYCLES){ planePoint1Meas[cycleCounter] = calcPlanePoint(calibPoint_X, 0, 1); planePoint2Meas[cycleCounter] = calcPlanePoint(calibPoint_Y, 0, 1); @@ -340,6 +350,7 @@ void ofApp::measurementCycleRaw(){ } void ofApp::measurementCycleFine(){ + //ofLog(OF_LOG_NOTICE) << "Entered measurementCycleFine"; if(cycleCounter < N_MEASURMENT_CYCLES){ ofVec3f p1meas = calcPlanePoint(calibPoint_X, 0, 1); ofVec3f p2meas = calcPlanePoint(calibPoint_Y, 0, 1); @@ -375,6 +386,7 @@ void ofApp::measurementCycleFine(){ //-------------------------------------------------------------- void ofApp::updateCalc(){ + //ofLog(OF_LOG_NOTICE) << "Entered updateCalc"; // This algorithm calculates the transformation matrix to // transform from the camera centered coordinate system to the @@ -465,6 +477,7 @@ void ofApp::updateCalc(){ //-------------------------------------------------------------- void ofApp::updateMatrix(){ + //ofLog(OF_LOG_NOTICE) << "Entered updateMatrix"; sphere_X.setPosition(planePoint_X); sphere_Y.setPosition(planePoint_Y); @@ -483,10 +496,12 @@ void ofApp::updateMatrix(){ //-------------------------------------------------------------- glm::vec3 ofApp::calcPlanePoint(ofParameter & cpoint, int _size, int _step){ + //ofLog(OF_LOG_NOTICE) << "Entered calcPlanePoint"; glm::vec3 ppoint; - int width = realSense->getDepthWidth(); - int height = realSense->getDepthHeight(); + // TODO: these values are in nuitrack.config + int width = 640;// realSense->getDepthWidth(); + int height = 480;// realSense->getDepthHeight(); int size = _size; int step = _step; @@ -504,7 +519,7 @@ glm::vec3 ofApp::calcPlanePoint(ofParameter & cpoint, int _size, int _s for(int y = minY; y <= maxY; y = y + step) { for(int x = minX; x <= maxX; x = x + step) { - coord = realSense->getSpacePointFromInfraLeftFrameCoord(glm::vec2(x, y)); + coord = Tracker::fromVector3(tracker->depthTracker->convertProjToRealCoords(nuitrack::Vector3(x, y, 0))); if(coord.z > 0) { ppoint += coord; counter++; @@ -520,54 +535,34 @@ glm::vec3 ofApp::calcPlanePoint(ofParameter & cpoint, int _size, int _s //-------------------------------------------------------------- void ofApp::update(){ - + tracker->poll(); ofBackground(100, 100, 100); - - // there is a new frame and we are connected - if(realSense->update(ofxRealSenseTwo::PointCloud::INFRALEFT)) { - if(bUpdateMeasurment){ - measurementCycleRaw(); - } - if(bUpdateMeasurmentFine){ - measurementCycleFine(); - } - - if (bUpdateImageMask) { - //skeletonFinder.captureMaskBegin(); - drawCapturePointCloud(true); - //skeletonFinder.captureMaskEnd(); - } - else { - ////////////////////////////////// - // Cature captureCloud to FBO - ////////////////////////////////// + if(bUpdateMeasurment){ + measurementCycleRaw(); + } - //skeletonFinder.captureBegin(); - drawCapturePointCloud(false); - //skeletonFinder.captureEnd(); + if(bUpdateMeasurmentFine){ + measurementCycleFine(); + } - ////////////////////////////////// - // BlobFinding on the captured FBO - ///////////////////////////////////// - //skeletonFinder.update(); + // TODO: remove useless (?) + drawCapturePointCloud(true); - networkMng.update(skeletonFinder, realSenseFrustum, transformation.get()); - } - - } + networkMng.update(skeletonFinder, realSenseFrustum, transformation.get()); } //-------------------------------------------------------------- void ofApp::draw(){ + //ofLog(OF_LOG_NOTICE) << "Entered draw"; ofSetColor(255, 255, 255); //ofLogNotice() << "draw next frame"; if(bShowVisuals){ //Draw viewport previews - realSense->drawDepthStream(viewGrid[0]); - realSense->drawInfraLeftStream(viewGrid[1]); + // TODO: basic rgb & depth streams + //skeletonFinder.grayImage.draw(viewGrid[2]); //skeletonFinder.contourFinder.draw(viewGrid[3]); @@ -576,11 +571,13 @@ void ofApp::draw(){ switch (iMainCamera) { case 0: - realSense->drawDepthStream(viewMain); + // TODO + //realSense->drawDepthStream(viewMain); drawCalibrationPoints(); break; case 1: - realSense->drawInfraLeftStream(viewMain); + // TODO + //realSense->drawInfraLeftStream(viewMain); drawCalibrationPoints(); break; case 2: @@ -660,6 +657,7 @@ void ofApp::draw(){ } void ofApp::drawPreview() { + //ofLog(OF_LOG_NOTICE) << "Entered drawPreview"; glPointSize(4); glEnable(GL_DEPTH_TEST); @@ -669,7 +667,8 @@ void ofApp::drawPreview() { //ofTranslate(-planeCenterPoint.x, -planeCenterPoint.y, 0); ofMultMatrix(deviceTransform); if (bPreviewPointCloud) { - realSense->draw(); + // TODO: what does this do + pointCloudManager.draw(); } ofFill(); ofSetColor(255, 0, 0); @@ -693,6 +692,7 @@ void ofApp::drawPreview() { skeletonFinder.drawSensorBox(); ofNoFill(); + glLineWidth(5); ofSetColor(255, 100, 255); skeletonFinder.drawSkeletons(); //skeletonFinder.drawBodyBlobsBox(); @@ -701,7 +701,6 @@ void ofApp::drawPreview() { ofFill(); ofSetColor(255, 100, 100); //skeletonFinder.drawGazePoint(); - glDisable(GL_DEPTH_TEST); ofPopMatrix(); @@ -709,6 +708,7 @@ void ofApp::drawPreview() { } void ofApp::drawCapturePointCloud(bool _mask) { + //ofLog(OF_LOG_NOTICE) << "Entered drawCapturePointCloud"; glEnable(GL_DEPTH_TEST); shader.begin(); @@ -731,7 +731,8 @@ void ofApp::drawCapturePointCloud(bool _mask) { ofPushMatrix(); ofMultMatrix(deviceTransform); - realSense->draw(); + //realSense->draw(); + pointCloudManager.draw(); ofPopMatrix(); shader.end(); @@ -741,6 +742,7 @@ void ofApp::drawCapturePointCloud(bool _mask) { } void ofApp::drawCalibrationPoints(){ + //ofLog(OF_LOG_NOTICE) << "Entered drawCalibrationPoints"; glDisable(GL_DEPTH_TEST); ofPushStyle(); ofSetColor(255, 0, 0); @@ -758,9 +760,7 @@ void ofApp::drawCalibrationPoints(){ //-------------------------------------------------------------- void ofApp::exit() { ofLog(OF_LOG_NOTICE, "exiting application..."); - - realSense->stop(); - + // Nuitrack auto-releases on destroy ... } void ofApp::createHelp(){ @@ -775,7 +775,6 @@ void ofApp::createHelp(){ help += "press x|y|z and then mouse-click -> to change the calibration points in viewport 1\n"; help += "press k -> to update the calculation\n"; help += "press r -> to show calculation results \n"; - help += "press t -> to terminate the connection, connection is: " + ofToString(realSense->isRunning()) + "\n"; help += "press o -> to open the connection again\n"; help += "ATTENTION: Setup-Settings (ServerID and Video) will only apply after restart\n"; help += "Broadcasting ip: "+networkMng.broadcastIP.get()+" port: "+ofToString(networkMng.broadcastPort.get())+" serverID: "+ofToString(networkMng.mServerID)+" \n"; @@ -823,7 +822,7 @@ void ofApp::keyPressed(int key){ //skeletonFinder.saveMask(); networkMng.panel->saveToFile("broadcast.xml"); post->saveToFile("postprocessing.xml"); - device->saveToFile(realSense->getSerialNumber(-1) + ".xml"); + device->saveToFile("device.xml"); guitransform->saveToFile("transformation.xml"); break; @@ -833,7 +832,7 @@ void ofApp::keyPressed(int key){ //skeletonFinder.loadMask(); networkMng.panel->loadFromFile("broadcast.xml"); post->loadFromFile("postprocessing.xml"); - device->loadFromFile(realSense->getSerialNumber(-1) + ".xml"); + device->loadFromFile("device.xml"); guitransform->loadFromFile("transformation.xml"); break; diff --git a/src/ofApp.h b/src/ofApp.h index 15b6e3a59951abe3aa98a3799bfb868e88d8184d..4cca18333be9c05290ea9e416a1f3c2e38a9816c 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -10,9 +10,10 @@ #include "TrackingNetworkManager.h" #include "Frustum.h" #include "CaptureMeshArray.h" +#include "PointCloudManager.h" -#include "ofxRealSenseTwo.h" -#include // Include RealSense Cross Platform API +#include "ofxNuitrack.h" +#include #include @@ -31,7 +32,7 @@ #define N_MEASURMENT_CYCLES 10 using namespace std; -using namespace ofxRealSenseTwo; +using namespace ofxnui; //helpfull links during development: // https://github.com/openframeworks/openFrameworks/issues/3817 @@ -39,7 +40,6 @@ using namespace ofxRealSenseTwo; class ofApp : public ofBaseApp{ public: - //ofApp() {} void setup(); void update(); @@ -78,6 +78,7 @@ class ofApp : public ofBaseApp{ ////////////////// //viewports + void initViewports(); void setupViewports(); ofRectangle viewMain; @@ -98,14 +99,13 @@ class ofApp : public ofBaseApp{ ///////////// //RealSense// ///////////// + + void initNuitrack(); - RSDevicePtr realSense; + TrackerRef tracker; + PointCloudManager pointCloudManager; ofMatrix4x4 unprojection; - - #ifdef USE_TWO_KINECTS - ofxKinect kinect2; - #endif bool dispRaw; @@ -129,7 +129,7 @@ class ofApp : public ofBaseApp{ void createGUIDeviceParams(); void createFrustumCone(); - void updateFrustumCone(int & value); + void updateFrustumCone(int& val); ///////////////// //COLOR CONTOUR// @@ -145,6 +145,7 @@ class ofApp : public ofBaseApp{ /////////////// //CALCULATION// /////////////// + void updateCalc(); void updateMatrix(); void measurementCycleRaw(); @@ -189,6 +190,9 @@ class ofApp : public ofBaseApp{ ////////////// //PROPERTIES// ////////////// + void setupCalibGui(); + void setupTransformGui(); + ofxGui gui; ofxGuiPanel *setupCalib; diff --git a/the-hole.vcxproj b/the-hole.vcxproj index 78cde14cb41c56640272626914c8def8ed95a219..1048fe1932bd7032fb402a211140147668e1aeb8 100644 --- a/the-hole.vcxproj +++ b/the-hole.vcxproj @@ -193,6 +193,7 @@ + @@ -260,6 +261,7 @@ + diff --git a/the-hole.vcxproj.filters b/the-hole.vcxproj.filters index 58b93adfe502836c8310d895e9670fce66450edb..7a9f11b59c0f216566f279f8b812553cb8bd6ef8 100644 --- a/the-hole.vcxproj.filters +++ b/the-hole.vcxproj.filters @@ -205,6 +205,9 @@ src + + src + @@ -1811,6 +1814,9 @@ src + + src +