Developing Natural User

Interface Applications With

RealSenseTM DevicesPete O’Hanlon @peteohanlon

Evolution of Perceptual Computing

Allows computer to perceive depth in a similar way we do

Uses 2D and 3D depth camera

Allows us to interact with computers in a completely new way

Different types of camera

Front facing camera

Track hands and fingers

Capture face movements

Detect and differentiate between foreground and background

Rear facing camera

Scan, measure rooms and objects

Snapshot camera

The SDK

Runs on Windows 8.1 as well as in the browser and Android

Available for different languages

C++

C#

Java

JavaScript

Processing (OS framework for developing visual oriented apps – Java like)

Runs on popular game engines

Unity supported

What you need

Hardware 4th generation Intel® Core™ Haswell

Minimum 8GB free hard disk

USB 3 port

RealSenseTM 3D Camera

Software Windows 8.1 64 bit

Visual Studio 2010+

.NET 4+

Unity PRO 4.1.0+

Web Browser

Processing 2.1.2+

JDK 1.7.0_11+

IR CameraFull HD 1080p Image Sensor

IR Laser Projector

SDK Based Application

Language specific interfaces

SDK interfaces

SDK Core I/O Algorithm Algorithm

Natural User Interfaces

Framework features

Emotion detection

Face tracking

Facial recognition

Hand tracking

Gesture recognition

Object tracking

Speech recognition

Speech synthesis

Touchless controller

Segmentation

The potential

Touch free kiosks

Games

Interactive books

Collaboration

Enhanced tutoring

Many more…

DevelopingSetting up

Initialisation

senseManager = PXCMSenseManager.CreateInstance();

private PXCMSenseManager senseManager;private PXCMHandModule handModule;private PXCMHandConfiguration handConfiguration;private PXCMSpeechSynthesis speechSynthesis;private PXCMAudio.AudioInfo audioInfo;private PXCMTouchlessController touchlessController;private PXCMFaceModule faceModule;private PXCMFaceConfiguration faceConfiguration;

if (status >= pxcmStatus.PXCM_STATUS_NO_ERROR){PXCMCapture.Device device =senseManager.captureManager.device;

device.SetDepthConfidenceThreshold(10);device.SetIVCAMLaserPower(16);device.SetIVCAMAccuracy(PXCMCapture.Device.IVCAMAccuracy.IVCAM_ACCURACY_COARSE);device.SetIVCAMMotionRangeTradeOff(21);device.SetIVCAMFilterOption(6);

if (senseManager.StreamFrames(false) >= pxcmStatus.PXCM_STATUS_NO_ERROR)

{// We are streaming data,,,

}}

Emotion detection

senseManager.EnableEmotion();

private pxcmStatus ProcessFrame(…{switch (moduleId){case PXCMEmotion.CUID:ProcessEmotionData(module);break;

}return pxcmStatus.PXCM_STATUS_NO_ERROR;

}

private void ProcessEmotionData(PXCMBase module)

{PXCMEmotion emotion = module.QueryInstance<PXCMEmotion>();

int numberOfFaces = emotion.QueryNumFaces();for (int face = 0; face < numberOfFaces;

face++){PXCMEmotion.EmotionData[] emotionData =

new PXCMEmotion.EmotionData[10];emotion.QueryAllEmotionData(face,

out emotionData);FindStrongestEmotion(emotionData);

}}

private void FindStrongestEmotion(PXCMEmotion.EmotionData[] emotionData)

{PXCMEmotion.EmotionData strongestEmotion = emotionData[0];

int index = 0;for (int emotionIndex = 1; emotionIndex < 7;emotionIndex++){if (emotionData[emotionIndex].evidence >

strongestEmotion.evidence){

index = emotionIndex;strongestEmotion = emotionData[emotionIndex];

}}

}

Segmentation

senseManager.Enable3DSeg();

private pxcmStatus ProcessFrame(int moduleId, PXCMBase module, PXCMCapture.Sample captureSample)

{switch (moduleId){case PXCM3DSeg.CUID:ProcessSegmentData(module);break;

}return pxcmStatus.PXCM_STATUS_NO_ERROR;

}

private void ProcessSegmentData(PXCMBase module)

{PXCM3DSeg segment = module.QueryInstance<PXCM3DSeg>();

if (segment != null){using (PXCMImage image =

segment.AcquireSegmentedImage()){

if (image != null){

ExtractForeground (image);}

}}

}

private void ExtractForeground(PXCMImage image){PXCMImage.ImageData segmentImage;

image.AcquireAccess(PXCMImage.Access.ACCESS_READ_WRITE,PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32,out segmentImage);

try{ProcessImage(image, segmentImage);

}finally{image.ReleaseAccess(segmentImage);

}WriteableBitmap bmp = segmentImage.ToWritableBitmap(0, image.info.width, image.info.height, 96, 96);

}

PXCMImage.ImageInfo info = image.QueryInfo();

for (int y = 0; y < height; y++){unsafe{byte* pixel = (byte*)segmentImage.planes[0] + y * segmentImage.pitches[0];

for (int x = 0; x < width; x++){if (pixel[3] == 0){

for (int ch = 0; ch < 3; ch++){pixel[ch] = (byte)0;

}}pixel += 4;

}}

}

HandsTouchless Control

senseManager.EnableTouchlessController(null);

touchlessController = senseManager.QueryTouchlessController();touchlessController.SubscribeEvent(new PXCMTouchlessController.OnFiredUXEventDelegate(OnTouchlessControllerUXEvent));

PXCMTouchlessController.ProfileInfoprofileInfo;touchlessController.QueryProfile(outprofileInfo);profileInfo.config = PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Scroll_Horizontally;

touchlessController.SetProfile(profileInfo);

private void OnTouchlessControllerUXEvent(PXCMTouchlessController.UXEventData data)

{switch (data.type){case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_StartScroll:break;

case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Scroll:break;

case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Select:break;

}}

Speech Synthesis

private void SetupSpeechSynthesis(){if (senseManager.session.CreateImpl<PXCMSpeechSynthesis>(out speechSynthesis) >= pxcmStatus.PXCM_STATUS_NO_ERROR)

{PXCMSpeechSynthesis.ProfileInfo profileInfo;if (speechSynthesis.QueryProfile(0,

out profileInfo) >= pxcmStatus.PXCM_STATUS_NO_ERROR)

{profileInfo.voice =

PXCMSpeechSynthesis.VoiceType.VOICE_ANY;

audioInfo = profileInfo.outputs;speechSynthesis

.SetProfile(profileInfo);}

}}

public void Say(string sentence){if (speechSynthesis != null){speechSynthesis.BuildSentence(1, sentence);int buffer =

speechSynthesis.QueryBufferNum(1);VoiceOut voiceOut =

new VoiceOut(audioInfo);for (int i = 0; i < buffer; i++){

PXCMAudio audio = speechSynthesis.QueryBuffer(1, i);

voiceOut.RenderAudio(audio);}voiceOut.Close();speechSynthesis.ReleaseSentence(1);

}}

Face tracking

senseManager.EnableFace();

PXCMFaceModule faceModule = senseManager.QueryFace();PXCMFaceConfiguration config = faceModule.CreateActiveConfiguration();

config.detection.isEnabled = true;config.detection.maxTrackedFaces = 2;config.landmarks.isEnabled = true;config.ApplyChanges();

private pxcmStatus ProcessFrame(int moduleId,PXCMBase module,PXCMCapture.Sample captureSample)

{switch (moduleId){case PXCMFaceModule.CUID:

ProcessFace(module);break;

}return pxcmStatus.PXCM_STATUS_NO_ERROR;

}

private void ProcessFace(PXCMBase module){using (PXCMFaceModule faceModule =

module.QueryInstance<PXCMFaceModule>()){using (PXCMFaceData faceData = faceModule.CreateOutput()){

int faceCount = faceData.QueryNumberOfDetectedFaces();

for (int faceIndex = 0; faceIndex < faceCount;faceIndex++)

{PXCMFaceData.Face face = faceData.QueryFaceByIndex(faceIndex);

GetFaceBounds(face);GetLandmarkData(face);

}}

}}

private static void GetLandmarkData(PXCMFaceData.Face face)

{PXCMFaceData.LandmarkPoint[] landmarks;PXCMFaceData.LandmarksData landmarkData =face.QueryLandmarks();

landmarkData.QueryPoints(out landmarks);}

private static void GetFaceBounds(PXCMFaceData.Face face)

{PXCMFaceData.DetectionData detectionData =face.QueryDetection();

PXCMRectI32 boundingRect;detectionData.QueryBoundingRect(out boundingRect);

}

Starting the loop

PXCMSenseManager.Handler handler = newPXCMSenseManager.Handler{onModuleProcessedFrame = ProcessFrame

};

private pxcmStatus ProcessFrame(int moduleId, PXCMBase module, PXCMCapture.Sample captureSample)

{return pxcmStatus.PXCM_STATUS_NO_ERROR;

}

pxcmStatus status = senseManager.Init(handler);

What next?

Order your camera

http://click.intel.com/intel-realsense-developer-kit.html

Download the SDK

https://software.intel.com/en-us/realsense/intel-realsense-sdk-for-windows

Code