HandTrackingMessage.h

Go to the documentation of this file.

#pragma once

#include <string>
#include <vector>
#include <array>
#include <iosfwd>
#include <cmath>
#include "HandTrackingClient/VecMath.h"
#include "HandTrackingClient/OpenCVCamera.h"











namespace HandTrackingClient
{

const static int N_JOINTS = 17;

const static int N_FINGERS = 5;

const static int N_HANDS = 2;

const static int N_POSES = 7;

const static int N_FINGER_DOFS_PER_HAND = 16;

enum JointFrameIndex
{
    ROOT_JOINT = 0,           
    WRIST_JOINT = 1,          
    THUMB_PROXIMAL = 2,       
    THUMB_INTERMEDIATE = 3,   
    THUMB_DISTAL = 4,         
    INDEX_PROXIMAL = 5,       
    INDEX_INTERMEDIATE = 6,   
    INDEX_DISTAL = 7,         
    MIDDLE_PROXIMAL = 8,      
    MIDDLE_INTERMEDIATE = 9,  
    MIDDLE_DISTAL = 10,       
    RING_PROXIMAL = 11,       
    RING_INTERMEDIATE = 12,   
    RING_DISTAL = 13,         
    PINKY_PROXIMAL = 14,      
    PINKY_INTERMEDIATE = 15,  
    PINKY_DISTAL = 16,        
};

typedef JointFrameIndex JointIndex;

enum FingerDOF
{
    THUMB_CMC_AA  = 0,     
    THUMB_CMC_FE  = 1,     
    THUMB_MCP     = 2,     
    THUMB_IP      = 3,     
    INDEX_MCP_AA  = 4,     
    INDEX_MCP_FE  = 5,     
    INDEX_PIP     = 6,     
    MIDDLE_MCP_AA = 7,     
    MIDDLE_MCP_FE = 8,     
    MIDDLE_PIP    = 9,     
    RING_MCP_AA   = 10,    
    RING_MCP_FE   = 11,    
    RING_PIP      = 12,    
    PINKY_MCP_AA  = 13,    
    PINKY_MCP_FE  = 14,    
    PINKY_PIP     = 15     
};

class HandState
{
public:
    HandState(const Vector3f& position, 
              const Quaternionf& rotation, 
              const int clickCount) 
        : _position(position), 
          _rotation(rotation),
          _clickCount(clickCount) { }

    HandState() { }

    const Vector3f& getPosition() const { return _position; }

    const Quaternionf& getRotation() const { return _rotation; }

    const int getClickCount() const { return _clickCount; }

private:
    Vector3f _position;
    Quaternionf _rotation;
    int _clickCount;
};

enum Hand
{
    LEFT_HAND = 0,        
    RIGHT_HAND = 1,       
    BOTH_HANDS = 2,       
    INVALID_HAND = 10000  
};

const char* handToString(Hand hand);

Hand stringToHand(const std::string& str);


class ParseException
    : public std::exception
{
public:
    ParseException (const char* message)
        : _msg (message) {}
    virtual ~ParseException() throw() {}
    
    virtual const char* what() const throw() { return _msg; }

private:
    const char* _msg;
};

class HandTrackingMessage
{
public:
    enum MessageType  
    {
        WELCOME = 0,                    
        USER = 100,                     
        CALIBRATION = 101,              
        POSE = 200,                     
        PRESSED = 300,                  
        DRAGGED = 301,                  
        RELEASED = 302,                 
        MOVED = 303,                    
        SIMULTANEOUSLY_PRESSED = 400,   
        INDIVIDUALLY_PRESSED = 401,     
        SIMULTANEOUSLY_RELEASED = 402,  
        INDIVIDUALLY_RELEASED = 403,    
        DRAGGED_BIMANUAL = 404,         
        POINT = 500,                    
        INVALID_DATA = 10000            
    };

    static const char* messageTypeToString(MessageType m);

    static MessageType stringToMessageType(const std::string& str);

    static HandTrackingMessage* deserialize(const std::string& data);

    virtual MessageType getType() const = 0 ;

    virtual std::string serialize() const = 0;

    virtual ~HandTrackingMessage();
protected:
    HandTrackingMessage() { }
};

class BasicMessage : public HandTrackingMessage
{
public:
    MessageType getType() const { return _type; }

    const HandState& getHandState(int hand) const { return _hands[hand]; }

    BasicMessage(const MessageType type, 
                 const Vector3f& positionLeft,
                 const Quaternionf& rotationLeft,
                 const int clickCountLeft,
                 const Vector3f& positionRight,
                 const Quaternionf& rotationRight,
                 const int clickCountRight);
    virtual ~BasicMessage();

    std::string serialize() const;

private:
    MessageType _type;
    std::array<HandState, N_HANDS> _hands;
};

class PinchMessage : public BasicMessage
{
public:
    PinchMessage(const MessageType type, 
                 const Hand hand,
                 const Vector3f& positionLeft,
                 const Quaternionf& rotationLeft,
                 const int clickCountLeft,
                 const Vector3f& positionRight,
                 const Quaternionf& rotationRight,
                 const int clickCountRight);

    const Hand getHand() const { return _hand; }

    std::string serialize() const;

private:
    Hand _hand;
};

class BimanualPinchMessage : public BasicMessage
{
public:
    BimanualPinchMessage(const MessageType type, 
                         const Hand hand,
                         const Vector3f& positionLeft,
                         const Quaternionf& rotationLeft,
                         const int clickCountLeft,
                         const Vector3f& positionRight,
                         const Quaternionf& rotationRight,
                         const int clickCountRight);

    const Hand getHand() const { return _hand; }

    std::string serialize() const;

private:
    Hand _hand;
};

class PointMessage : public HandTrackingMessage
{
public:
    PointMessage(const Hand hand,
                 const Vector3f& pointStart,
                 const Vector3f& pointEnd,
                 const float confidence);

    const Hand getHand() const { return _hand; }

    std::string serialize() const;

    virtual MessageType getType() const { return HandTrackingMessage::POINT; }

    Vector3f getPointStart() const { return _pointStart; }

    Vector3f getPointEnd() const { return _pointEnd; }

    Vector3f getPointDir() const { return (_pointEnd - _pointStart).normalized(); }

    float getConfidence() const  { return _confidence; }

private:
    Hand _hand;
    Vector3f _pointStart;
    Vector3f _pointEnd;
    float _confidence;
};

class PoseMessage : public BasicMessage
{

public:
    enum HandPose
    {
        HAND_CURLED = 0,       
        HAND_ELL = 1,          
        HAND_OKAY = 2,         
        HAND_PINCH = 3,        
        HAND_POINTING = 4,     
        HAND_RELAXEDOPEN = 5,  
        HAND_SPREAD = 6        
    };

    PoseMessage(const Vector3f& positionLeft,
                const Quaternionf& rotationLeft,
                const int clickCountLeft,
                const Vector3f& positionRight,
                const Quaternionf& rotationRight,
                const int clickCountRight,
                const std::array<float, N_HANDS>& confidenceEstimates,
                const std::array<std::array<Quaternionf, N_JOINTS>, N_HANDS>& jointRotations,
                const std::array<std::array<Vector3f, N_JOINTS>, N_HANDS>& jointTranslations,
                const std::array<std::array<float, N_FINGER_DOFS_PER_HAND>, N_HANDS>& fingerDOFs,
                const std::array<std::array<Vector3f, N_FINGERS>, N_HANDS>& fingerTips,
                const std::array<std::array<float, N_POSES>, N_HANDS>& handPoseConfidences);

    std::string serialize() const;

    float getConfidenceEstimate(size_t hand) const { return _confidenceEstimates[hand]; }

    std::array<Matrix4f, N_JOINTS> getJointFrames(size_t hand) const;

    std::array<Transformf, N_JOINTS> getJointTransforms(size_t hand) const;

    const std::array<float, N_FINGER_DOFS_PER_HAND>& getFingerDOFs(size_t hand) const { return _fingerDOFs[hand]; }

    const std::array<Vector3f, N_FINGERS>& getFingerTips(size_t hand) const { return _fingerTips[hand]; }

    const std::array<float, N_POSES>& getHandPoseConfidences(size_t hand) const { return _handPoseConfidences[hand]; }

private:
    std::array<float, N_HANDS> _confidenceEstimates;
    std::array<std::array<Quaternionf, N_JOINTS>, N_HANDS> _jointRotations;
    std::array<std::array<Vector3f, N_JOINTS>, N_HANDS> _jointTranslations;
    std::array<std::array<float, N_FINGER_DOFS_PER_HAND>, N_HANDS> _fingerDOFs;
    std::array<std::array<Vector3f, N_FINGERS>, N_HANDS> _fingerTips;
    std::array<std::array<float, N_POSES>, N_HANDS> _handPoseConfidences;
};


class WelcomeMessage : public HandTrackingMessage
{
public:
    WelcomeMessage (const std::string& serverVersion, 
                    const std::string& protocolVersion,
                    const std::vector<OpenCVCamera>& cameras);

    MessageType getType() const { return HandTrackingMessage::WELCOME; }

    const std::string getServerVersion() const { return _serverVersion; }

    const std::string getProtocolVersion() const { return _protocolVersion; }

    std::string serialize() const;

    const std::vector<OpenCVCamera>& getCameras() const { return _cameras; }

private:
    std::string _serverVersion;
    std::string _protocolVersion;

    std::vector<OpenCVCamera> _cameras;
};

class UserMessage : public HandTrackingMessage 
{
public:
    typedef std::array<int,3> Triangle;

    // A list of bone influence indices (for a single vertex).
    typedef std::vector<int> IndicesVector;

    // A list of bone influence weights (for a single vertex).
    typedef std::vector<float> WeightsVector;

    const std::string& getUserProfileName() const { return _userProfileName; };

    const std::vector<Vector3f>& getRestPositions(int hand) const { return _restPositions[hand]; } 

    const std::vector<Triangle>& getTriangles(int hand) const { return _triangles[hand]; }

    const std::vector<IndicesVector>& getSkinningIndices(int hand) const { return _skinningIndices[hand]; }

    const std::vector<WeightsVector>& getSkinningWeights(int hand) const { return _skinningWeights[hand]; }

    std::array<Matrix4f, N_JOINTS> getRestJointFrames(int hand) const;

    std::array<Transformf, N_JOINTS> getRestJointTransforms(int hand) const; 

    UserMessage(const std::string& userProfileName, 
                const std::array<std::vector<Vector3f>, N_HANDS>& restPositions,
                const std::array<std::vector<Triangle>, N_HANDS>& triangles,
                const std::array<std::vector<IndicesVector>, N_HANDS>& skinningIndices,
                const std::array<std::vector<WeightsVector>, N_HANDS>& skinningWeights,
                const std::array<std::array<Quaternionf, N_JOINTS>, N_HANDS>& restJointRotations,
                const std::array<std::array<Vector3f, N_JOINTS>, N_HANDS>& restJointTranslations);

    MessageType getType() const { return HandTrackingMessage::USER; }

    std::string serialize() const;

private:
    std::string _userProfileName;
    std::array<std::vector<Vector3f>, N_HANDS> _restPositions;
    std::array<std::vector<Triangle>, N_HANDS> _triangles;
    std::array<std::vector<IndicesVector>, N_HANDS> _skinningIndices;
    std::array<std::vector<WeightsVector>, N_HANDS> _skinningWeights;
    std::array<std::array<Quaternionf, N_JOINTS>, N_HANDS> _restJointRotations;
    std::array<std::array<Vector3f, N_JOINTS>, N_HANDS> _restJointTranslations;
};

class CalibrationMessage : public HandTrackingMessage 
{
public:
    CalibrationMessage(const float percentComplete,
                       const float currentScale) : _percentComplete(percentComplete),
                                                   _currentScale(currentScale) {}

    float getPercentComplete() const { return _percentComplete; }

    float getCurrentScale() const { return _currentScale; }

    MessageType getType() const { return HandTrackingMessage::CALIBRATION; }

    std::string serialize() const;

private:
    float _percentComplete;
    float _currentScale;
};

} // namespace HandTrackingClient