Hello, I am working on extending the DReyeVR driving simulator by integrating an external open-source eye tracker, name Pupil-Lab Core Glasses. This eye-tracker uses ZeroMQ PUB-SUB and MsgPack to share the data to its subscribers. I am interested in “surface” topic of the eye tracker whose message looks like (see this for details):
{
"topic": "surfaces.surface_name",
"name": "surface_name",
"surf_to_img_trans": (
(-394.2704714040225, 62.996680859974035, 833.0782341017057),
(24.939461954010476, 264.1698344383364, 171.09768247735033),
(-0.0031580300961504023, 0.07378146751738948, 1.0),
),
"img_to_surf_trans": (
(-0.002552357406770253, 1.5534025217146223e-05, 2.1236555655143734),
(0.00025853538051076233, 0.003973842600569134, -0.8952954577358644),
(-2.71355412859636e-05, -0.00029314688183396006, 1.0727627809231568),
),
"gaze_on_surfaces": (
{
"topic": "gaze.3d.1._on_surface",
"norm_pos": (-0.6709809899330139, 0.41052111983299255),
"confidence": 0.5594810076623645,
"on_surf": False,
"base_data": ("gaze.3d.1.", 714040.132285),
"timestamp": 714040.132285,
},
...,
),
# list of fixations associated with
"fixations_on_surfaces": (
{
"topic": "fixations_on_surface",
"norm_pos": (-0.9006409049034119, 0.7738968133926392),
"confidence": 0.8663407531808505,
"on_surf": False,
"base_data": ("fixations", 714039.771958),
"timestamp": 714039.771958,
"id": 27,
"duration": 306.62299995310605, # in milliseconds
"dispersion": 1.4730711610581475, # in degrees
},
...,
),
# timestamp of the world video frame in which the surface was detected
"timestamp": 714040.103912,
}
In my Project, I have implemented two methods, EstablishEyeTrackerConnection() and GetSurfaceData() which are responsible for establishing a connection and retriving the data, as shown below:
bool AEgoVehicle::EstablishEyeTrackerConnection() {
try {
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Attempting to establish connection"));
// Prepare our context and subscriber
Context = new zmq::context_t(1);
std::string Address = "127.0.0.1";
std::string RequestPort = "50020";
zmq::socket_t Requester(*Context, ZMQ_REQ);
Requester.connect("tcp://" + Address + ":" + RequestPort);
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Connected to the TCP port"));
// Get the SUBSRIBE port to connect for communication
std::string RequestString = "SUB_PORT";
zmq::message_t Request(RequestString.begin(), RequestString.end());
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Sending request to get SUB PORT"));
Requester.send(Request);
zmq::message_t Reply;
Requester.recv(&Reply);
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Received SUB PORT"));
std::string SubscribePort = std::string(static_cast<char*>(Reply.data()), Reply.size());
// Setup the Subscriber socket
Subscriber = new zmq::socket_t(*Context, ZMQ_SUB);
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Connecting to the SUB PORT"));
Subscriber->connect("tcp://" + Address + ":" + SubscribePort);
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Connection successful"));
Subscriber->setsockopt(ZMQ_SUBSCRIBE, "surface", 7);
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Subscribed to surface topic"));
}
catch (...) {
// Log a generic error message
UE_LOG(LogTemp, Error, TEXT("ZeroMQ: Failed to connect to the Pupil labs Network API"));
return false;
}
UE_LOG(LogTemp, Display, TEXT("ZeroMQ: Established connection to the Pupil labs Network API"));
return true;
}
FDcResult AEgoVehicle::GetSurfaceData() {
// Note: using raw C++ types in the following code as it does not interact with UE interface
// Establish connection if not already
if (Subscriber == nullptr) {
if (!EstablishEyeTrackerConnection()) {
UE_LOG(LogTemp, Error, TEXT("ZeroMQ: Failed to connect to the Pupil labs Network API"));
}
}
// Receive an update and update to a string
zmq::message_t Update;
if (!Subscriber->recv(&Update)) {
UE_LOG(LogTemp, Error, TEXT("ZeroMQ: Failed to receive update from subscriber"));
}
std::string Topic(static_cast<char*>(Update.data()), Update.size());
// Receive a message from the server
zmq::message_t Message;
if (!Subscriber->recv(&Message)) {
UE_LOG(LogTemp, Error, TEXT("ZeroMQ: Failed to receive message from subscriber"));
}
// Store the serialized data into a TArray
TArray<uint8> DataArray;
DataArray.Append(static_cast<uint8*>(Message.data()), Message.size());
// Create a destination variable and deserializer
FSurfaceData Destination;
FDcDeserializer Deserializer;
// Prepare context for this run
FDcPropertyDatum Datum(&Destination);
FDcMsgPackReader Reader(FDcBlobViewData::From(DataArray));
FDcPropertyWriter Writer(Datum);
FDcDeserializeContext Ctx;
Ctx.Reader = &Reader;
Ctx.Writer = &Writer;
Ctx.Deserializer = &Deserializer;
DC_TRY(Ctx.Prepare());
DC_TRY(Deserializer.Deserialize(Ctx));
return DcOk();
}
As shown in your example, I have implemented a few USTRUCTS to store the data:
// FFloatArray is a USTRUCT that contains an array of floats.
// This is used to represent the arrays of floats in the original data structure.
USTRUCT()
struct FFloatArray
{
GENERATED_BODY()
UPROPERTY()
TArray<float> data;
};
// FGazeOnSurface and FFixationsOnSurface are USTRUCTs that represent the individual elements
// in the gaze_on_surfaces and fixations_on_surfaces arrays, respectively.
USTRUCT()
struct FGazeOnSurface
{
GENERATED_BODY()
UPROPERTY()
FString topic;
UPROPERTY()
TArray<float> norm_pos;
UPROPERTY()
float confidence;
UPROPERTY()
bool on_surf;
UPROPERTY()
TArray<FString> base_data;
UPROPERTY()
float timestamp;
};
USTRUCT()
struct FFixationsOnSurface
{
GENERATED_BODY()
UPROPERTY()
FString topic;
UPROPERTY()
TArray<float> norm_pos;
UPROPERTY()
float confidence;
UPROPERTY()
bool on_surf;
UPROPERTY()
TArray<FString> base_data;
UPROPERTY()
float timestamp;
UPROPERTY()
float duration;
UPROPERTY()
float dispersion;
};
// FSurfaceData is a USTRUCT that contains all the data from the original data structure.
// It uses the other USTRUCTs to represent the nested arrays.
USTRUCT()
struct FSurfaceData
{
GENERATED_BODY()
UPROPERTY()
FString topic;
UPROPERTY()
FString name;
UPROPERTY()
TArray<FFloatArray> surf_to_img_trans;
UPROPERTY()
TArray<FFloatArray> img_to_surf_trans;
UPROPERTY()
TArray<FFloatArray> surf_to_dist_img_trans;
UPROPERTY()
TArray<FFloatArray> dist_img_to_surf_trans;
UPROPERTY()
TArray<FGazeOnSurface> gaze_on_surfaces;
UPROPERTY()
TArray<FFixationsOnSurface> fixations_on_surfaces;
UPROPERTY()
float timestamp;
};
The following implementation does not seem to work. Earlier, I tried to use DcEnv().FlushDiags() just below the deserialization call to get the errors, I got the following error in the logs, although I have been unable to replicate this error again.
[2023.07.06-23.44.16:714][140]DataConfig: Error: # DataConfig Error: No matching handler, Property: 'SurfaceData' 'ScriptStruct'
- [DcMsgPackReader] Last read: Int8
- [DcPropertyWriter] Writing property: (FSurfaceData)$root
- [DcSerializer]
### Properties: (1) :
- SurfaceData
### Objects: (0).
I am really lost. I have no clue what the problem is. Could you please help me out? I am just a beginner and would greatly appreciate your assistance. I have also attached the full logs from UE if it helps.
CarlaUE4.log
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent