From 2aafa06920a0002778b1e13f2a3060f83f70cb59 Mon Sep 17 00:00:00 2001 From: Corrie Van Sice <41913149+CorrieVS@users.noreply.github.com> Date: Mon, 16 Aug 2021 23:29:04 -0500 Subject: [PATCH 1/7] Added Compression Boolean Function Added compression boolean funciton to check if the incoming data is compressed. Also adjusted the compressed image getter function to return a constructor/init list. --- Private/RobofleetBPFunctionLibrary.cpp | 9 +++++++++ Private/RobofleetClientBase.cpp | 27 +++++++++++++++++--------- Private/RobofleetClientBase.h | 3 +++ Public/RobofleetBPFunctionLibrary.h | 3 +++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Private/RobofleetBPFunctionLibrary.cpp b/Private/RobofleetBPFunctionLibrary.cpp index af92845..9ff486c 100644 --- a/Private/RobofleetBPFunctionLibrary.cpp +++ b/Private/RobofleetBPFunctionLibrary.cpp @@ -74,6 +74,15 @@ TArray URobofleetBPFunctionLibrary::GetRobotImage(const FString& RobotNam return TArray(); } +bool URobofleetBPFunctionLibrary::IsRobotImageCompressed(const FString& RobotName) +{ + if (FRobofleetUnrealClientModule::Get()->IsSessionRunning()) + { + return FRobofleetUnrealClientModule::Get()->RobofleetClient->IsRobotImageCompressed(RobotName); + } + return bool(); +} + void URobofleetBPFunctionLibrary::PrintRobotsSeen() { if (FRobofleetUnrealClientModule::Get()->IsSessionRunning()) diff --git a/Private/RobofleetClientBase.cpp b/Private/RobofleetClientBase.cpp index f4d4e9d..3bef2b0 100644 --- a/Private/RobofleetClientBase.cpp +++ b/Private/RobofleetClientBase.cpp @@ -1,5 +1,6 @@ #include "RobofleetClientBase.h" #include "GameFramework/Actor.h" +#include URobofleetBase::URobofleetBase() @@ -46,6 +47,7 @@ void URobofleetBase::Initialize(FString HostUrl, const UObject* WorldContextObje RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); + RegisterRobotSubscription("imagecompressed", "*"); bIsInitilized = true; } @@ -148,6 +150,7 @@ void URobofleetBase::RefreshRobotList() UE_LOG(LogRobofleet, Log, TEXT("Refreshing robot list")); RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); + RegisterRobotSubscription("imagecompressed", "*"); //PruneInactiveRobots(); } } @@ -175,9 +178,7 @@ void URobofleetBase::DecodeMsg(const void* Data, FString topic, FString RobotNam RobotMap[RobotNamespace]->Location = rl; } - else if (topic == "image_raw/compressed") { - //call function to convert msg to bitmap - //return bitmap + else if (topic == "imagecompressed") { RobotImageMap[RobotNamespace] = DecodeMsg(Data); OnImageReceived.Broadcast(RobotNamespace); } @@ -221,15 +222,23 @@ FVector URobofleetBase::GetRobotPosition(const FString& RobotName) TArray URobofleetBase::GetRobotImage(const FString& RobotName) { - //needs to return type that Texture expects FString RobotNamestd = FString(TCHAR_TO_UTF8(*RobotName)); - TArray imageData; - imageData.Append(&RobotImageMap[RobotNamestd].data[0], RobotImageMap[RobotNamestd].data.size()); - // you may want an TArray - // FColor pixelColor = {0, &RobotImageMap[Name].data[i] : i+3} - return imageData; + //returns the constructor/ init list: TArray(arrayPtr, arraySize) + return TArray(&RobotImageMap[RobotNamestd].data[0], RobotImageMap[RobotNamestd].data.size()); } +bool URobofleetBase::IsRobotImageCompressed(const FString& RobotName) +{ + FString RobotNamestd = FString(TCHAR_TO_UTF8(*RobotName)); + if (RobotImageMap[RobotNamestd].format.find("compressed") != std::string::npos) + { + return true; + } + else return false; + +} + + TArray URobofleetBase::GetAllRobotsAtSite(const FString& Location) { TArray RobotsAtSite; diff --git a/Private/RobofleetClientBase.h b/Private/RobofleetClientBase.h index e6be084..a0a019e 100644 --- a/Private/RobofleetClientBase.h +++ b/Private/RobofleetClientBase.h @@ -21,6 +21,7 @@ struct RobotData { RobotStatus Status; bool IsAlive; }; + //Define Log Category and Verbosity DECLARE_LOG_CATEGORY_EXTERN(LogRobofleet, Log, All); @@ -106,6 +107,8 @@ class ROBOFLEETUNREALCLIENT_API URobofleetBase : public UObject TArray GetRobotImage(const FString& RobotName); + bool IsRobotImageCompressed(const FString& RobotName); + TArray GetAllRobotsAtSite(const FString& Location); bool IsRobotOk(const FString& RobotName); diff --git a/Public/RobofleetBPFunctionLibrary.h b/Public/RobofleetBPFunctionLibrary.h index bc3d7a3..a04caf7 100644 --- a/Public/RobofleetBPFunctionLibrary.h +++ b/Public/RobofleetBPFunctionLibrary.h @@ -40,6 +40,9 @@ class ROBOFLEETUNREALCLIENT_API URobofleetBPFunctionLibrary : public UBlueprintF UFUNCTION(BlueprintCallable, Category = "Robofleet") static TArray GetRobotImage(const FString& RobotName); + UFUNCTION(BlueprintCallable, Category = "Robofleet") + static bool IsRobotImageCompressed(const FString& RobotName); + UFUNCTION(BlueprintCallable, Category = "Robofleet") static void PrintRobotsSeen(); From eceee53f93a640a8e5c53cae2c5d25ccfa244820 Mon Sep 17 00:00:00 2001 From: zotoli Date: Thu, 19 Aug 2021 23:19:03 +0100 Subject: [PATCH 2/7] Adds buffering functionality for longer websocket messages. --- Private/RobofleetClientBase.cpp | 11 ++++----- Private/WebsocketClient.cpp | 42 +++++++++++++++++++++++++++++---- Private/WebsocketClient.h | 7 ++++++ Private/robofleet_client_lib | 2 +- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/Private/RobofleetClientBase.cpp b/Private/RobofleetClientBase.cpp index 3bef2b0..c6fbe53 100644 --- a/Private/RobofleetClientBase.cpp +++ b/Private/RobofleetClientBase.cpp @@ -28,6 +28,7 @@ bool URobofleetBase::IsConnected() { if (IsValid(SocketClient)) { + //UE_LOG(LogRobofleet, Warning, TEXT("Websocket is Valid")) return SocketClient->Socket->IsConnected(); } return false; @@ -48,6 +49,7 @@ void URobofleetBase::Initialize(FString HostUrl, const UObject* WorldContextObje RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); RegisterRobotSubscription("imagecompressed", "*"); + UE_LOG(LogRobofleet, Log, TEXT("RobofleetBase initialized")); bIsInitilized = true; } @@ -108,11 +110,12 @@ void URobofleetBase::PruneInactiveRobots() { void URobofleetBase::WebsocketDataCB(const void* Data) { const fb::MsgWithMetadata* msg = flatbuffers::GetRoot(Data); + std::string MsgTopic = msg->__metadata()->topic()->c_str(); int NamespaceIndex = MsgTopic.substr(1, MsgTopic.length()).find('/'); FString RobotNamespace = FString(MsgTopic.substr(1, NamespaceIndex).c_str()); - FString TopicIsolated = FString(MsgTopic.substr(NamespaceIndex+2, MsgTopic.length()).c_str()); + FString TopicIsolated = FString(MsgTopic.substr(NamespaceIndex + 2, MsgTopic.length()).c_str()); RobotsSeenTime[RobotNamespace] = FDateTime::Now(); @@ -124,10 +127,6 @@ void URobofleetBase::WebsocketDataCB(const void* Data) RobotsSeen.insert(RobotNamespace); DecodeMsg(Data, TopicIsolated, RobotNamespace); - - if (Verbosity) - PrintRobotsSeen(); - } void URobofleetBase::PrintRobotsSeen() { @@ -150,7 +149,7 @@ void URobofleetBase::RefreshRobotList() UE_LOG(LogRobofleet, Log, TEXT("Refreshing robot list")); RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); - RegisterRobotSubscription("imagecompressed", "*"); + //RegisterRobotSubscription("imagecompressed", "*"); //PruneInactiveRobots(); } } diff --git a/Private/WebsocketClient.cpp b/Private/WebsocketClient.cpp index c499a94..5d0c85a 100644 --- a/Private/WebsocketClient.cpp +++ b/Private/WebsocketClient.cpp @@ -29,6 +29,7 @@ void UWebsocketClient::Initialize(FString ServerURL /*= TEXT("ws://localhost:808 Socket->OnConnected().AddUFunction(this, FName("OnConnected")); Socket->OnConnectionError().AddUFunction(this, FName("OnConnectionError")); Socket->OnMessageSent().AddUFunction(this, FName("OnMessageSent")); + Socket->OnClosed().AddUFunction(this, FName("OnClosed")); // Unreal having a problem with binding to a raw function, and a UFUNCTION doesn't like having a void* as a arg // For now, use lambda directly @@ -55,6 +56,11 @@ void UWebsocketClient::OnConnected() UE_LOG(LogTemp, Log, TEXT("Connected to websocket.")) } +void UWebsocketClient::OnClosed() +{ + UE_LOG(LogTemp, Log, TEXT("Websocket Closed.")) +} + void UWebsocketClient::OnConnectionError() { UE_LOG(LogTemp, Warning, TEXT("Encountered error while trying to connect to websocket.")) @@ -62,11 +68,38 @@ void UWebsocketClient::OnConnectionError() void UWebsocketClient::OnMessageReceived(const void* Data, SIZE_T Size, SIZE_T BytesRemaining) { - UE_LOG(LogTemp, Verbose, TEXT("Message Received")); - - if (callbackRegistered) { - OnReceivedCB(Data); + // TODO: fix it frank, actually its not too terrible atm. + + if (BytesRemaining) + { + if (!bIsBuffering) + { + DataBuffer = new char[Size + BytesRemaining]; + memcpy(DataBuffer, Data, Size); + PrevSize = Size; + bIsBuffering = true; + } + else + { + memcpy(DataBuffer+PrevSize, Data, Size); + PrevSize = Size; + } + } + else + { + if (bIsBuffering) + { + memcpy(DataBuffer + PrevSize, Data, Size); + PrevSize = 0; + bIsBuffering = false; + OnReceivedCB(DataBuffer); + delete DataBuffer; + } + else + { + OnReceivedCB(Data); + } } } @@ -77,7 +110,6 @@ void UWebsocketClient::OnMessageSent() void UWebsocketClient::Send(const void* ptr, uint32_t size, bool isBinary) { - UE_LOG(LogTemp, Verbose, TEXT("Message Sending")); Socket->Send(ptr, size, isBinary); } diff --git a/Private/WebsocketClient.h b/Private/WebsocketClient.h index e0e8549..33ac7ee 100644 --- a/Private/WebsocketClient.h +++ b/Private/WebsocketClient.h @@ -36,6 +36,9 @@ class UWebsocketClient : public UObject UFUNCTION() void OnConnected(); + + UFUNCTION() + void OnClosed(); UFUNCTION() void OnConnectionError(); @@ -53,4 +56,8 @@ class UWebsocketClient : public UObject void IsCallbackRegistered(bool val); + // TODO: Make a buffer struct and clean up the raw pointer, optimization wise the current implementation should be close to fastest but can be organized a bit better and made safer. + char* DataBuffer; + int PrevSize; + bool bIsBuffering = false; }; diff --git a/Private/robofleet_client_lib b/Private/robofleet_client_lib index 2bf2baa..8616ad3 160000 --- a/Private/robofleet_client_lib +++ b/Private/robofleet_client_lib @@ -1 +1 @@ -Subproject commit 2bf2baaefe6ce25e231263497de7bbaa310948c3 +Subproject commit 8616ad3412f5a8eb2e5f63d777e9c49a5ea3b51e From 5401a6f516f7c448e6ac2698a47b5fc31c94add7 Mon Sep 17 00:00:00 2001 From: Corrie Van Sice <41913149+CorrieVS@users.noreply.github.com> Date: Tue, 14 Sep 2021 15:09:53 -0500 Subject: [PATCH 3/7] updated image topic names --- Private/RobofleetClientBase.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Private/RobofleetClientBase.cpp b/Private/RobofleetClientBase.cpp index c6fbe53..aac314a 100644 --- a/Private/RobofleetClientBase.cpp +++ b/Private/RobofleetClientBase.cpp @@ -48,7 +48,7 @@ void URobofleetBase::Initialize(FString HostUrl, const UObject* WorldContextObje RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); - RegisterRobotSubscription("imagecompressed", "*"); + RegisterRobotSubscription("image_compressed/main", "*"); UE_LOG(LogRobofleet, Log, TEXT("RobofleetBase initialized")); bIsInitilized = true; } @@ -149,7 +149,7 @@ void URobofleetBase::RefreshRobotList() UE_LOG(LogRobofleet, Log, TEXT("Refreshing robot list")); RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); - //RegisterRobotSubscription("imagecompressed", "*"); + RegisterRobotSubscription("image_compressed/main", "*"); //PruneInactiveRobots(); } } @@ -177,7 +177,7 @@ void URobofleetBase::DecodeMsg(const void* Data, FString topic, FString RobotNam RobotMap[RobotNamespace]->Location = rl; } - else if (topic == "imagecompressed") { + else if (topic == "image_compressed/main") { RobotImageMap[RobotNamespace] = DecodeMsg(Data); OnImageReceived.Broadcast(RobotNamespace); } From 794752e3aad4db41aa4caba46dd3d313fed0254f Mon Sep 17 00:00:00 2001 From: Corrie Van Sice <41913149+CorrieVS@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:54:23 -0600 Subject: [PATCH 4/7] added error logging for image stream --- Private/RobofleetClientBase.cpp | 7 +++++++ Private/WebsocketClient.cpp | 2 +- RobofleetUnrealClient.Build.cs | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Private/RobofleetClientBase.cpp b/Private/RobofleetClientBase.cpp index aac314a..5d3a524 100644 --- a/Private/RobofleetClientBase.cpp +++ b/Private/RobofleetClientBase.cpp @@ -177,6 +177,12 @@ void URobofleetBase::DecodeMsg(const void* Data, FString topic, FString RobotNam RobotMap[RobotNamespace]->Location = rl; } + // Missing flatbuffer definitions + //else if (topic == "image/main") { + // RobotImageMap[RobotNamespace] = DecodeMsg(Data); + // OnImageReceived.Broadcast(RobotNamespace); + //} + else if (topic == "image_compressed/main") { RobotImageMap[RobotNamespace] = DecodeMsg(Data); OnImageReceived.Broadcast(RobotNamespace); @@ -223,6 +229,7 @@ TArray URobofleetBase::GetRobotImage(const FString& RobotName) { FString RobotNamestd = FString(TCHAR_TO_UTF8(*RobotName)); //returns the constructor/ init list: TArray(arrayPtr, arraySize) + UE_LOG(LogTemp, Log, TEXT("Creating local image message of type TArray.")); return TArray(&RobotImageMap[RobotNamestd].data[0], RobotImageMap[RobotNamestd].data.size()); } diff --git a/Private/WebsocketClient.cpp b/Private/WebsocketClient.cpp index 5d0c85a..3d906a7 100644 --- a/Private/WebsocketClient.cpp +++ b/Private/WebsocketClient.cpp @@ -83,7 +83,7 @@ void UWebsocketClient::OnMessageReceived(const void* Data, SIZE_T Size, SIZE_T B else { memcpy(DataBuffer+PrevSize, Data, Size); - PrevSize = Size; + PrevSize += Size; } } else diff --git a/RobofleetUnrealClient.Build.cs b/RobofleetUnrealClient.Build.cs index f7c94cb..995cf6b 100644 --- a/RobofleetUnrealClient.Build.cs +++ b/RobofleetUnrealClient.Build.cs @@ -19,7 +19,7 @@ public RobofleetUnrealClient(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; - PublicDependencyModuleNames.AddRange(new string[] {"Core", "CoreUObject", "Engine", "WebSockets" }); + PublicDependencyModuleNames.AddRange(new string[] {"Core", "CoreUObject", "Engine", "WebSockets", "ImageWrapper" }); PublicIncludePaths.Add(Path.Combine(ModuleDirectory, "Private/robofleet_client_lib/include")); PublicIncludePaths.Add(Path.Combine(ModuleDirectory, "Private/robofleet_client_lib")); PublicIncludePaths.Add(Path.Combine(ModuleDirectory, "Private")); From c204845061985a7363745b24ee5fb7ea2da15bc5 Mon Sep 17 00:00:00 2001 From: CorrieVS Date: Thu, 10 Mar 2022 21:10:12 -0600 Subject: [PATCH 5/7] changed name of compressed image topic --- Private/RobofleetClientBase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Private/RobofleetClientBase.cpp b/Private/RobofleetClientBase.cpp index 5d3a524..86dce38 100644 --- a/Private/RobofleetClientBase.cpp +++ b/Private/RobofleetClientBase.cpp @@ -149,7 +149,7 @@ void URobofleetBase::RefreshRobotList() UE_LOG(LogRobofleet, Log, TEXT("Refreshing robot list")); RegisterRobotStatusSubscription(); RegisterRobotSubscription("localization", "*"); - RegisterRobotSubscription("image_compressed/main", "*"); + RegisterRobotSubscription("image_raw/compressed", "*"); //PruneInactiveRobots(); } } @@ -183,7 +183,7 @@ void URobofleetBase::DecodeMsg(const void* Data, FString topic, FString RobotNam // OnImageReceived.Broadcast(RobotNamespace); //} - else if (topic == "image_compressed/main") { + else if (topic == "image_raw/compressed") { RobotImageMap[RobotNamespace] = DecodeMsg(Data); OnImageReceived.Broadcast(RobotNamespace); } From 98cae39e02507549224a8224698f19fff1cac7d6 Mon Sep 17 00:00:00 2001 From: CorrieVS Date: Thu, 10 Mar 2022 22:54:31 -0600 Subject: [PATCH 6/7] added GetDetectedImage function --- Public/RobofleetBPFunctionLibrary.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Public/RobofleetBPFunctionLibrary.h b/Public/RobofleetBPFunctionLibrary.h index b6cb12a..767720a 100644 --- a/Public/RobofleetBPFunctionLibrary.h +++ b/Public/RobofleetBPFunctionLibrary.h @@ -65,6 +65,9 @@ class ROBOFLEETUNREALCLIENT_API URobofleetBPFunctionLibrary : public UBlueprintF UFUNCTION(BlueprintCallable, Category = "Robofleet") static FVector GetDetectedPositionGlobal(const FString& RobotName); + UFUNCTION(BlueprintCallable, Category = "Robofleet") + static TArray GetDetectedImage(const FString& RobotName); + // Publish Messages to Robofleet UFUNCTION(BlueprintCallable, Category = "Robofleet") static void PublishStatusMsg(const FString& RobotName, const FRobotStatus& StatusMsg); From 6fe51de01b7ed33615b7c22c7b18ae3d9bfaeef3 Mon Sep 17 00:00:00 2001 From: Corrie Van Sice <41913149+CorrieVS@users.noreply.github.com> Date: Fri, 11 Mar 2022 16:12:46 -0600 Subject: [PATCH 7/7] switched robofleet_client_lib to devel_1.0.1 --- Private/robofleet_client_lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Private/robofleet_client_lib b/Private/robofleet_client_lib index 8616ad3..5f74030 160000 --- a/Private/robofleet_client_lib +++ b/Private/robofleet_client_lib @@ -1 +1 @@ -Subproject commit 8616ad3412f5a8eb2e5f63d777e9c49a5ea3b51e +Subproject commit 5f74030966af053bb6fb897de61e877f18386422