Merge pull request #3119 from JdeRobot/new_digit_classifier_exercise

[New Exercise] New Digit Classification Exercise using Deep Learning

Author: javizqh

database/exercises/db.sql

@@ -100,7 +100,12 @@ COPY public.exercises (id, exercise_id, name, description, tags, status, templat
 14	laser_mapping	Laser Mapping	Build a map based on sensor readings	{"tags": "ROS2"}	ACTIVE	RoboticsAcademy/exercises/static/exercises/laser_mapping/python_template/
 15	basic_computer_vision	Basic Computer Vision	Basic Computer Vision exercise using React and RAM	{"tags": "ROS2"}	ACTIVE	RoboticsAcademy/exercises/static/exercises/basic_computer_vision/python_template/
 16	follow_road	Follow Road	Follow Road exercise	{"tags": "ROS2"}	ACTIVE	RoboticsAcademy/exercises/static/exercises/follow_road/python_template/
+<<<<<<< new_digit_classifier_exercise
+17	pick_place	Pick and Place	Pick and Place exercise	{"tags": "ROS2"}	PROTOTYPE	RoboticsAcademy/exercises/static/exercises/pick_place/python_template/
+18	digit_classification	Digit Classification	Deep learning-based Digit Classification.	{"tags": ["Computer Vision","Deep Learning","Classification"]}	PROTOTYPE	RoboticsAcademy/exercises/static/exercises/digit_classification/python_template/
+=======
 17	pick_place	Pick and Place	Pick and Place exercise	{"tags": "ROS2"}	ACTIVE	RoboticsAcademy/exercises/static/exercises/pick_place/python_template/
+>>>>>>> humble-devel
 \.
 -- 16	rescue_people_classic	Rescue People Old	Rescue People exercise With Gazebo Classic	{"tags": "ROS2"}	ACTIVE	RoboticsAcademy/exercises/static/exercises/rescue_people_classic/python_template/
 

exercises/static/exercises/assets/img/digit_classification_teaser.png

@@ -8,10 +8,7 @@
 import numpy as np
 
 # Graphical User Interface Class
-
-
 class GUI(MeasuringThreadingGUI):
-
     def __init__(self, host="ws://127.0.0.1:2303"):
         super().__init__(host)
 

exercises/static/exercises/basic_computer_vision/python_template/ros2_humble/GUI.py

@@ -0,0 +1,128 @@
+import json
+import cv2
+import base64
+import threading
+
+from gui_interfaces.general.measuring_threading_gui_no_sim import MeasuringThreadingGUI
+from console_interfaces.general.console import start_console
+import numpy as np
+
+# Graphical User Interface Class
+class GUI(MeasuringThreadingGUI):
+    def __init__(self, host="ws://127.0.0.1:2303"):
+        super().__init__(host)
+
+        # Execution control vars
+        self.image_to_be_shown = None
+        self.image_to_be_shown_updated = False
+        self.image_show_lock = threading.Lock()
+
+        self.payload = {"image": "", "shape": ""}
+        self.frame_rgb = None
+        self.frame_rgb_lock = threading.Lock()
+
+        self.has_received_img = False
+
+        self.start()
+
+    # Process incoming messages to the GUI
+    def gui_in_thread(self, ws, message):
+        # time frame size
+        time_frame_size = 20
+
+        if "ack" in message:
+            with self.ack_lock:
+                self.ack = True
+                self.ack_frontend = True
+
+        if "pick" in message:
+            self.has_received_img = True
+
+            # Image from the frontend
+            base64_buffer = message[4:-time_frame_size]
+            time = message[-time_frame_size:]
+
+            if base64_buffer.startswith("data:image/jpeg;base64,"):
+                base64_buffer = base64_buffer[len("data:image/jpeg;base64,") :]
+
+            # Decodificar la cadena base64 a bytes
+            image_data = base64.b64decode(base64_buffer)
+
+            # Convertir los bytes a un array de numpy
+            nparr = np.frombuffer(image_data, np.uint8)
+
+            # Decodificar la imagen (convertirla a formato OpenCV)
+            img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+
+            with self.frame_rgb_lock:
+                self.frame_rgb = img
+                ack_message = {"ack_img": "ack", "time": time}
+                self.send_to_client(json.dumps(ack_message))
+
+        if "introspection" in message:
+            info = message[len("introspection:") :]
+            self.fps, self.lat = info.split("/")
+
+    # Prepares and sends a map to the websocket server
+    def update_gui(self):
+
+        payload = self.payloadImage()
+        self.payload["image"] = json.dumps(payload)
+
+        message = json.dumps(self.payload)
+        self.send_to_client(message)
+
+        if not self.has_received_img:
+            ack_message = {"ack_img": "ack", "time": ""}
+            self.send_to_client(json.dumps(ack_message))
+
+    # Function to prepare image payload
+    # Encodes the image as a JSON string and sends through the WS
+    def payloadImage(self):
+        with self.image_show_lock:
+            image_to_be_shown_updated = self.image_to_be_shown_updated
+            image_to_be_shown = self.image_to_be_shown
+
+        image = image_to_be_shown
+        payload = {"image": "", "shape": ""}
+
+        if not image_to_be_shown_updated:
+            return payload
+
+        shape = image.shape
+        frame = cv2.imencode(".JPEG", image)[1]
+        encoded_image = base64.b64encode(frame)
+
+        payload["image"] = encoded_image.decode("utf-8")
+        payload["shape"] = shape
+
+        with self.image_show_lock:
+            self.image_to_be_shown_updated = False
+
+        return payload
+
+    # Function for student to call
+    def showImage(self, image):
+        with self.image_show_lock:
+            self.image_to_be_shown = image
+            self.image_to_be_shown_updated = True
+
+    def getImage(self):
+        with self.frame_rgb_lock:
+            return self.frame_rgb
+
+
+host = "ws://127.0.0.1:2303"
+gui = GUI(host)
+
+# Redirect the console
+start_console()
+
+
+# Expose the user functions
+def showImage(image):
+    gui.showImage(image)
+
+
+def getImage():
+    return gui.getImage()

exercises/static/exercises/digit_classification/python_template/ros2_humble/GUI.py

@@ -0,0 +1,7 @@
+model_dir_path = "/workspace/code/model.onnx"
+
+# This file is part of the Human Detection ROS2 package.
+def model_path_func()-> str:
+    return model_dir_path
+
+model_path = model_path_func()

exercises/static/exercises/dl_digit_classifier_react/python_template/ros2_humble/shared/__init__.py renamed to exercises/static/exercises/digit_classification/python_template/ros2_humble/HAL.py

@@ -0,0 +1,14 @@
+import * as React from "react";
+import {Fragment} from "react";
+
+import "./css/BasicComputerVisionRR.css";
+
+const BasicComputerVisionRR = (props) => {
+  return (
+    <Fragment>
+    {props.children}
+    </Fragment>
+  );
+};
+
+export default BasicComputerVisionRR;

exercises/static/exercises/digit_classification/python_template/ros2_humble/model.py

@@ -0,0 +1,56 @@
+import * as React from "react";
+import noImage from "../../assets/img/noImage.png";
+
+import "./css/GUICanvas.css"
+
+function SpecificBasicComputerVision(props) {
+  React.useEffect(() => {
+    console.log("TestShowScreen subscribing to ['update'] events");
+    const callback = (message) => {
+
+      if (message.data.update.image) {
+        var canvas = document.getElementById("gui_canvas");
+
+        // Request Animation Frame to remove the flickers
+        function decode_utf8(s) {
+            return decodeURIComponent(escape(s))
+        }
+
+        // Parse the Image Data
+        var image_data = JSON.parse(message.data.update.image),
+          source = decode_utf8(image_data.image),
+          shape = image_data.shape;
+
+        if (source != "" && shape instanceof Array) {
+          canvas.src = "data:image/jpeg;base64," + source;
+          canvas.width = shape[1];
+          canvas.height = shape[0];
+        }
+      }
+
+      // Send the ACK of the msg
+      window.RoboticsExerciseComponents.commsManager.send("gui", "ack");
+    };
+
+    window.RoboticsExerciseComponents.commsManager.subscribe(
+      [window.RoboticsExerciseComponents.commsManager.events.UPDATE],
+      callback
+    );
+
+    return () => {
+      console.log("TestShowScreen unsubscribing from ['state-changed'] events");
+      window.RoboticsExerciseComponents.commsManager.unsubscribe(
+        [window.RoboticsExerciseComponents.commsManager.events.UPDATE],
+        callback
+      );
+    };
+  }, []);
+
+  return (
+    <div style={{display: "flex", width: "100%", height: "100%", position:"relative", justifyContent: "center"}}>
+      <img className="image" id="gui_canvas" src={noImage}/>
+    </div>
+  );
+}
+
+export default SpecificBasicComputerVision;

exercises/static/exercises/digit_classification/react-components/BasicComputerVisionRR.js

@@ -0,0 +1,28 @@
+* {
+    box-sizing: border-box;
+}
+
+body, html {
+    width: 100%; height: 100%;
+}
+
+#exercise-container {
+    position: absolute;
+    top: 0; left: 0; bottom: 0; right: 0;
+    overflow: hidden;
+    display: flex;
+    flex-direction: column;
+}
+
+#exercise-container #content {
+    width: 100%;
+    height: 100%;
+    overflow: hidden;
+}
+
+#exercise-container #content #content-exercise {
+    display: flex;
+    flex-direction: column;
+    width: 100%;
+    height: 100%;
+}

exercises/static/exercises/digit_classification/react-components/SpecificBasicComputerVision.js

@@ -0,0 +1,5 @@
+.image {
+  position: absolute;
+  width: fit-content;
+  height: 100%;
+}