update to use QVGA (320x240) for transmission to camera-display app on ESP32-S3-BOX. Updated camera-streamer code and python test code to multicast to the group so that the camera-streamer app can automatically find the server.

Author: finger563

display_frames.py

@@ -1,67 +1,110 @@
 import argparse
 import cv2
+import math
 import socket
 from turbojpeg import TurboJPEG
 import time
+import threading
+import re
+
+quitting = False
+got_connection = False
+
+def publish_mc_group():
+    global quitting
+    global got_connection
+    MCAST_GRP = '239.1.1.1'
+    MCAST_PORT = 5000
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
+    sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32)
+    while not quitting and not got_connection:
+        try:
+            sock.sendto(b'Hello World!', (MCAST_GRP, MCAST_PORT))
+            time.sleep(1)
+        except:
+            break
+    print("Multicast thread exiting")
 
 def main(host="0.0.0.0", port=8888):
+    global quitting
+    global got_connection
     header = b'\xaa\xbb\xcc\xdd'
     jpeg = TurboJPEG()
+    t = threading.Thread(target=publish_mc_group, args=[])
+    t.start()
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         s.bind((host, port))
         print("server lisening on {}:{}".format(host,port))
         s.listen()
         conn, addr = s.accept()
         with conn:
             print(f"Connected by {addr}")
+            got_connection = True
             total_bytes = 0
             total_frames = 0
             start = None
             while True:
                 # try to get all chunks of the message
+                # print("Trying to get new image")
                 img_data = bytearray()
-                max_recv_size = 40*1024
+                max_recv_size = 1*1024
                 remaining = max_recv_size
                 got_header = False
                 while True:
                     recv_len = remaining if remaining < max_recv_size else max_recv_size
                     # print("receiving", recv_len, "bytes")
-                    data = conn.recv(recv_len)
-                    if not data:
-                        print("breaking loop because (not data) == true")
-                        break
+                    try:
+                        data = conn.recv(recv_len)
+                    except KeyboardInterrupt:
+                        print("quitting...")
+                        quitting = True
+                        t.join()
+                        return
                     if not start:
                         start = time.time()
-                    if data[0:4] == header:
-                        length = (int(data[4]) << 24) + (int(data[5]) << 16) + (int(data[6]) << 8) + int(data[7])
-                        remaining = length - len(data) + 8
-                        img_data.extend(data[8:])
-                        got_header = True
-                        # print("Got header, image length:", length)
-                        # print("remaining:", remaining)
+                    # find the header in the bytearray
+                    if not got_header:
+                        try:
+                            index = data.index(header)
+                            if index >= 0:
+                                got_header = True
+                                length = \
+                                    (int(data[index+4]) << 24) + \
+                                    (int(data[index+5]) << 16) + \
+                                    (int(data[index+6]) << 8) + \
+                                    int(data[index+7])
+                                data_start = index + 8
+                                data_bytes = len(data) - data_start
+                                num_img_bytes = min(length, data_bytes)
+                                remaining = length - num_img_bytes
+                                data_end = data_start + num_img_bytes + 1
+                                img_data.extend(data[data_start:data_end])
+                                # print("Got index:", index)
+                                # print("Got data_start:", data_start)
+                                # print("Got data_end:", data_end)
+                                # print("Got header, image length:", length)
+                                # print("remaining:", remaining)
+                        except:
+                            pass
                     elif got_header and remaining > 0:
-                        if remaining >= len(data):
-                            remaining = remaining - len(data)
-                            img_data.extend(data)
-                        else:
-                            print("got more data than we needed:", len(data), remaining)
-                            img_data.extend(data[:remaining])
-                            remaining = 0
-                    if got_header and remaining <= 0:
+                        num_img_bytes = min(remaining, len(data))
+                        img_data.extend(data[:num_img_bytes])
+                        remaining = remaining - num_img_bytes
+
+                    if got_header and remaining == 0:
                         break;
+
                 end = time.time()
                 elapsed = end - start
                 total_bytes += len(img_data)
                 total_frames += 1
-                print("got image data, len:", len(img_data))
+                # print("got image data, len:", len(img_data))
                 print("metrics: {:.0f} KB/s, {:.1f} FPS".format((total_bytes / 1024) / elapsed, total_frames / elapsed))
                 try:
                     img = jpeg.decode(img_data)
                     cv2.imshow('Remote Camera', img)
                     cv2.waitKey(1)
-                except Exception as e:
-                    # print("Could not open jpeg image:")
-                    # print(e)
+                except:
                     pass
 
 if __name__ == "__main__":

main/main.cpp

@@ -14,6 +14,7 @@
 #include "oneshot_adc.hpp"
 #include "task.hpp"
 #include "tcp_socket.hpp"
+#include "udp_socket.hpp"
 #include "wifi_sta.hpp"
 
 #include "esp_camera.h"
@@ -76,6 +77,36 @@ extern "C" void app_main(void) {
     logger.info("waiting for wifi connection...");
     std::this_thread::sleep_for(1s);
   }
+  // use UDP multicast to listen for where the server is
+  std::string multicast_group = "239.1.1.1";
+  size_t port = 5000;
+  espp::UdpSocket server_socket({.log_level=espp::Logger::Verbosity::WARN});
+  auto server_task_config = espp::Task::Config{
+    .name = "UdpServer",
+    .callback = nullptr,
+    .stack_size_bytes = 6 * 1024,
+  };
+  std::atomic<bool> found_receiver = false;
+  std::string receiver_ip;
+  auto server_config = espp::UdpSocket::ReceiveConfig{
+    .port = port,
+    .buffer_size = 1024,
+    .is_multicast_endpoint = true,
+    .multicast_group = multicast_group,
+    .on_receive_callback = [&found_receiver, &receiver_ip](auto& data, auto& source) -> auto {
+      fmt::print("Server received: {}\n"
+                 "    from source: {}:{}\n",
+                 data, source.address, source.port);
+      receiver_ip = source.address;
+      found_receiver = true;
+      return std::nullopt;
+    }
+  };
+  server_socket.start_receiving(server_task_config, server_config);
+  while (!found_receiver) {
+    logger.info("waiting for receiver to multicast their info over 239.1.1.1...");
+    std::this_thread::sleep_for(1s);
+  }
   // initialize camera
   /**
    * @note display sizes supported:
@@ -120,7 +151,7 @@ extern "C" void app_main(void) {
     .ledc_channel = LEDC_CHANNEL_0,
 
     .pixel_format = PIXFORMAT_JPEG,//YUV422,GRAYSCALE,RGB565,JPEG
-    .frame_size = FRAMESIZE_UXGA,// QVGA-UXGA, For ESP32, do not use sizes above QVGA when not JPEG. The performance of the ESP32-S series has improved a lot, but JPEG mode always gives better frame rates.
+    .frame_size = FRAMESIZE_QVGA,// QVGA-UXGA, For ESP32, do not use sizes above QVGA when not JPEG. The performance of the ESP32-S series has improved a lot, but JPEG mode always gives better frame rates.
 
     .jpeg_quality = 15, //0-63, for OV series camera sensors, lower number means higher quality
     .fb_count = 2, //When jpeg mode is used, if fb_count more than one, the driver will work in continuous mode.
@@ -269,16 +300,15 @@ extern "C" void app_main(void) {
   // make the tcp_client to transmit to the network
   std::atomic<int> num_frames_transmitted{0};
   std::atomic<float> transmission_elapsed{0};
-  auto transmit_task_fn = [&transmit_queue, &num_frames_transmitted, &transmission_elapsed, &logger](auto& m, auto& cv) {
-    static std::string ip_address = "192.168.1.23";
+  auto transmit_task_fn = [&receiver_ip, &transmit_queue, &num_frames_transmitted, &transmission_elapsed, &logger](auto& m, auto& cv) {
     static size_t port = 8888;
     static Image image;
     static auto start = std::chrono::high_resolution_clock::now();
     static auto tcp_client = std::make_shared<espp::TcpSocket>(espp::TcpSocket::Config{});
     // wait on the queue until we have an image ready to transmit
     if (xQueueReceive(transmit_queue, &image, portMAX_DELAY) == pdPASS) {
       if (!tcp_client->is_connected()) {
-        if (!tcp_client->connect({.ip_address = ip_address, .port = port})) {
+        if (!tcp_client->connect({.ip_address = receiver_ip, .port = port})) {
           // destroy the socket and try again on the next go-around?
           tcp_client.reset();
           tcp_client = std::make_shared<espp::TcpSocket>(espp::TcpSocket::Config{});