Streaming MLX90640 IR Video as MJPEG over HTTP Web Server using ESP32

  arduino-esp32, c++, esp32, http, mjpeg

I am entirely inexperienced when it comes to HTTP or any web server code. I am trying to stream my MLX90640 video on a web server using an ESP32. Currently, I have the MLX90640 connected to the ESP32 and can print the pixels repeatedly in the Serial Monitor to show an ASCII video. I am now struggling with going from this, to seeing a video on my web server.

My MLX90640 is output into a 32×24 array of float’s. I then map this to 0-255 and use that as an input to an array of 16 bit camera colors. I presume I am having issues when trying to convert the uint16_t to whatever type of data is necessary to describe the jpeg frame.

I am currently seeing an all black screen with a small little white box in the center of the screen when I pull up my server on chrome. Below, I have attached, all the relevant functions to this portion of the project, but I will monitor and can post the rest of my code if need be!

I have read a lot of similar projects, but most of them use a different camera type, and I haven’t been able to replicate the functions that would be needed for the MLX90640. Any suggestions on what to try or read next would be greatly appreciated!

Thanks in advance.

float frame[32*24]; // buffer for full frame of temperatures

float MinTemp;
float MaxTemp;

const char HEADER[] = "HTTP/1.1 200 OKrn" 
                      "Access-Control-Allow-Origin: *rn" 
                      "Content-Type: multipart/x-mixed-replace; boundary=123456789000000000000987654321rn";
const char BOUNDARY[] = "rn--123456789000000000000987654321rn";
const char CTNTTYPE[] = "Content-Type: image/jpegrnContent-Length: ";
const int hdrLen = strlen(HEADER);
const int bdrLen = strlen(BOUNDARY);
const int cntLen = strlen(CTNTTYPE);

const uint16_t camColors[] = {0x480F,
                              0x400F, 0x400F, 0x400F, 0x4010, 0x3810, 0x3810, 0x3810, 0x3810, 0x3010, 0x3010,
                              0x3010, 0x2810, 0x2810, 0x2810, 0x2810, 0x2010, 0x2010, 0x2010, 0x1810, 0x1810,
                              0x1811, 0x1811, 0x1011, 0x1011, 0x1011, 0x0811, 0x0811, 0x0811, 0x0011, 0x0011,
                              0x0011, 0x0011, 0x0011, 0x0031, 0x0031, 0x0051, 0x0072, 0x0072, 0x0092, 0x00B2,
                              0x00B2, 0x00D2, 0x00F2, 0x00F2, 0x0112, 0x0132, 0x0152, 0x0152, 0x0172, 0x0192,
                              0x0192, 0x01B2, 0x01D2, 0x01F3, 0x01F3, 0x0213, 0x0233, 0x0253, 0x0253, 0x0273,
                              0x0293, 0x02B3, 0x02D3, 0x02D3, 0x02F3, 0x0313, 0x0333, 0x0333, 0x0353, 0x0373,
                              0x0394, 0x03B4, 0x03D4, 0x03D4, 0x03F4, 0x0414, 0x0434, 0x0454, 0x0474, 0x0474,
                              0x0494, 0x04B4, 0x04D4, 0x04F4, 0x0514, 0x0534, 0x0534, 0x0554, 0x0554, 0x0574,
                              0x0574, 0x0573, 0x0573, 0x0573, 0x0572, 0x0572, 0x0572, 0x0571, 0x0591, 0x0591,
                              0x0590, 0x0590, 0x058F, 0x058F, 0x058F, 0x058E, 0x05AE, 0x05AE, 0x05AD, 0x05AD,
                              0x05AD, 0x05AC, 0x05AC, 0x05AB, 0x05CB, 0x05CB, 0x05CA, 0x05CA, 0x05CA, 0x05C9,
                              0x05C9, 0x05C8, 0x05E8, 0x05E8, 0x05E7, 0x05E7, 0x05E6, 0x05E6, 0x05E6, 0x05E5,
                              0x05E5, 0x0604, 0x0604, 0x0604, 0x0603, 0x0603, 0x0602, 0x0602, 0x0601, 0x0621,
                              0x0621, 0x0620, 0x0620, 0x0620, 0x0620, 0x0E20, 0x0E20, 0x0E40, 0x1640, 0x1640,
                              0x1E40, 0x1E40, 0x2640, 0x2640, 0x2E40, 0x2E60, 0x3660, 0x3660, 0x3E60, 0x3E60,
                              0x3E60, 0x4660, 0x4660, 0x4E60, 0x4E80, 0x5680, 0x5680, 0x5E80, 0x5E80, 0x6680,
                              0x6680, 0x6E80, 0x6EA0, 0x76A0, 0x76A0, 0x7EA0, 0x7EA0, 0x86A0, 0x86A0, 0x8EA0,
                              0x8EC0, 0x96C0, 0x96C0, 0x9EC0, 0x9EC0, 0xA6C0, 0xAEC0, 0xAEC0, 0xB6E0, 0xB6E0,
                              0xBEE0, 0xBEE0, 0xC6E0, 0xC6E0, 0xCEE0, 0xCEE0, 0xD6E0, 0xD700, 0xDF00, 0xDEE0,
                              0xDEC0, 0xDEA0, 0xDE80, 0xDE80, 0xE660, 0xE640, 0xE620, 0xE600, 0xE5E0, 0xE5C0,
                              0xE5A0, 0xE580, 0xE560, 0xE540, 0xE520, 0xE500, 0xE4E0, 0xE4C0, 0xE4A0, 0xE480,
                              0xE460, 0xEC40, 0xEC20, 0xEC00, 0xEBE0, 0xEBC0, 0xEBA0, 0xEB80, 0xEB60, 0xEB40,
                              0xEB20, 0xEB00, 0xEAE0, 0xEAC0, 0xEAA0, 0xEA80, 0xEA60, 0xEA40, 0xF220, 0xF200,
                              0xF1E0, 0xF1C0, 0xF1A0, 0xF180, 0xF160, 0xF140, 0xF100, 0xF0E0, 0xF0C0, 0xF0A0,
                              0xF080, 0xF060, 0xF040, 0xF020, 0xF800,
                             };

uint16_t colorFrame(float color){
   uint8_t colorIndex = map(color, MinTemp-5.0, MaxTemp+5.0, 0, 255);
   colorIndex = constrain(colorIndex, 0, 255);

   return camColors[colorIndex];
}

void setMinMaxTemps(){
    MaxTemp = frame[0];            // Get first data to find Max and Min Temperature
    MinTemp = frame[0];
    
    for (int x = 0 ; x < 768 ; x++)     // Find Maximum and Minimum Temperature
    {
      MaxTemp = max(MaxTemp, frame[x]);
      MinTemp = min(MinTemp, frame[x]);
    }
}

void handle_videostream(){  

  WiFiClient client = server.client();
  client.write(HEADER, hdrLen);
  client.write(BOUNDARY, bdrLen);
 
  uint16_t frameColor[768];
  char buf[32];

  while (1)
  {
    if (!client.connected())
      break;

    Serial.println("Get the MLX frame!");
    if (mlx.getFrame(frame) != 0) {
      Serial.println("Failed to get the frame..");
    }
          
    setMinMaxTemps();
    for(int i=0; i<768; i++){
      frameColor[i] = colorFrame(frame[i]);
    }
    
    client.write(CTNTTYPE, cntLen);
    sprintf( buf, "%drnrn", 768*4 );
    client.write(buf, strlen(buf));
    Serial.println(buf);
    
    client.write((char *)frameColor, 768*4);
    client.write(BOUNDARY, bdrLen);
    if (!client.connected())
      break;
  }
}

Source: Windows Questions C++

LEAVE A COMMENT