MLX60940 stream to v4l2loopback for Open CV


#1

I have the MLX60940 ir sensor working on my Raspberry Pi with the fbuf example code from the library. I also installed OpenCV and v4l2loopback.

I do not know how to change the code in the fbuf example to stream to a virtual device using v4l2loopback to use it in OpenCV.

Any help or thoughts is appreciated.

v4l2loopback - https://github.com/umlaeute/v4l2loopback

Example of Lepton i2c IR for v4l2loopback - https://github.com/groupgets/LeptonModule/tree/master/software/v4l2lepton


#2

Is there a way to change the double for loop at the end to export to a Mat that could be read by OpenCV? This way v4l2loopback would not be needed.

Mat OpenCV Documentation

Example below from: http://answers.opencv.org/question/26682/opencv-create-mat-object-with-a-loop/

   // Allocate memory for your matrix (e.g. 8x8 int matrix)
Mat temp_matrix (15, 15, CV_8UC1);
// Make a double loop over indexes and assign values
for(int rows; rows < 15; rows++){
   for(int cols; cols < 15; cols++){
      temp_matrix.at<uchar>(rows,cols) = selected_value;
   }
}

But I can’t get the code to work that way. I made this modification but it does not compile.

while (1){
    auto start = std::chrono::system_clock::now();
    MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
    MLX90640_InterpolateOutliers(frame, eeMLX90640);

    eTa = MLX90640_GetTa(frame, &mlx90640);
    MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);

Mat test_mat (24, 32, CV8_UC1);

    for(int y = 0; y < 24; y++){
        for(int x = 0; x < 32; x++){
            float val = mlx90640To[32 * (23-y) + x];
            test_mat((y*IMAGE_SCALE), (x*IMAGE_SCALE)) = val;
        }
    }
    auto end = std::chrono::system_clock::now();
    auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
    std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}

return 0;
}

#3

The CV8_UC1 datatype was wrong for a float. With the code below I still get one compile error. Making progress toward eventually having the MLX90640 be readable by OpenCV.

Error:

error:no match for 'operator[[]' (operand types are 'cv::Mat' and 'int')
test_mat[y,x] = val;
        ^

Code:

while (1){
auto start = std::chrono::system_clock::now();
MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
MLX90640_InterpolateOutliers(frame, eeMLX90640);

eTa = MLX90640_GetTa(frame, &mlx90640);
MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);

Mat test_mat (24, 32, CV_32FC1, data);

for(int y = 0; y < 24; y++){
    for(int x = 0; x < 32; x++){
        float val = mlx90640To[32 * (23-y) + x];
        test_mat[y, x] = val;
    }
}
auto end = std::chrono::system_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}

return 0;
}

#4

Now I have code that compiles but it does not do what I want. I am trying to show the Mat data in OpenCV with the imshow function but have not had success yet.

Hopefully will find a solution. Suggestions are welcome, kinda stuck now.

while (1){
    auto start = std::chrono::system_clock::now();
    MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
    MLX90640_InterpolateOutliers(frame, eeMLX90640);

    eTa = MLX90640_GetTa(frame, &mlx90640);
    MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);

    Mat test_mat (24, 32, CV_32FC1, data);

    for(int y = 0; y < 24; y++){
        for(int x = 0; x < 32; x++){
            float val = mlx90640To[32 * (23-y) + x];
            test_mat.at<float>(y,x) = val;
        }

    }

    namedWindow( "Test Window", WINDOW_AUTOSIZE);
    imshow ("Test Window", test_mat);

    auto end = std::chrono::system_clock::now();
    auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
    std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}

#5

With the help of StackOverflow I was able to come up with a solution to get the MLX90640 data into OpenCV. The working code is below. I am sure it can be made more efficient.

#include <stdint.h>
#include <iostream>
#include <cstring>
#include <fstream>
#include <chrono>
#include <thread>
#include <math.h>
#include "headers/MLX90640_API.h"

#include "opencv2/opencv.hpp"

using namespace cv;
using namespace std;

#define MLX_I2C_ADDR 0x33

// Valid frame rates are 1, 2, 4, 8, 16, 32 and 64
// The i2c baudrate is set to 1mhz to support these
#define FPS 8
#define FRAME_TIME_MICROS (1000000/FPS)

// Despite the framerate being ostensibly FPS hz
// The frame is often not ready in time
// This offset is added to the FRAME_TIME_MICROS
// to account for this.
#define OFFSET_MICROS 850

int main(){
    static uint16_t eeMLX90640[832];
    float emissivity = 1;
    uint16_t frame[834];
    static float image[768];
    static float mlx90640To[768];
    float eTa;
    static uint16_t data[768*sizeof(float)];

    auto frame_time = std::chrono::microseconds(FRAME_TIME_MICROS + OFFSET_MICROS);

    MLX90640_SetDeviceMode(MLX_I2C_ADDR, 0);
    MLX90640_SetSubPageRepeat(MLX_I2C_ADDR, 0);
    switch(FPS){
        case 1:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b001);
            break;
        case 2:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b010);
            break;
        case 4:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b011);
            break;
        case 8:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b100);
            break;
        case 16:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b101);
            break;
        case 32:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b110);
            break;
        case 64:
            MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b111);
            break;
        default:
            printf("Unsupported framerate: %d", FPS);
            return 1;
    }
    MLX90640_SetChessMode(MLX_I2C_ADDR);

    paramsMLX90640 mlx90640;
    MLX90640_DumpEE(MLX_I2C_ADDR, eeMLX90640);
    MLX90640_ExtractParameters(eeMLX90640, &mlx90640);

    while (1){
        auto start = std::chrono::system_clock::now();
        MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
        MLX90640_InterpolateOutliers(frame, eeMLX90640);

        eTa = MLX90640_GetTa(frame, &mlx90640);
        MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);

        Mat IR_mat (32,24, CV_32FC1, data); 

        for(int y = 0; y < 24; y++){
            for(int x = 0; x < 32; x++){
                float val = mlx90640To[32 * (23-y) + x];
                IR_mat.at<float>(x,y) = val;
            }
        }

        // Normalize the mat
        Mat normal_mat;
        normalize(IR_mat, normal_mat, 0,1.0, NORM_MINMAX, CV_32FC1);

        // Convert Mat to CV_U8 to use applyColorMap
        double minVal, maxVal;
        minMaxLoc(normal_mat, &minVal, &maxVal);
        Mat u8_mat;
        normal_mat.convertTo(u8_mat, CV_8U, 255.0/(maxVal - minVal), -minVal);

        // Resize mat
        Mat size_mat;
        resize(u8_mat, size_mat, Size(240,320), INTER_CUBIC);

        // Apply false color
        Mat falsecolor_mat;
        applyColorMap(size_mat, falsecolor_mat, COLORMAP_JET);

        // Display stream in window
        namedWindow( "IR Camera Window");
        imshow ("IR Camera Window", falsecolor_mat);
        waitKey(1);

        auto end = std::chrono::system_clock::now();
        auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
        std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
    }


    return 0;
}