I have the MLX60940 ir sensor working on my Raspberry Pi with the fbuf example code from the library. I also installed OpenCV and v4l2loopback.
I do not know how to change the code in the fbuf example to stream to a virtual device using v4l2loopback to use it in OpenCV.
Any help or thoughts is appreciated.
v4l2loopback - https://github.com/umlaeute/v4l2loopback
Example of Lepton i2c IR for v4l2loopback - https://github.com/groupgets/LeptonModule/tree/master/software/v4l2lepton
Is there a way to change the double for loop at the end to export to a Mat that could be read by OpenCV? This way v4l2loopback would not be needed.
Example below from: http://answers.opencv.org/question/26682/opencv-create-mat-object-with-a-loop/
// Allocate memory for your matrix (e.g. 8x8 int matrix)
Mat temp_matrix (15, 15, CV_8UC1);
// Make a double loop over indexes and assign values
for(int rows; rows < 15; rows++){
for(int cols; cols < 15; cols++){
temp_matrix.at<uchar>(rows,cols) = selected_value;
}
}
But I can’t get the code to work that way. I made this modification but it does not compile.
while (1){
auto start = std::chrono::system_clock::now();
MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
MLX90640_InterpolateOutliers(frame, eeMLX90640);
eTa = MLX90640_GetTa(frame, &mlx90640);
MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);
Mat test_mat (24, 32, CV8_UC1);
for(int y = 0; y < 24; y++){
for(int x = 0; x < 32; x++){
float val = mlx90640To[32 * (23-y) + x];
test_mat((y*IMAGE_SCALE), (x*IMAGE_SCALE)) = val;
}
}
auto end = std::chrono::system_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}
return 0;
}The CV8_UC1 datatype was wrong for a float. With the code below I still get one compile error. Making progress toward eventually having the MLX90640 be readable by OpenCV.
Error:
error:no match for 'operator[[]' (operand types are 'cv::Mat' and 'int')
test_mat[y,x] = val;
^
Code:
while (1){
auto start = std::chrono::system_clock::now();
MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
MLX90640_InterpolateOutliers(frame, eeMLX90640);
eTa = MLX90640_GetTa(frame, &mlx90640);
MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);
Mat test_mat (24, 32, CV_32FC1, data);
for(int y = 0; y < 24; y++){
for(int x = 0; x < 32; x++){
float val = mlx90640To[32 * (23-y) + x];
test_mat[y, x] = val;
}
}
auto end = std::chrono::system_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}
return 0;
}Now I have code that compiles but it does not do what I want. I am trying to show the Mat data in OpenCV with the imshow function but have not had success yet.
Hopefully will find a solution. Suggestions are welcome, kinda stuck now.
while (1){
auto start = std::chrono::system_clock::now();
MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
MLX90640_InterpolateOutliers(frame, eeMLX90640);
eTa = MLX90640_GetTa(frame, &mlx90640);
MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);
Mat test_mat (24, 32, CV_32FC1, data);
for(int y = 0; y < 24; y++){
for(int x = 0; x < 32; x++){
float val = mlx90640To[32 * (23-y) + x];
test_mat.at<float>(y,x) = val;
}
}
namedWindow( "Test Window", WINDOW_AUTOSIZE);
imshow ("Test Window", test_mat);
auto end = std::chrono::system_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}With the help of StackOverflow I was able to come up with a solution to get the MLX90640 data into OpenCV. The working code is below. I am sure it can be made more efficient.
#include <stdint.h>
#include <iostream>
#include <cstring>
#include <fstream>
#include <chrono>
#include <thread>
#include <math.h>
#include "headers/MLX90640_API.h"
#include "opencv2/opencv.hpp"
using namespace cv;
using namespace std;
#define MLX_I2C_ADDR 0x33
// Valid frame rates are 1, 2, 4, 8, 16, 32 and 64
// The i2c baudrate is set to 1mhz to support these
#define FPS 8
#define FRAME_TIME_MICROS (1000000/FPS)
// Despite the framerate being ostensibly FPS hz
// The frame is often not ready in time
// This offset is added to the FRAME_TIME_MICROS
// to account for this.
#define OFFSET_MICROS 850
int main(){
static uint16_t eeMLX90640[832];
float emissivity = 1;
uint16_t frame[834];
static float image[768];
static float mlx90640To[768];
float eTa;
static uint16_t data[768*sizeof(float)];
auto frame_time = std::chrono::microseconds(FRAME_TIME_MICROS + OFFSET_MICROS);
MLX90640_SetDeviceMode(MLX_I2C_ADDR, 0);
MLX90640_SetSubPageRepeat(MLX_I2C_ADDR, 0);
switch(FPS){
case 1:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b001);
break;
case 2:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b010);
break;
case 4:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b011);
break;
case 8:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b100);
break;
case 16:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b101);
break;
case 32:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b110);
break;
case 64:
MLX90640_SetRefreshRate(MLX_I2C_ADDR, 0b111);
break;
default:
printf("Unsupported framerate: %d", FPS);
return 1;
}
MLX90640_SetChessMode(MLX_I2C_ADDR);
paramsMLX90640 mlx90640;
MLX90640_DumpEE(MLX_I2C_ADDR, eeMLX90640);
MLX90640_ExtractParameters(eeMLX90640, &mlx90640);
while (1){
auto start = std::chrono::system_clock::now();
MLX90640_GetFrameData(MLX_I2C_ADDR, frame);
MLX90640_InterpolateOutliers(frame, eeMLX90640);
eTa = MLX90640_GetTa(frame, &mlx90640);
MLX90640_CalculateTo(frame, &mlx90640, emissivity, eTa, mlx90640To);
Mat IR_mat (32,24, CV_32FC1, data);
for(int y = 0; y < 24; y++){
for(int x = 0; x < 32; x++){
float val = mlx90640To[32 * (23-y) + x];
IR_mat.at<float>(x,y) = val;
}
}
// Normalize the mat
Mat normal_mat;
normalize(IR_mat, normal_mat, 0,1.0, NORM_MINMAX, CV_32FC1);
// Convert Mat to CV_U8 to use applyColorMap
double minVal, maxVal;
minMaxLoc(normal_mat, &minVal, &maxVal);
Mat u8_mat;
normal_mat.convertTo(u8_mat, CV_8U, 255.0/(maxVal - minVal), -minVal);
// Resize mat
Mat size_mat;
resize(u8_mat, size_mat, Size(240,320), INTER_CUBIC);
// Apply false color
Mat falsecolor_mat;
applyColorMap(size_mat, falsecolor_mat, COLORMAP_JET);
// Display stream in window
namedWindow( "IR Camera Window");
imshow ("IR Camera Window", falsecolor_mat);
waitKey(1);
auto end = std::chrono::system_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::this_thread::sleep_for(std::chrono::microseconds(frame_time - elapsed));
}
return 0;
}