Agfdhyk

I am working on a school project with MLX90621 16x4 IR sensor array and a Raspberry Pi with Python to get the temperature array data and turn it into a thermal colored overlay. This overlay is able to align with a person`s forehead using Haarcascade and it can show the reading of the forehead temperature.

I have managed to do the forehead tracking code with python and get the thermo-data overlaying the camera image with this link:
https://hackaday.io/project/6416-raspberry-pi-thermal-imaging

However, I can only to get it to work separately, I am struggling to get both codes to work together. I have done a lot of research on the subject but I seem to only find Arduino and C++ tutorials or theory related posts.

I am new to programming and raspberry pi.

Here is the code of forehead tracking using Haarcascade and Pi Camera:

from picamera.array import PiRGBArray

from picamera import PiCamera

import time

import cv2



# initialize the camera and grab a reference to the raw camera capture

camera = PiCamera()

camera.resolution = (640, 480)

camera.framerate = 32

rawCapture = PiRGBArray(camera, size=(640, 480))



# allow the camera to warmup

time.sleep(0.1)



#Use the trained xml classifier for eyes detection in the video

eye_cascade = cv2.CascadeClassifier('/home/pi/Desktop/mlxd-master_terickson/testScripts/haarcascade_eye.xml')

#Use the trained xml classifier for faces detection in the video

face_cascade = cv2.CascadeClassifier('/home/pi/Desktop/mlxd-master_terickson/testScripts/haarcascade_frontalface_default.xml')







# capture frames from the camera

for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):

    # grab the raw NumPy array representing the image, then initialize the timestamp

    # and occupied/unoccupied text

    image = frame.array

    gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)

    font = cv2.FONT_HERSHEY_SIMPLEX



    eyes = eye_cascade.detectMultiScale(gray, 1.3, 6)               # To detect eyes in the video   

    for(ex,ey,ew,eh) in eyes:                                       

        cv2.rectangle(image, (ex,ey), (ex+ew,ey+eh), (0,255,0),2)   # To draw rectangle boxes around the eyes



    faces = face_cascade.detectMultiScale(gray, 1.3, 6)

    for (x,y,w,h) in faces:

        forehead_center = int(x+w/1), int(y+h/4)                    # Location of the forehead in face

        cv2.rectangle(image,(x,y),(x+w,y+h),(255,0,0),2)            # To draw rectangle boxes around the face

        cv2.rectangle(image,(x,y),forehead_center,(255,255,0),2)    # To draw rectangle boxes around the forehead of the face

        cv2.putText(image,'x: {0} | y: {1}'.format(forehead_center[0], forehead_center[1]),(10,50), font, 1.2,(0,0,255),1)  # To put the x and y cord of the forehead



    # show the frame

    cv2.imshow("Eye, Face and Forehead tracking", image)

    key = cv2.waitKey(1) & 0xFF



    # clear the stream in preparation for the next frame

    rawCapture.truncate(0)



    # if the `q` key was pressed, break from the loop

    if cv2.waitKey(1) & 0xFF == ord('q'):  

        break

Here is the code to get the temperature array into an overlay and display on the video preview using Pi Camera:

import picamera

import numpy as np

import subprocess

import skimage

from skimage import io, exposure, transform, img_as_float, img_as_ubyte

import matplotlib.pyplot as plt

from time import sleep

import cv2



# IR registration parameters

ROT = np.deg2rad(90)

SCALE = (36.2, 36.4)

OFFSET = (580, 170)



def getImage():

    fn = r'/home/pi/tmp.jpg'

    proc = subprocess.Popen('raspistill -o %s -w 640 -h 480 -n -t 3' % (fn),

                            shell=True, stderr=subprocess.STDOUT)

    proc.wait()

    im = io.imread(fn, as_grey=True)

    im = exposure.equalize_hist(im)

    return skimage.img_as_ubyte(im)





def get_overlay(fifo):

    # get the whole FIFO

    ir_raw = fifo.read()

    # trim to 128 bytes

    ir_trimmed = ir_raw[0:128]

    # go all numpy on it

    ir = np.frombuffer(ir_trimmed, np.uint16)

    # set the array shape to the sensor shape (16x4)

    ir = ir.reshape((16, 4))[::-1, ::-1]

    ir = img_as_float(ir)

    # stretch contrast on our heat map

    p2, p98 = np.percentile(ir, (2, 98))

    ir = exposure.rescale_intensity(ir, in_range=(p2, p98))

    # increase even further? (optional)

    ir = exposure.equalize_hist(ir)



    # turn our array into pretty colors

    cmap = plt.get_cmap('Spectral')

    rgba_img = cmap(ir)

    rgb_img = np.delete(rgba_img, 3, 2)



    # align the IR array with the camera

    tform = transform.AffineTransform(scale=SCALE, rotation=ROT, translation=OFFSET)

    ir_aligned = transform.warp(rgb_img, tform.inverse, mode='constant', output_shape=im.shape)

    # turn it back into a ubyte so it'll display on the preview overlay

    ir_byte = img_as_ubyte(ir_aligned)

    # return buffer

    return np.getbuffer(ir_byte)



    #GaussianBlur = cv2.GaussianBlur(camera, (20,20),0)

    #medianFiltered = cv2.medianBlur(camera,5)



im = getImage()





with picamera.PiCamera() as camera:

    camera.led = True

    camera.resolution = (640, 480)

    camera.framerate = 35

    camera.start_preview()



    # get the temperature array, and align with the image

    fifo = open('/var/run/mlx90621.sock', 'r')

    o = camera.add_overlay(get_overlay(fifo), layer=3, alpha=90)



    # update loop

    while True:

        sleep(0.001)

        o.update(get_overlay(fifo))



    print('Error! Closing...')

    camera.remove_overlay(o)

    fifo.close()