# Inteligent Gesture Engine APDS-9960 & Algorithms

I have been playing with this breakout quite a lot and finally cracked something I can use.

Idea is a magic mirror carousel but also to checkout if its possible to do more than up, down, left, right.
Haven’t had a look at the arduino examples but the circuit python ones don’t actually use ‘gestures’ just simple leading edge detection, which is equally valid.

I have just done something off the top of my head with the SiN certainty engine where I have bodged full gestures of what is often approx 25 bytes for each direction register.
I am going to have to do some google on pattern recognition and that simple ‘card counting’ algo to maybe geek up on a bit of science.

Anyways what I have expects the flat of your hand and its quite accurate and try and use the longer flat of the little finger edge, always lead the same way and try and keep square to both axis.

If anyone is or has anything been playing with gesture engine ideas like this please share even if its just an article you know of.

The code here is not great I guess as 4 days ago I had totally forgotten (MS doesn’t help) python or any lang but its good enough and being singular its easy to see and tweak what is going on.

Code is here and just install the circuitpython apds9960 examples to grab that great i2c register lib. The actual python example is a bit mweh IMHO and it almost blind sighted into thinking the APDS is not capable of anything but extremely simple edge detection.
This isn’t a intelligent gesture, as its done by me, but pretty sure the SiN certainty engine would be capable of tacking on a few more gestures without gesture bleed and improving this vastly.

Not really interested in coding as yeah its just rough and ready and mainly to give as comprehensive picture of the gesture movement just displayed and somehow you have to think of a way of how to acurately capture that without false signals or bleeds into other gestures.

So yeah if you have any ideas or algorithms what I have here is a ratio of 2 diodes on the same axis and of 15 to 20 plots its expect ratio increments / decrements depending on direction from whatever near to far axis you choose.
If you are accurate with your hand movements you an be accurate with your gesture on a APDS9960 and the code proves that so if anyone has a better gesture detection engine that can run on a Pi0w please give me a shout.

``````import time
import digitalio
from board import SCL, SDA
from busio import I2C
from adafruit_bus_device.i2c_device import I2CDevice
from adafruit_register.i2c_bits import RWBits
from adafruit_register.i2c_bits import ROBits
from adafruit_register.i2c_bit import RWBit
from adafruit_register.i2c_bit import ROBit
from micropython import const

#pylint: disable-msg=bad-whitespace
#APDS9960_RAM        = const(0x00)
APDS9960_ENABLE     = const(0x80)
APDS9960_ATIME      = const(0x81)
APDS9960_WTIME      = const(0x83)
APDS9960_AILTL     = const(0x84)
APDS9960_AILTH      = const(0x85)
APDS9960_AIHTL      = const(0x86)
APDS9960_AIHTH      = const(0x87)
APDS9960_PILT       = const(0x89)
APDS9960_PIHT       = const(0x8B)
APDS9960_PERS       = const(0x8C)
APDS9960_CONFIG1    = const(0x8D)
APDS9960_PPULSE     = const(0x8E)
APDS9960_CONTROL    = const(0x8F)
APDS9960_CONFIG2    = const(0x90)
APDS9960_ID         = const(0x92)
APDS9960_STATUS     = const(0x93)
APDS9960_CDATAL     = const(0x94)
APDS9960_CDATAH     = const(0x95)
APDS9960_RDATAL     = const(0x96)
APDS9960_RDATAH     = const(0x97)
APDS9960_GDATAL     = const(0x98)
APDS9960_GDATAH     = const(0x99)
APDS9960_BDATAL     = const(0x9A)
APDS9960_BDATAH     = const(0x9B)
APDS9960_PDATA      = const(0x9C)
APDS9960_POFFSET_UR = const(0x9D)
APDS9960_POFFSET_DL = const(0x9E)
APDS9960_CONFIG3    = const(0x9F)
APDS9960_GPENTH     = const(0xA0)
APDS9960_GEXTH      = const(0xA1)
APDS9960_GCONF1     = const(0xA2)
APDS9960_GCONF2     = const(0xA3)
APDS9960_GOFFSET_U  = const(0xA4)
APDS9960_GOFFSET_D  = const(0xA5)
APDS9960_GOFFSET_L  = const(0xA7)
APDS9960_GOFFSET_R  = const(0xA9)
APDS9960_GPULSE     = const(0xA6)
APDS9960_GCONF3     = const(0xAA)
APDS9960_GCONF4     = const(0xAB)
APDS9960_GFLVL      = const(0xAE)
APDS9960_GSTATUS    = const(0xAF)
# APDS9960_IFORCE     = const(0xE4)
APDS9960_PICLEAR    = const(0xE5)
APDS9960_CICLEAR    = const(0xE6)
APDS9960_AICLEAR    = const(0xE7)
APDS9960_GFIFO_U    = const(0xFC)
# APDS9960_GFIFO_D    = const(0xFD)
# APDS9960_GFIFO_L    = const(0xFE)
# APDS9960_GFIFO_R    = const(0xFF)
DEVICE_ADDRESS      = const(0x39)
ROUTINE_PROXIMITY   = const(0x01)
ROUTINE_A_CLS       = const(0x02)
ROUTINE_GESTURE     = const(0X03)
#pylint: enable-msg=bad-whitespace

def calculate_color_temperature(r, g, b):
"""Converts the raw R/G/B values to color temperature in degrees Kelvin"""

#  1. Map RGB values to their XYZ counterparts.
#   Based on 6500K fluorescent, 3000K fluorescent
#    and 60W incandescent values for a wide range.
#    Note: Y = Illuminance or lux
x = (-0.14282 * r) + (1.54924 * g) + (-0.95641 * b)
y = (-0.32466 * r) + (1.57837 * g) + (-0.73191 * b)
z = (-0.68202 * r) + (0.77073 * g) + (0.56332 * b)

#  2. Calculate the chromaticity co-ordinates
xchrome = x / (x + y + z)
ychrome = y / (x + y + z)

#  3. Use   to determine the CCT
n = (xchrome - 0.3320) / (0.1858 - ychrome)

#  4. Calculate the final CCT
cct = (449.0 * pow(n, 3)) + (3525.0 * pow(n, 2)) + (6823.3 * n) + 5520.33

#    Return the results in degrees Kelvin
return cct

def calculate_lux(r, g, b):
"""Calculate ambient light values"""
#   This only uses RGB ... how can we integrate clear or calculate lux
#   based exclusively on clear since this might be more reliable?
illuminance = (-0.32466 * r) + (1.57837 * g) + (-0.73191 * b)

return illuminance

class DeviceControl: #pylint: disable-msg=too-few-public-methods
def __init__(self, i2c):
self.i2c_device = i2c
device_enable = RWBit(APDS9960_ENABLE, 0)
deviceID = ROBits(8, APDS9960_ID, 0)

# Shared register block
proximity_gesture_led_boost = RWBits(2, APDS9960_CONFIG2, 4)
proximity_gesture_saturation = ROBit(APDS9960_STATUS, 6)
c_als_proximity_persistance = RWBits(4, APDS9960_PERS, 4)
all_non_gesture_interupt_clear = ROBits(8, APDS9960_AICLEAR, 0)

# Proximity register block
proximity_enable = RWBit(APDS9960_ENABLE, 2)
proximity_interrupt_enable = RWBit(APDS9960_ENABLE, 5)
proximity_low_threshold = ROBits(8, APDS9960_PILT, 0)
proximity_high_threshold = ROBits(8, APDS9960_PIHT, 0)
proximity_pulse_length = RWBits(2, APDS9960_PPULSE, 6)
proximity_pulse_count = RWBits(6, APDS9960_PPULSE, 0)
proximity_gain_control = RWBits(2, APDS9960_CONTROL, 2)
proximity_led_drive_strength = RWBits(2, APDS9960_CONTROL, 6)
proximity_saturation_interupt_enable = RWBit(APDS9960_CONFIG2, 7)
proximity_interupt = ROBit(APDS9960_STATUS, 5)
proximity_valid = ROBit(APDS9960_STATUS, 1)
proximity_data = ROBits(8, APDS9960_PDATA, 0)
proximity_offset_up_right = ROBits(8, APDS9960_POFFSET_UR, 0)
proximity_offset_down_left = ROBits(8, APDS9960_POFFSET_DL, 0)
proximity_gain_compensation_enable = RWBit(APDS9960_CONFIG3, 5)
proximity_mask_up_enable = RWBit(APDS9960_CONFIG3, 3)
proximity_mask_down_enable = RWBit(APDS9960_CONFIG3, 2)
proximity_mask_left_enable = RWBit(APDS9960_CONFIG3, 1)
proximity_mask_right_enable = RWBit(APDS9960_CONFIG3, 0)
proximity_interupt_clear = RWBits(8, APDS9960_PICLEAR, 0)

# Color / ALS register block
c_als_enable = RWBit(APDS9960_ENABLE, 1)
c_als_interupt_enable = RWBit(APDS9960_ENABLE, 4)
c_als_wait_enable = RWBit(APDS9960_ENABLE, 3)
c_als_adc_integration_time = RWBits(8, APDS9960_ATIME, 0)
c_als_wait_time = RWBits(8, APDS9960_WTIME, 0)
c_als_low_threshold_lower_byte = RWBits(8, APDS9960_AILTL, 0)
c_als_low_threshold_upper_byte = RWBits(8, APDS9960_AILTH, 0)
c_als_high_threshold_lower_byte = RWBits(8, APDS9960_AIHTL, 0)
c_als_high_threshold_upper_byte = RWBits(8, APDS9960_AIHTH, 0)
c_als_wait_long_enable = RWBit(APDS9960_CONFIG1, 1)
c_als_gain_control =  RWBits(2, APDS9960_CONTROL, 0)
c_als_clear_diode_saturation_interupt_enable = RWBit(APDS9960_CONFIG2, 6)
c_als_clear_diode_saturation = ROBit(APDS9960_STATUS, 7)
c_als_interupt = ROBit(APDS9960_STATUS, 4)
c_als_valid = ROBit(APDS9960_STATUS, 0)
c_als_clear_channel_interupt_clear = RWBits(8, APDS9960_CICLEAR, 0)

# Gesture register block
gesture_enable= RWBit(APDS9960_ENABLE, 6)
gesture_entry_threshold = RWBits(8, APDS9960_GPENTH, 0)
gesture_exit_threshold = RWBits(8, APDS9960_GEXTH, 0)
gesture_fifo_threshold = RWBits(2, APDS9960_GCONF1, 6)
gesture_exit_mask = RWBits(4, APDS9960_GCONF1, 2)
gesture_exit_persistance = RWBits(2, APDS9960_GCONF1, 0)
gesture_gain_control = RWBits(2, APDS9960_GCONF2, 5)
gesture_led_drive_strength = RWBits(2, APDS9960_GCONF2, 3)
gesture_wait_time = RWBits(3, APDS9960_GCONF2, 0)
gesture_offset_up = RWBits(8, APDS9960_GOFFSET_U, 0)
gesture_offset_down = RWBits(8, APDS9960_GOFFSET_D, 0)
gesture_offset_left = RWBits(8, APDS9960_GOFFSET_L, 0)
gesture_offset_right = RWBits(8, APDS9960_GOFFSET_R, 0)
gesture_pulse_count = RWBits(6, APDS9960_GPULSE, 0)
gesture_pulse_length = RWBits(2, APDS9960_GPULSE, 6)
gesture_dimension_select = RWBits(2, APDS9960_GCONF3, 0)
gesture_fifo_clear= RWBit(APDS9960_GCONF4, 2)
gesture_interupt_enable= RWBit(APDS9960_GCONF4, 1)
gesture_mode = RWBit(APDS9960_GCONF4, 0)
gesture_fifo_level = RWBits(8, APDS9960_GFLVL, 0)
gesture_fifo_overflow = ROBit(APDS9960_GSTATUS, 1)
gesture_valid = ROBit(APDS9960_GSTATUS, 0)

#pylint: enable-msg=too-few-public-methods

# The follow is for I2C communications
comm_port = I2C(SCL, SDA)
device = I2CDevice(comm_port, DEVICE_ADDRESS)
registers = DeviceControl(device)

# Raise error if deviceID not found
if registers.deviceID != 0xAB:
raise RuntimeError()

# Choose routine and change below line ROUTINE_PROXIMITY ROUTINE_A_CLS ROUTINE_GESTURE
routine = ROUTINE_GESTURE
# Device Power ON
registers.device_enable = 0x01

if routine == ROUTINE_PROXIMITY:
registers.c_als_enable = 0x00
registers.gesture_enable = 0x00
registers.proximity_enable = 0x01
registers. proximity_interrupt_enable = 0x00
#registers.proximity_low_threshold = 0x00
#registers.proximity_high_threshold = 0xFF
registers.c_als_proximity_persistance = 0x00
#registers.proximity_pulse_length = 0x00
#registers.proximity_pulse_count = 0x00
registers.proximity_gain_control = 0x00
registers.proximity_led_drive_strength = 0x00
#registers.proximity_saturation_interupt_enable = 0x00
registers.proximity_gesture_led_boost = 0x00
#registers.proximity_offset_up_right = 0x00
#registers.proximity_offset_down_left = 0x00
#registers.proximity_gain_compensation_enable = 0x00
#registers.proximity_mask_up_enable = 0x00
#registers.proximity_mask_down_enable = 0x00
#registers.proximity_mask_left_enable = 0x00
#registers.proximity_mask_right_enable = 0x00
print("all_non_gesture_interupt_clear {}".format(registers.all_non_gesture_interupt_clear))

while True:
print("Data: {}; Valid: {}; Saturation: {}; Interupt: {}; Interupt Clear: {}".format(registers.proximity_data, registers.proximity_valid, registers.proximity_gesture_saturation, registers.proximity_interupt, registers.proximity_interupt_clear))

elif routine == ROUTINE_A_CLS:
registers.proximity_enable = 0x00
registers.gesture_enable = 0x00
registers.c_als_enable = 0x01
registers.c_als_gain_control = 0x00
registers.c_als_adc_integration_time = 0x01

while True:
#wait for color data to be ready
while not registers.c_als_valid:
time.sleep(0.005)

buf = bytearray(2)

buf[0] = APDS9960_CDATAL
device.write(buf, end=1, stop=False)
device.readinto(buf)
c = buf[1] << 8 | buf[0]

buf[0] = APDS9960_RDATAL
device.write(buf, end=1, stop=False)
device.readinto(buf)
r = buf[1] << 8 | buf[0]

buf[0] = APDS9960_BDATAL
device.write(buf, end=1, stop=False)
device.readinto(buf)
b = buf[1] << 8 | buf[0]

buf[0] = APDS9960_GDATAL
device.write(buf, end=1, stop=False)
device.readinto(buf)
g = buf[1] << 8 | buf[0]

print("red: ", r)
print("green: ", g)
print("blue: ", b)
print("clear: ", c)
print("color temp {}".format(calculate_color_temperature(r, g, b)))
print("light lux {}".format(calculate_lux(r, g, b)))
time.sleep(0.5)

elif routine == ROUTINE_GESTURE:

registers.c_als_enable = 0x00

registers.c_als_proximity_persistance = 0x00
registers.proximity_gain_control = 0x03
registers.proximity_led_drive_strength = 0x00

registers.proximity_enable = 0x01

registers.gesture_wait_time = 0x02
registers.gesture_entry_threshold = 0x0C
registers.gesture_exit_threshold = 0x0C
registers.gesture_fifo_threshold = 0x00
#registers.gesture_exit_mask = 0x00
registers.gesture_exit_persistance = 0x00
registers.gesture_gain_control = 0x03
registers.gesture_led_drive_strength = 0x00
proximity_gesture_led_boost = 0x03
#registers.gesture_pulse_count = 0x40
#registers.gesture_pulse_length = 0x01
#registers.gesture_dimension_select = 0x00
#registers.gesture_offset_up = 0x00
#registers.gesture_offset_down = 0x00
#registers.gesture_offset_left = 0x00
#registers.gesture_offset_right = 0x00

registers.gesture_enable = 0x01

first = 0
buffer = bytearray(129)

while True:

if registers.gesture_mode == 0:
if first == 0:
print ("---------------------------------------------------------------")
first = 1
left_right_direction = 0
up_down_direction = 0
left_right_valid_count = 0
up_down_valid_count = 0
last_left_right_ratio = 1
last_up_down_ratio = 1
gesture_count = 0
dataset_count = 0
else:
first = 0
datasets = registers.gesture_fifo_level
while datasets > 0:

print("Datasets: {}".format(datasets))

buffer[0]= APDS9960_GFIFO_U

device.write(buffer, end=1, stop=False)
device.readinto(buffer, start=1, end=(datasets *4) + 1)

x = 0
#print ("Len gesture: {}".format(len(gesture)))
while x < datasets:

up =  max(1, buffer[1 + (x * 4)])
down = max(1, buffer[2 + (x * 4)])
left = max(1, buffer[3 + (x * 4)])
right = max(1, buffer[4 + (x * 4)])

left_right_ratio = left / right
up_down_ratio = up / down
gesture_count = gesture_count + 1
if x < 2 and dataset_count == 0:

left_right_valid_count = left_right_valid_count + 1
up_down_valid_count = up_down_valid_count + 1

if min(last_left_right_ratio, left_right_ratio) < 1:
left_right_direction = 1
else:
left_right_direction = -1

if min(last_up_down_ratio, up_down_ratio) < 1:
up_down_direction = 1
else:
up_down_direction = -1

last_left_right_ratio = left_right_ratio
last_up_down_ratio = up_down_ratio

if x == 1 and dataset_count == 0:
if left_right_direction == 1:
last_left_right_ratio = min(last_left_right_ratio, left_right_ratio)
else:
last_left_right_ratio = max(last_left_right_ratio, left_right_ratio)
if up_down_direction == 1:
last_up_down_ratio = min(last_up_down_ratio, up_down_ratio)
else:
last_up_down_ratio = max(last_up_down_ratio, up_down_ratio)
else:
if left_right_direction == 1:
if last_left_right_ratio < left_right_ratio:
left_right_valid_count = left_right_valid_count + 1
last_left_right_ratio = left_right_ratio
else:
if last_left_right_ratio > left_right_ratio:
left_right_valid_count = left_right_valid_count + 1
last_left_right_ratio = left_right_ratio
if up_down_direction == 1:
if last_up_down_ratio < up_down_ratio:
up_down_valid_count = up_down_valid_count + 1
last_up_down_ratio = up_down_ratio
else:
if last_up_down_ratio > up_down_ratio:
up_down_valid_count = up_down_valid_count + 1
last_up_down_ratio = up_down_ratio

print("Up:{:3d} Down: {:3d} | Left: {:3d} Right: {:3d} | up_down_ratio: {:03.2f} | left_right_ratio: {:03.2f}".format(up, down, left, right, up_down_ratio, left_right_ratio))
x = x + 1

datasets = registers.gesture_fifo_level

up_down_certainty = up_down_valid_count / gesture_count
left_right_certainty = left_right_valid_count / gesture_count
print("Up_down_direction: {:3d} Count {:4d} Certainity {:03.2f} | Left_right_direction: {:3d} Count {:4d} Certainity {:03.2f}".format(up_down_direction, up_down_valid_count, up_down_certainty, left_right_direction, left_right_valid_count, left_right_certainty))
dataset_count = dataset_count + 1``````