Mediapipe: https://github.com/google/mediapipe
MediaPipe Hand: https://google.github.io/mediapipe/solutions/hands
MediaPipeはBazarevskyらがCVPR2019で発表したオープンソースの機械学習用フレームワークで，そこで用いられている手認識機能は，single-shot手のひら認識アルゴリズムとlandmark認識モデルが組み合わされたものです（Google AI Blog: On-Device, Real-Time Hand Tracking with MediaPipe）
手の形状は以下の各ランドマークの座標として取得できます．毎フレーム推論が走り，cv座標での値が取得できます．
返ってくるlandmarkのオブジェクトは以下のようにして座標のfloatが取り出せます
1
2
3
4
for hand_idx, landmarks in enumerate(multi_hand_landmarks):
for point_idx, points in enumerate(landmarks.landmark):
print(f"Hand: {hand_idx}, {HAND_LANDMARK_NAMES[point_idx]},"
+ f"x:{points.x} y:{points.y} z:{points.z}")この時のHAND_LANDMARK_NAMESは，以下のような順番になっています．
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
HAND_LANDMARK_NAMES = [
"wrist",
"thumb_1",
"thumb_2",
"thumb_3",
"thumb_4",
"index_1",
"index_2",
"index_3",
"index_4",
"middle_1",
"middle_2",
"middle_3",
"middle_4",
"ring_1",
"ring_2",
"ring_3",
"ring_4",
"pinky_1",
"pinky_2",
"pinky_3",
"pinky_4"
]今回はこのMediaPipeによるリアルタイム手認識を用いて何かしらのインタラクティブ作品やWekinator等を用いるジェスチャ認識などのためのOpen Sound Controlでのデータ送信をプロトタイプします．
スクリプト
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# Atsuya Kobayashi 2020-12-22
# Reference: https://google.github.io/mediapipe/solutions/hands
# LICENCE: MIT
from itertools import chain
import mediapipe as mp
from cv2 import cv2
from pythonosc import udp_client
IP = "127.0.0.1"
PORT = 7474
VIDEO_DEVICE_ID = 0
RELATIVE_AXIS_MODE = True
HAND_LANDMARK_NAMES = [
"wrist",
"thumb_1",
"thumb_2",
"thumb_3",
"thumb_4",
"index_1",
"index_2",
"index_3",
"index_4",
"middle_1",
"middle_2",
"middle_3",
"middle_4",
"ring_1",
"ring_2",
"ring_3",
"ring_4",
"pinky_1",
"pinky_2",
"pinky_3",
"pinky_4"
]
def extract_detected_hands_points(multi_hand_landmarks,
send_osc_client=None):
if multi_hand_landmarks is not None:
for hand_idx, landmarks in enumerate(multi_hand_landmarks):
for point_idx, points in enumerate(landmarks.landmark):
# if you want to check data on console
print(f"Hand: {hand_idx}, {HAND_LANDMARK_NAMES[point_idx]},"
+ f"x:{points.x} y:{points.y} z:{points.z}")
"""
if you want to send data to addresses correspoding
to landmarks names on detected hands, use berow
"""
# if send_osc_client is not None:
# send_osc_client.send_message(f"/{HAND_LANDMARK_NAMES[point_idx]}",
# [points.x, points.y])
"""if you want to send data to single input address, use berow"""
if send_osc_client is not None:
send_osc_client.send_message(
f"/YOUR_OSC_ADDRESS",
list(chain.from_iterable([[p.x, p.y] for p in landmarks.landmark])))
if __name__ == "__main__":
mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
min_detection_confidence=0.5, min_tracking_confidence=0.5)
cap = cv2.VideoCapture(VIDEO_DEVICE_ID)
osc_client = udp_client.SimpleUDPClient(IP, PORT)
while cap.isOpened():
success, image = cap.read()
if not success:
print("Ignoring empty camera frame.")
# If loading a video, use 'break' instead of 'continue'.
continue
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
extract_detected_hands_points(results.multi_hand_landmarks,
send_osc_client=osc_client)
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
cv2.imshow('Detected Hands', image)
if cv2.waitKey(5) & 0xFF == 27:
break
hands.close()
cap.release()
