intégration de la détéction audio à l'interface

code/backend_reconnaissance/audio_detector.py

@@ -119,7 +119,6 @@ def init_database():
         if not os.path.isfile(os.path.join(data_dir, word)):
             for file in os.listdir(os.path.join(data_dir,word)):
                 if os.path.isfile(os.path.join(data_dir, word,file)):
-                    print(word,os.path.join(data_dir, word,file))
                     words.append(word)
                     files.append(os.path.join(data_dir, word,file))
     return words,files
@@ -130,7 +129,23 @@ def get_word_metadata(word):
         return data[word]
 
 #Todo : detecte si pas de note donnée
-def get_grade():
+def record():
+    sr = 44100  # fréquence d'échantillonnage
+    duration = 6  # durée d'enregistrement en secondes
+    filename = "recording"  # nom du fichier à enregistrer
+    record_audio(filename, duration, sr)
+    audio_query,sr = librosa.load(f'{filename}.wav', sr=sr)
+    return audio_query,sr
+
+def analyze(audio_query,sr):
+    coupe_silence(audio_query)
+    words, files = init_database()
+    audio_train_list = [librosa.load(file, sr=sr)[0] for file in files]
+    recognized_word_index = recognize_speech(audio_query, audio_train_list, sr)
+    recognized_word = words[recognized_word_index]
+    return get_word_metadata(recognized_word)
+
+def test():
     sr = 44100  # fréquence d'échantillonnage
     duration = 6  # durée d'enregistrement en secondes
     filename = "recording"  # nom du fichier à enregistrer

code/backend_reconnaissance/hand_detector.py

@@ -1,13 +1,18 @@
 import cv2
 import mediapipe as mp
 import numpy as np
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
 
 class HandDetector():
 	def __init__(self):
+		self.camera_id = int(os.getenv("CAMERA_ID"))
 		self.mp_drawing = mp.solutions.drawing_utils
 		self.mp_drawing_styles = mp.solutions.drawing_styles
 		self.mp_hands = mp.solutions.hands
-		self.cap = cv2.VideoCapture(0)
+		self.cap = cv2.VideoCapture(self.camera_id)
 		self.hands = self.mp_hands.Hands(
 				model_complexity=0,
 				min_detection_confidence=0.5,
@@ -51,8 +56,9 @@ class HandDetector():
 			# To improve performance, optionally mark the image as not writeable to
 			# pass by reference.
 			image.flags.writeable = False
+			image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 			results = self.hands.process(image)
-			# print(results)
+
 			if results.multi_hand_landmarks:
 				handsPositions = []
 				for hand_landmarks in results.multi_hand_landmarks:

code/backend_reconnaissance/manager.py

@@ -1,5 +1,5 @@
 from hand_detector import HandDetector
-from audio_detector import get_grade
+from audio_detector import record, analyze, test
 from network import ApiClient, WebsocketServer
 import time
 
@@ -57,23 +57,26 @@ class Manager():
             state, coords, size, finalDecision = res
             self.server.sendMessage({"type": "effects", "effects": [{"type": state, "x":coords[0], "y": coords[1], "width": size, "height": size}]})
             self.isLastHandPacketEmpty = False
+            self.timeLastChange = time.time()
             if(finalDecision != False):
                 self.avis["note"] = 10 if finalDecision == "thumbs_up" else 0
                 self.state = 2
-                self.timeLastChange = time.time()
                 self.server.sendMessage({"type": "state", "state": 2})
         elif self.isLastHandPacketEmpty == False:
             self.server.sendMessage({"type":"effects","effects":[]})
             self.isLastHandPacketEmpty = True
 
     def audio(self):
-        result = get_grade()
+        audio_query,sr = record()
+        self.server.sendMessage({"type":"recording_done"})
+        result = analyze(audio_query,sr)
+        # result = test()
         if(result != False):
+            print("mot detecté : " + result["display"] + " avec une note de " + str(result["grade"]))
             self.server.sendMessage({"type":"new_grade","word":result["display"]})
             self.avis["notes_autres"]["test"] = result["grade"]
             time.sleep(3)
             self.state = 3
-            self.timeLastChange = time.time()
             self.server.sendMessage({"type": "state", "state": 3})
 
     def thankYou(self):

code/backend_reconnaissance/network.py

@@ -4,6 +4,9 @@ import json
 import os
 import threading
 import websockets
+from dotenv import load_dotenv
+
+load_dotenv()
 
 class WebsocketServer(threading.Thread):
     def __init__(self, onMessage, port=os.getenv("PORT"), host=os.getenv("HOST")):