diff --git a/code/backend_reconnaissance/__pycache__/audio_detector.cpython-38.pyc b/code/backend_reconnaissance/__pycache__/audio_detector.cpython-38.pyc new file mode 100644 index 0000000..ac00a11 Binary files /dev/null and b/code/backend_reconnaissance/__pycache__/audio_detector.cpython-38.pyc differ diff --git a/code/backend_reconnaissance/__pycache__/hand_detector.cpython-38.pyc b/code/backend_reconnaissance/__pycache__/hand_detector.cpython-38.pyc new file mode 100644 index 0000000..1302b58 Binary files /dev/null and b/code/backend_reconnaissance/__pycache__/hand_detector.cpython-38.pyc differ diff --git a/code/backend_reconnaissance/__pycache__/manager.cpython-38.pyc b/code/backend_reconnaissance/__pycache__/manager.cpython-38.pyc new file mode 100644 index 0000000..a19b3da Binary files /dev/null and b/code/backend_reconnaissance/__pycache__/manager.cpython-38.pyc differ diff --git a/code/backend_reconnaissance/__pycache__/network.cpython-38.pyc b/code/backend_reconnaissance/__pycache__/network.cpython-38.pyc new file mode 100644 index 0000000..6fd3d75 Binary files /dev/null and b/code/backend_reconnaissance/__pycache__/network.cpython-38.pyc differ diff --git a/code/backend_reconnaissance/audio_data.zip b/code/backend_reconnaissance/audio_data.zip deleted file mode 100644 index 46e243c..0000000 Binary files a/code/backend_reconnaissance/audio_data.zip and /dev/null differ diff --git a/code/backend_reconnaissance/audio_detector.py b/code/backend_reconnaissance/audio_detector.py index a3e170f..7a06d3f 100644 --- a/code/backend_reconnaissance/audio_detector.py +++ b/code/backend_reconnaissance/audio_detector.py @@ -119,7 +119,6 @@ def init_database(): if not os.path.isfile(os.path.join(data_dir, word)): for file in os.listdir(os.path.join(data_dir,word)): if os.path.isfile(os.path.join(data_dir, word,file)): - print(word,os.path.join(data_dir, word,file)) words.append(word) files.append(os.path.join(data_dir, word,file)) return words,files @@ -130,7 +129,23 @@ def get_word_metadata(word): return data[word] #Todo : detecte si pas de note donnée -def get_grade(): +def record(): + sr = 44100 # fréquence d'échantillonnage + duration = 6 # durée d'enregistrement en secondes + filename = "recording" # nom du fichier à enregistrer + record_audio(filename, duration, sr) + audio_query,sr = librosa.load(f'{filename}.wav', sr=sr) + return audio_query,sr + +def analyze(audio_query,sr): + coupe_silence(audio_query) + words, files = init_database() + audio_train_list = [librosa.load(file, sr=sr)[0] for file in files] + recognized_word_index = recognize_speech(audio_query, audio_train_list, sr) + recognized_word = words[recognized_word_index] + return get_word_metadata(recognized_word) + +def test(): sr = 44100 # fréquence d'échantillonnage duration = 6 # durée d'enregistrement en secondes filename = "recording" # nom du fichier à enregistrer diff --git a/code/backend_reconnaissance/hand_detector.py b/code/backend_reconnaissance/hand_detector.py index 4010cbe..8f78b55 100644 --- a/code/backend_reconnaissance/hand_detector.py +++ b/code/backend_reconnaissance/hand_detector.py @@ -1,13 +1,18 @@ import cv2 import mediapipe as mp import numpy as np +import os +from dotenv import load_dotenv +load_dotenv() + class HandDetector(): def __init__(self): + self.camera_id = int(os.getenv("CAMERA_ID")) self.mp_drawing = mp.solutions.drawing_utils self.mp_drawing_styles = mp.solutions.drawing_styles self.mp_hands = mp.solutions.hands - self.cap = cv2.VideoCapture(0) + self.cap = cv2.VideoCapture(self.camera_id) self.hands = self.mp_hands.Hands( model_complexity=0, min_detection_confidence=0.5, @@ -51,8 +56,9 @@ class HandDetector(): # To improve performance, optionally mark the image as not writeable to # pass by reference. image.flags.writeable = False + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) results = self.hands.process(image) - # print(results) + if results.multi_hand_landmarks: handsPositions = [] for hand_landmarks in results.multi_hand_landmarks: diff --git a/code/backend_reconnaissance/manager.py b/code/backend_reconnaissance/manager.py index 62da53a..77ce753 100644 --- a/code/backend_reconnaissance/manager.py +++ b/code/backend_reconnaissance/manager.py @@ -1,5 +1,5 @@ from hand_detector import HandDetector -from audio_detector import get_grade +from audio_detector import record, analyze, test from network import ApiClient, WebsocketServer import time @@ -57,23 +57,26 @@ class Manager(): state, coords, size, finalDecision = res self.server.sendMessage({"type": "effects", "effects": [{"type": state, "x":coords[0], "y": coords[1], "width": size, "height": size}]}) self.isLastHandPacketEmpty = False + self.timeLastChange = time.time() if(finalDecision != False): self.avis["note"] = 10 if finalDecision == "thumbs_up" else 0 self.state = 2 - self.timeLastChange = time.time() self.server.sendMessage({"type": "state", "state": 2}) elif self.isLastHandPacketEmpty == False: self.server.sendMessage({"type":"effects","effects":[]}) self.isLastHandPacketEmpty = True def audio(self): - result = get_grade() + audio_query,sr = record() + self.server.sendMessage({"type":"recording_done"}) + result = analyze(audio_query,sr) + # result = test() if(result != False): + print("mot detecté : " + result["display"] + " avec une note de " + str(result["grade"])) self.server.sendMessage({"type":"new_grade","word":result["display"]}) self.avis["notes_autres"]["test"] = result["grade"] time.sleep(3) self.state = 3 - self.timeLastChange = time.time() self.server.sendMessage({"type": "state", "state": 3}) def thankYou(self): diff --git a/code/backend_reconnaissance/network.py b/code/backend_reconnaissance/network.py index 4223f14..af007ed 100644 --- a/code/backend_reconnaissance/network.py +++ b/code/backend_reconnaissance/network.py @@ -4,6 +4,9 @@ import json import os import threading import websockets +from dotenv import load_dotenv + +load_dotenv() class WebsocketServer(threading.Thread): def __init__(self, onMessage, port=os.getenv("PORT"), host=os.getenv("HOST")): diff --git a/code/backend_reconnaissance/recording.wav b/code/backend_reconnaissance/recording.wav index 5d5625a..b73dffa 100644 Binary files a/code/backend_reconnaissance/recording.wav and b/code/backend_reconnaissance/recording.wav differ diff --git a/code/docker-compose.yaml b/code/docker-compose.yaml index 91b7c98..af464b8 100644 --- a/code/docker-compose.yaml +++ b/code/docker-compose.yaml @@ -85,22 +85,22 @@ services: # #Backend de la borne : scripts pythons de reconnaissances video et audio # #Envoient les infos a l'interface de la borne par websocket pour mettre a jour l'interface rapidement # #Met a jour les avis en faisant des requêtes a l'API - backend_reconnaissance: - build: ./backend_reconnaissance - container_name: backend_reconnaissance - restart: always - devices: - - /dev/video3:/dev/video0 - - /dev/snd:/dev/snd - environment: - - PORT=5000 - - HOST=backend_reconnaissance - - API_HOST=reviews_api - - API_PORT=8080 - ports: - #Ce container est le serveur websocker dont le client est l'interface de la borne qui tourne dans le navigateur - - 5000:5000 - user: root + # backend_reconnaissance: + # build: ./backend_reconnaissance + # container_name: backend_reconnaissance + # restart: always + # devices: + # - /dev/video3:/dev/video0 + # - /dev/snd:/dev/snd + # environment: + # - PORT=5000 + # - HOST=backend_reconnaissance + # - API_HOST=reviews_api + # - API_PORT=8080 + # ports: + # #Ce container est le serveur websocker dont le client est l'interface de la borne qui tourne dans le navigateur + # - 5000:5000 + # user: root video_loopback: build: ./video_loopback diff --git a/code/interface_borne/assets/js/audio_page.js b/code/interface_borne/assets/js/audio_page.js index 6da88e2..ea7d3c5 100644 --- a/code/interface_borne/assets/js/audio_page.js +++ b/code/interface_borne/assets/js/audio_page.js @@ -7,16 +7,21 @@ class AudioPage { set enabled(isEnabled) { this.isEnabled = isEnabled; this.DOMElement.style.display = isEnabled ? "block" : "none"; - document.getElementById("grade").innerHTML = ""; + document.getElementById("audio_status").innerHTML = "Enregistrement..."; + } + onRecordingDone() { + if(this.isEnabled) { + document.getElementById("audio_status").innerHTML = "Traitement..."; + } } setGrade(grade) { if(this.isEnabled) { - document.getElementById("grade").innerHTML = grade.toString(); + document.getElementById("audio_status").innerHTML = grade; } } reset() { - document.getElementById("grade").innerHTML = ""; + document.getElementById("audio_status").innerHTML = "Enregistrement..."; } } \ No newline at end of file diff --git a/code/interface_borne/assets/js/network.js b/code/interface_borne/assets/js/network.js index 90c95b2..7f92900 100644 --- a/code/interface_borne/assets/js/network.js +++ b/code/interface_borne/assets/js/network.js @@ -1,5 +1,5 @@ class WebsocketClient { - constructor(onNewEffects, onNewState, onNewGrade, onReset) { + constructor(onNewEffects, onNewState, onNewGrade, onReset, onRecordingDone) { this.socket = new WebSocket("ws://localhost:5000"); this.socket.addEventListener("open", (event) => { this.socket.send("connected"); @@ -13,10 +13,13 @@ class WebsocketClient { }else if(msg.type == "state") { onNewState(msg.state); }else if(msg.type == "new_grade") { - onNewGrade(Number(msg.grade)); + onNewGrade(msg.word); }else if(msg.type == "reset") { onReset(); } + else if(msg.type == "recording_done") { + onRecordingDone(); + } }; } } \ No newline at end of file diff --git a/code/interface_borne/assets/js/state_manager.js b/code/interface_borne/assets/js/state_manager.js index 9ab68b8..c25f0b4 100644 --- a/code/interface_borne/assets/js/state_manager.js +++ b/code/interface_borne/assets/js/state_manager.js @@ -21,6 +21,7 @@ class StateManager { (state) => this.setState(state), (grade) => this._audioPage.setGrade(grade), () => this.reset(), + () => this._audioPage.onRecordingDone(), ); this._sleepingPage.enabled = true; diff --git a/code/interface_borne/index.html b/code/interface_borne/index.html index 59182a1..56f7497 100644 --- a/code/interface_borne/index.html +++ b/code/interface_borne/index.html @@ -35,17 +35,15 @@

Dites-nous en plus

-

Donnez une note sur 10 au critère suivant

- - - - - - - -
Critère - Note / 10 -
Calme/10
+

Comment avez vous trouvé l'exposition ... ?

+

Dites un mot parmis la liste suivante

+ +

Mot détécté :

diff --git a/code/Interface_Lounes/determinationGenre.py b/code_autre/Interface_Lounes/determinationGenre.py similarity index 100% rename from code/Interface_Lounes/determinationGenre.py rename to code_autre/Interface_Lounes/determinationGenre.py diff --git a/code/Interface_Lounes/reconnaissancePouce.py b/code_autre/Interface_Lounes/reconnaissancePouce.py similarity index 100% rename from code/Interface_Lounes/reconnaissancePouce.py rename to code_autre/Interface_Lounes/reconnaissancePouce.py diff --git a/code/Traitement Langage Naturel/.gitkeep b/code_autre/Traitement Langage Naturel/.gitkeep similarity index 100% rename from code/Traitement Langage Naturel/.gitkeep rename to code_autre/Traitement Langage Naturel/.gitkeep diff --git a/code/Traitement Langage Naturel/LouvreAvis.txt b/code_autre/Traitement Langage Naturel/LouvreAvis.txt similarity index 100% rename from code/Traitement Langage Naturel/LouvreAvis.txt rename to code_autre/Traitement Langage Naturel/LouvreAvis.txt diff --git a/code/Traitement Langage Naturel/ReviewAnalysis.py b/code_autre/Traitement Langage Naturel/ReviewAnalysis.py similarity index 100% rename from code/Traitement Langage Naturel/ReviewAnalysis.py rename to code_autre/Traitement Langage Naturel/ReviewAnalysis.py diff --git a/code/Traitement Langage Naturel/TAL.ipynb b/code_autre/Traitement Langage Naturel/TAL.ipynb similarity index 100% rename from code/Traitement Langage Naturel/TAL.ipynb rename to code_autre/Traitement Langage Naturel/TAL.ipynb diff --git a/code/Traitement Langage Naturel/TALnegation__1_.ipynb b/code_autre/Traitement Langage Naturel/TALnegation__1_.ipynb similarity index 100% rename from code/Traitement Langage Naturel/TALnegation__1_.ipynb rename to code_autre/Traitement Langage Naturel/TALnegation__1_.ipynb diff --git a/code/Traitement Langage Naturel/fr_lexicon.txt b/code_autre/Traitement Langage Naturel/fr_lexicon.txt similarity index 100% rename from code/Traitement Langage Naturel/fr_lexicon.txt rename to code_autre/Traitement Langage Naturel/fr_lexicon.txt