diff --git a/.vim/coc-settings.json b/.vim/coc-settings.json new file mode 100644 index 0000000..7c7af84 --- /dev/null +++ b/.vim/coc-settings.json @@ -0,0 +1,6 @@ +{ + "python.linting.pylintEnabled": true, + "python.linting.flake8Enabled": false, + "python.linting.enabled": true, + "python.linting.pylamaEnabled": false +} \ No newline at end of file diff --git a/code/backend_reconnaissance/hand_detector.py b/code/backend_reconnaissance/hand_detector.py index 7427153..deecd52 100644 --- a/code/backend_reconnaissance/hand_detector.py +++ b/code/backend_reconnaissance/hand_detector.py @@ -5,116 +5,192 @@ import os from dotenv import load_dotenv load_dotenv() +camera_id = int(os.getenv("CAMERA_ID")) +mp_hands = mp.solutions.hands +cap = cv2.VideoCapture(camera_id) + +def prodScalaire(V1,V2): + return V1[0]*V2[0]+V1[1]*V2[1]/(np.sqrt(V1[0]**2+V1[1]**2)*np.sqrt(V2[0]**2+V2[1]**2)) + class HandDetector(): - def __init__(self): - self.camera_id = int(os.getenv("CAMERA_ID")) - self.mp_drawing = mp.solutions.drawing_utils - self.mp_drawing_styles = mp.solutions.drawing_styles - self.mp_hands = mp.solutions.hands - self.cap = cv2.VideoCapture(self.camera_id) - self.hands = self.mp_hands.Hands( - model_complexity=0, - min_detection_confidence=0.5, - min_tracking_confidence=0.5) - #Paramètres - self.BUFFER_LENGTH = 60 - self.DETECTION_THRESHOLD = 3/4 + def __init__(self): + self.hands = mp_hands.Hands( + model_complexity=0, + min_detection_confidence=0.5, + min_tracking_confidence=0.5) + #Paramètres + self.BUFFER_LENGTH = 200 + self.DETECTION_THRESHOLD = 1/2 + self.resultBuffer = [] - self.resultBuffer = [] - - def reset(self): - self.resultBuffer = [] + def reset(self): + self.resultBuffer = [] - def reconnaissancePouce(self,handLandmarks): - etatDuPouce=["neutre","thumbs_down","thumbs_up"] - i=0 - j=0 - for cpt in range (0,4): - V1=[handLandmarks[(4*cpt)+5][0]-handLandmarks[(4*cpt)+0][0],handLandmarks[(4*cpt)+5][1]-handLandmarks[(4*cpt)+0][1]] - V2=[handLandmarks[(4*cpt)+8][0]-handLandmarks[(4*cpt)+5][0],handLandmarks[(4*cpt)+8][1]-handLandmarks[(4*cpt)+5][1]] - j1=np.dot(V1,V2) - V1=[handLandmarks[(4*cpt)+6][0]-handLandmarks[(4*cpt)+5][0],handLandmarks[(4*cpt)+6][1]-handLandmarks[(4*cpt)+5][1]] - V2=[handLandmarks[(4*cpt)+8][0]-handLandmarks[(4*cpt)+6][0],handLandmarks[(4*cpt)+8][1]-handLandmarks[(4*cpt)+6][1]] - j2=np.dot(V1,V2) - if (j1>0 and j2>0): - return etatDuPouce[0] - V1=[handLandmarks[4][0]-handLandmarks[1][0],handLandmarks[4][1]-handLandmarks[1][1]] - V2=[handLandmarks[2][0]-handLandmarks[1][0],handLandmarks[2][1]-handLandmarks[1][1]] - if((np.dot(V1,V2))>0 and (handLandmarks[4][1]>handLandmarks[2][1])): - i=1 - elif(np.dot(V1,V2)>0 and handLandmarks[4][1]0 and j2>0): + return etatDuPouce[0] + V1=[handLandmarks[4][0]-handLandmarks[1][0],handLandmarks[4][1]-handLandmarks[1][1]] + V2=[handLandmarks[2][0]-handLandmarks[1][0],handLandmarks[2][1]-handLandmarks[1][1]] + if((np.dot(V1,V2))>0 and (handLandmarks[4][1]>handLandmarks[2][1])): + i=1 + elif(np.dot(V1,V2)>0 and handLandmarks[4][1] self.BUFFER_LENGTH): - self.resultBuffer.pop(0) - - thumbsUpCount = sum(map(lambda x : x == "thumbs_up", self.resultBuffer)) - thumbsDownCount = sum(map(lambda x : x == "thumbs_down", self.resultBuffer)) + self.resultBuffer.append(thumbState) + if(len(self.resultBuffer) > self.BUFFER_LENGTH): + self.resultBuffer.pop(0) + + thumbsUpCount = sum(map(lambda x : x == "thumbs_up", self.resultBuffer)) + thumbsDownCount = sum(map(lambda x : x == "thumbs_down", self.resultBuffer)) - if(thumbsUpCount > self.DETECTION_THRESHOLD * self.BUFFER_LENGTH): - result = "thumbs_up" - elif(thumbsDownCount > self.DETECTION_THRESHOLD * self.BUFFER_LENGTH): - result = "thumbs_down" - else: - result = False + if(thumbsUpCount > self.DETECTION_THRESHOLD * self.BUFFER_LENGTH): + result = "thumbs_up" + elif(thumbsDownCount > self.DETECTION_THRESHOLD * self.BUFFER_LENGTH): + result = "thumbs_down" + else: + result = False - progress = 0 - if thumbState == "thumbs_up": - progress = thumbsUpCount / (self.BUFFER_LENGTH * self.DETECTION_THRESHOLD) - elif thumbState == "thumbs_down": - progress = thumbsDownCount / (self.BUFFER_LENGTH * self.DETECTION_THRESHOLD) - - if(thumbState != "neutre"): - return thumbState, handLandmarks[9], np.linalg.norm(np.array(handLandmarks[9]) - np.array(handLandmarks[0])), result, progress - return False + progress = 0 + if thumbState == "thumbs_up": + progress = thumbsUpCount / (self.BUFFER_LENGTH * self.DETECTION_THRESHOLD) + elif thumbState == "thumbs_down": + progress = thumbsDownCount / (self.BUFFER_LENGTH * self.DETECTION_THRESHOLD) + + if(thumbState != "neutre"): + return thumbState, handLandmarks[9], np.linalg.norm(np.array(handLandmarks[9]) - np.array(handLandmarks[0])), result, progress + return False + +class FingerCountDetector(): + def __init__(self): + self.BUFFER_LENGTH = 20 + self.DETECTION_THRESHOLD = 1/2 + self.hands = mp_hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5) + self.buffer = [] + + def reset(self): + self.buffer = [] + + def getResult(self): + stats = [0] * 10 + for grade in self.buffer: + stats[grade-1] = stats[grade-1]+1 + stats = [stat / self.BUFFER_LENGTH for stat in stats] + print(stats) + + if max(stats) > self.DETECTION_THRESHOLD: + return stats.index(max(stats)) + 1 + + def prodScalaire(self,V1,V2): + return (V1[0]*V2[0]+V1[1]*V2[1])/((V1[0]**2+V1[1]**2)**(1/2)*(V2[0]**2+V2[1]**2)**(1/2)) #produit scalaire normalisé + +#Fait le prod scalaire entre deux vecteurs formées par les points d'index (id0,id1) et (id2,id3) dans la liste landmarks + def prodScalaireDoigts(self,landmarks,id0,id1,id2,id3): + V0= [landmarks[id0].x - landmarks[id1].x, landmarks[id0].y - landmarks[id1].y] + V1= [landmarks[id2].x - landmarks[id3].x, landmarks[id2].y - landmarks[id3].y] + return self.prodScalaire(V0,V1) +# initialisation de la caméra + +#Donne le nombre de doigts levé pour un landmak de main donnée + def analyseMain(self,hand_landmarks): + finger_count = 0 + pouce = self.prodScalaireDoigts(hand_landmarks,2,0,4,2) + index = self.prodScalaireDoigts(hand_landmarks,8,6,6,0) + majeur = self.prodScalaireDoigts(hand_landmarks,12,10,10,0) + annulaire= self.prodScalaireDoigts(hand_landmarks,16,14,14,0) + auriculaire = self.prodScalaireDoigts(hand_landmarks,20,18,18,0) + + if pouce > 0.905135675: + finger_count += 1 + if index > 0: + finger_count += 1 + if majeur > 0: + finger_count += 1 + if annulaire > 0: + finger_count += 1 + if auriculaire > 0: + finger_count += 1 + return finger_count + + def loop(self): + if cap.isOpened(): + # lecture de la vidéo + ret, frame = cap.read() + + # conversion de l'image en RGB + image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # détection des mains + results = self.hands.process(image) + + if results.multi_hand_landmarks: + # détection du nombre de doigts levés + finger_count = 0 + if len(results.multi_hand_landmarks) >0 : + finger_count += self.analyseMain(results.multi_hand_landmarks[0].landmark) + if len(results.multi_hand_landmarks) >1 : + finger_count += self.analyseMain(results.multi_hand_landmarks[1].landmark) + self.buffer.append(finger_count) + if(len(self.buffer) > self.BUFFER_LENGTH): + self.buffer.pop(0) + return self.getResult() + if __name__ == "__main__": - h = HandDetector() - while(1): - print(h.detect()) + h = FingerCountDetector() + while(1): + print(h.loop()) diff --git a/code/backend_reconnaissance/manager.py b/code/backend_reconnaissance/manager.py index a992d11..d87469c 100644 --- a/code/backend_reconnaissance/manager.py +++ b/code/backend_reconnaissance/manager.py @@ -1,4 +1,4 @@ -from hand_detector import HandDetector +from hand_detector import HandDetector, FingerCountDetector from audio_detector import record, analyze, test from network import ApiClient, WebsocketServer import time @@ -20,10 +20,14 @@ class Manager(): self.server = WebsocketServer(None) self.server.start() self.handDetector = HandDetector() + self.fingerCountDetector = FingerCountDetector() self.api = ApiClient() self.timeLastChange = time.time() self.isLastHandPacketEmpty = False self.recongizer = sr.Recognizer() + #For step 2 + self.criteria_list = self.api.get_criteria() + self.currentCriteria = None print("Backend ready") #Boucle principale @@ -34,14 +38,16 @@ class Manager(): if(self.state == 1): self.camera() if(self.state == 2): - self.audio() + self.grade() if(self.state == 3): + self.audio() + if(self.state == 4): self.thankYou() time.sleep(0.01) #Fonction qui est executée pendant que la borne est en veille, reveille la borne si une main est detectée def sleep(self): - res = self.handDetector.detect() + res = self.handDetector.loop() if(res != False): self.state = 1 self.timeLastChange = time.time() @@ -54,7 +60,7 @@ class Manager(): self.reset() return - res = self.handDetector.detect() + res = self.handDetector.loop() if(res != False): state, coords, size, finalDecision, progress = res self.server.sendMessage({"type": "effects", "effects": [{"type": "loading", "x":coords[0], "y": coords[1], "width": size, "height": size, "progress": progress},{"type": state, "x":coords[0], "y": coords[1], "width": size, "height": size}]}) @@ -68,6 +74,24 @@ class Manager(): self.server.sendMessage({"type":"effects","effects":[]}) self.isLastHandPacketEmpty = True + def grade(self): + if(self.currentCriteria == None): + self.currentCriteria = 0 + self.server.sendMessage({"type":"new_criteria","criteria":self.criteria_list[self.currentCriteria]}) + return + + grade = self.fingerCountDetector.loop() + if(grade != None): + self.avis["notes_autres"][self.criteria_list[self.currentCriteria]] = grade + self.currentCriteria+=1 + if(self.currentCriteria < len(self.criteria_list)): + self.server.sendMessage({"type":"new_criteria","criteria":self.criteria_list[self.currentCriteria]}) + self.fingerCountDetector.reset() + else: + self.state = 3 + self.server.sendMessage({"type": "state", "state": 3}) + + def audio(self): time.sleep(3) self.server.sendMessage({"type":"recording_started"}) @@ -96,8 +120,8 @@ class Manager(): #On passe a la suite time.sleep(5) - self.state = 3 - self.server.sendMessage({"type": "state", "state": 3}) + self.state = 4 + self.server.sendMessage({"type": "state", "state": 4}) def thankYou(self): #On envoie l'avis à l'api @@ -111,6 +135,7 @@ class Manager(): def reset(self): self.state = 0 + self.currentCriteria = None self.avis = self.defualtAvis self.handDetector.reset() diff --git a/code/backend_reconnaissance/network.py b/code/backend_reconnaissance/network.py index 08501c6..7ed6369 100644 --- a/code/backend_reconnaissance/network.py +++ b/code/backend_reconnaissance/network.py @@ -41,7 +41,6 @@ class ApiClient(): self.port = port def send(self,note,commentaire): - #Exemple ajout d'un commentaire depuis la borne (site ou geste) avis = { "note": note, @@ -50,3 +49,11 @@ class ApiClient(): } print(self.host, self.port) return requests.post("http://"+self.host+":"+self.port+"/add_review", data=avis) + def get_criteria(self): + res = [] + crits = json.loads(requests.get("http://"+self.host+":"+self.port+"/borne/get_criteres").text) + for c in crits: + res.append(c["nom"]) + return res + +print(ApiClient().get_criteria()) diff --git a/code/reviews_api/borne/get_handler.js b/code/reviews_api/borne/get_handler.js index 004efb8..10b6af9 100644 --- a/code/reviews_api/borne/get_handler.js +++ b/code/reviews_api/borne/get_handler.js @@ -64,7 +64,7 @@ const getReviewFromId = (id) => { const getCriteres = () => { return new Promise((resolve, reject) => { let sql = `SELECT * FROM borne_criteres`; - conn.query(sql, [limit], (err, res) => { + conn.query(sql, (err, res) => { if (err) { reject(err); } else { @@ -237,4 +237,4 @@ export const handleGetStats = (req, res) => { res.status(500).send("Error: " + err.message); }); } -} \ No newline at end of file +}