From d896767543954252d7bb33650e5db30c9b3ed475 Mon Sep 17 00:00:00 2001 From: Quentin Roussel Date: Wed, 22 Mar 2023 14:39:56 +0100 Subject: [PATCH] =?UTF-8?q?Int=C3=A9gration=20de=20la=20reco=20d'image=20?= =?UTF-8?q?=C3=A0=20l'interface=20borne?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- code/Reconnaissance Image/.gitkeep | 0 code/Reconnaissance Image/Fingers5.py | 75 ------------- .../compteurDoigt.py | 0 code/backend_reconnaissance/hands.py | 100 ++++++++++++------ code/backend_reconnaissance/main.py | 34 +++--- .../reconnaissancePouce.py | 0 code/backend_reconnaissance/requirements.txt | 3 +- code/docker-compose.yaml | 56 +++++----- code/interface_borne/assets/js/camera_page.js | 6 +- code/interface_borne/assets/js/network.js | 6 +- .../assets/js/state_manager.js | 7 +- 11 files changed, 131 insertions(+), 156 deletions(-) delete mode 100644 code/Reconnaissance Image/.gitkeep delete mode 100644 code/Reconnaissance Image/Fingers5.py rename code/{Interface_Lounes => backend_reconnaissance}/compteurDoigt.py (100%) rename code/{Interface_Lounes => backend_reconnaissance}/reconnaissancePouce.py (100%) diff --git a/code/Reconnaissance Image/.gitkeep b/code/Reconnaissance Image/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/code/Reconnaissance Image/Fingers5.py b/code/Reconnaissance Image/Fingers5.py deleted file mode 100644 index 2021ba4..0000000 --- a/code/Reconnaissance Image/Fingers5.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -from torchvision import datasets, transforms -import matplotlib.pyplot as plt - -trainSet = datasets.ImageFolder(r'C:\Users\kesha\Desktop\TelecomParis\PACT\DownloadedDataset\train', - transform = transforms.ToTensor()) -valSet = datasets.ImageFolder(r'C:\Users\kesha\Desktop\TelecomParis\PACT\DownloadedDataset\val', - transform = transforms.ToTensor()) - -trainloader = torch.utils.data.DataLoader(trainSet, - batch_size = 50, - shuffle = True) - -valloader = torch.utils.data.DataLoader(valSet, - batch_size = 50, - shuffle = True) - -class Net(nn.Module): - def __init__(self): - super().__init__() - #nn.Conv2d(channels_in, out_channels/number of filters, kernel size) - self.conv1 = nn.Conv2d(3, 16, 3) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(16, 32, 3) - self.conv3 = nn.Conv2d(32, 64, 3) - self.fc1 = nn.Linear(64*14*14, 16) - self.fc2 = nn.Linear(16, 6) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - #size = 16*126*126 then 16*63*63 - x = self.pool(F.relu(self.conv2(x))) - #size = 32*61*61 then 32*30*30 - x = self.pool(F.relu(self.conv3(x))) - #size = 64*28*28 then 64*14*14 - x = torch.flatten(x, 1) - x = F.relu(self.fc1(x)) - x = self.fc2(x) - return x - -net = Net() -print(net) - -criterion = nn.CrossEntropyLoss() -optimizer = optim.RMSprop(net.parameters(), lr=0.001) - -device = torch.device('cuda') -for epoch in range(1, 7): - print('Starting epoch ' + str(epoch)) - current_loss = 0 - Epoch = [] - Loss = [] - for i, data in enumerate(trainloader, 0): - inputs, labels = data - - #très important - optimizer.zero_grad() - - output = net(inputs) - loss = criterion(output, labels) - loss.backward() - optimizer.step() - - current_loss += loss.item() - print('epoch: ', epoch, " loss: ", current_loss) - Loss.append(current_loss) - Epoch.append(epoch) - -plt.plot(Epoch, Loss) -plt.title('Valeur de la fonction cost en fonction de l\'epoch') -plt.show() -#to save a model: torch.save(net.state_dict(), file_location) diff --git a/code/Interface_Lounes/compteurDoigt.py b/code/backend_reconnaissance/compteurDoigt.py similarity index 100% rename from code/Interface_Lounes/compteurDoigt.py rename to code/backend_reconnaissance/compteurDoigt.py diff --git a/code/backend_reconnaissance/hands.py b/code/backend_reconnaissance/hands.py index 2087d26..dab097c 100644 --- a/code/backend_reconnaissance/hands.py +++ b/code/backend_reconnaissance/hands.py @@ -1,44 +1,78 @@ import cv2 import mediapipe as mp +import numpy as np + mp_drawing = mp.solutions.drawing_utils mp_drawing_styles = mp.solutions.drawing_styles mp_hands = mp.solutions.hands - -# For webcam input: cap = cv2.VideoCapture(0) hands = mp_hands.Hands( - model_complexity=0, - min_detection_confidence=0.5, - min_tracking_confidence=0.5) + model_complexity=0, + min_detection_confidence=0.5, + min_tracking_confidence=0.5) +BUFFER_LENGTH = 30 +TH_FRACTION = 3/4 +resultBuffer = [] -def frame(): - if cap.isOpened(): - success, image = cap.read() - if not success: - print("Ignoring empty camera frame.") - # If loading a video, use 'break' instead of 'continue'. - return +def reconnaissancePouce(handLandmarks): + etatDuPouce=["neutre","thumbs_down","thumbs_up"] + i=0 + j=0 + for cpt in range (0,4): + V1=[handLandmarks[(4*cpt)+6][0]-handLandmarks[(4*cpt)+5][0],handLandmarks[(4*cpt)+6][1]-handLandmarks[(4*cpt)+5][1]] + V2=[handLandmarks[(4*cpt)+8][0]-handLandmarks[(4*cpt)+6][0],handLandmarks[(4*cpt)+8][1]-handLandmarks[(4*cpt)+6][1]] + j=np.dot(V1,V2) + if (j>0): + return etatDuPouce[0] + V1=[handLandmarks[4][0]-handLandmarks[1][0],handLandmarks[4][1]-handLandmarks[1][1]] + V2=[handLandmarks[2][0]-handLandmarks[1][0],handLandmarks[2][1]-handLandmarks[1][1]] + if((np.dot(V1,V2))>0 and (handLandmarks[4][1]>handLandmarks[2][1])): + i=1 + elif(np.dot(V1,V2)>0 and handLandmarks[4][1] BUFFER_LENGTH): + resultBuffer.pop(0) + + thumbsUpCount = sum(map(lambda x : x == "thumbs_up", resultBuffer)) + thumbsDownCount = sum(map(lambda x : x == "thumbs_down", resultBuffer)) + + print(thumbsUpCount,thumbsDownCount) + + if(thumbsUpCount > TH_FRACTION * BUFFER_LENGTH): + result = "thumbs_up" + elif(thumbsDownCount > TH_FRACTION * BUFFER_LENGTH): + result = "thumbs_down" + else: + result = False + + if(thumbState != "neutre"): + return thumbState, handLandmarks[9], np.linalg.norm(np.array(handLandmarks[9]) - np.array(handLandmarks[0])), result + return False + diff --git a/code/backend_reconnaissance/main.py b/code/backend_reconnaissance/main.py index 6b37cc4..1f37631 100644 --- a/code/backend_reconnaissance/main.py +++ b/code/backend_reconnaissance/main.py @@ -4,12 +4,14 @@ import math import websockets import random import os -import hands import time +from hands import getThumbState + -values = [] class WebsocketServer: def __init__(self,getEffects,port=os.getenv("PORT"),host=os.getenv("HOST")) -> None: + self.thumbResult = None + self.state = 0 self.host = host self.port = port self.getEffects = getEffects @@ -21,19 +23,23 @@ class WebsocketServer: async def handler(self,websocket): while True: - start = time.time() - messages = self.getEffects() - hands.frame() - await websocket.send(json.dumps(messages)) - # await asyncio.sleep(1/30) - delay = time.time() - start - values.append(1/delay) - avg = sum(values) / len(values) - dev = [(v - avg) ** 2 for v in values] - print(avg, math.sqrt(sum(dev)/len(dev))) -#Remplacer ça par la fonction qui récupère les effets (dans le module de reconnaissance de gestes) + if(self.state == 0): + messages, result = self.getEffects() + if(messages != False): + if(result == False): + await websocket.send(json.dumps(messages)) + else: + self.thumbResult = result + self.state = 1 + await websocket.send('{"type":"state","state":2}') + def getEffects(): - return {"type": "effects", "effects": [{"type": "thumbs_up", "x":random.randint(0,100), "y": random.randint(0,100), "width": 50, "height": 50}]} + res = getThumbState() + if(res != False): + state, coords, size, result = res + return {"type": "effects", "effects": [{"type": state, "x":coords[0], "y": coords[1], "width": size, "height": size}]}, result + else: + return False,False server = WebsocketServer(getEffects) asyncio.run(server.run()) \ No newline at end of file diff --git a/code/Interface_Lounes/reconnaissancePouce.py b/code/backend_reconnaissance/reconnaissancePouce.py similarity index 100% rename from code/Interface_Lounes/reconnaissancePouce.py rename to code/backend_reconnaissance/reconnaissancePouce.py diff --git a/code/backend_reconnaissance/requirements.txt b/code/backend_reconnaissance/requirements.txt index 2376504..6319855 100644 --- a/code/backend_reconnaissance/requirements.txt +++ b/code/backend_reconnaissance/requirements.txt @@ -1,4 +1,5 @@ websockets requests opencv-python -mediapipe \ No newline at end of file +mediapipe +numpy \ No newline at end of file diff --git a/code/docker-compose.yaml b/code/docker-compose.yaml index 6d95b6c..9b6cbdd 100644 --- a/code/docker-compose.yaml +++ b/code/docker-compose.yaml @@ -54,14 +54,14 @@ services: build: ./reviews_api restart: always - #Serveur web de l'interface de la borne - # interface_borne: - # image: httpd:latest - # volumes: - # - ./interface_borne:/usr/local/apache2/htdocs/ - # container_name: interface_borne - # ports: - # - 8888:80 + # Serveur web de l'interface de la borne + interface_borne: + image: httpd:latest + volumes: + - ./interface_borne:/usr/local/apache2/htdocs/ + container_name: interface_borne + ports: + - 8888:80 #Serveur web de l'interface admin interface_admin: @@ -75,24 +75,24 @@ services: # #Backend de la borne : scripts pythons de reconnaissances video et audio # #Envoient les infos a l'interface de la borne par websocket pour mettre a jour l'interface rapidement # #Met a jour les avis en faisant des requêtes a l'API - # backend_reconnaissance: - # build: ./backend_reconnaissance - # container_name: backend_reconnaissance - # restart: always - # devices: - # - /dev/video3:/dev/video0 - # environment: - # - PORT=5000 - # - HOST=backend_reconnaissance - # ports: - # #Ce container est le serveur websocker dont le client est l'interface de la borne qui tourne dans le navigateur - # - 5000:5000 + backend_reconnaissance: + build: ./backend_reconnaissance + container_name: backend_reconnaissance + restart: always + devices: + - /dev/video3:/dev/video0 + environment: + - PORT=5000 + - HOST=backend_reconnaissance + ports: + #Ce container est le serveur websocker dont le client est l'interface de la borne qui tourne dans le navigateur + - 5000:5000 - # video_loopback: - # build: ./video_loopback - # container_name: video_loopback - # restart: always - # devices: - # - /dev/video0:/dev/video0 - # - /dev/video2:/dev/video1 - # - /dev/video3:/dev/video2 \ No newline at end of file + video_loopback: + build: ./video_loopback + container_name: video_loopback + restart: always + devices: + - /dev/video0:/dev/video0 + - /dev/video2:/dev/video1 + - /dev/video3:/dev/video2 \ No newline at end of file diff --git a/code/interface_borne/assets/js/camera_page.js b/code/interface_borne/assets/js/camera_page.js index be3e262..9bdd61c 100644 --- a/code/interface_borne/assets/js/camera_page.js +++ b/code/interface_borne/assets/js/camera_page.js @@ -84,7 +84,6 @@ class CameraPage { _frame() { if (this.streaming && this.enabled && this.width && this.height) { this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); - // this.ctx.drawImage(this.video, 0, 0, this.width, this.height); this._drawEffects(); } if (this.enabled) { @@ -106,6 +105,11 @@ class CameraPage { _drawEffects() { for (let effect of this.activeEffects) { let { x, y, width, height } = this._scaleEffect(effect.x, effect.y, effect.width, effect.height); + width = width * this.videoWidth * 2; + height = height * this.videoHeight * 2; + x = x * this.videoWidth - width / 2; + y = y * this.videoHeight - height / 2; + console.log(width, height); if (effect.type == "thumbs_down") { this._drawThumbsDown(x, y, width, height); } diff --git a/code/interface_borne/assets/js/network.js b/code/interface_borne/assets/js/network.js index 7f9a113..38da92c 100644 --- a/code/interface_borne/assets/js/network.js +++ b/code/interface_borne/assets/js/network.js @@ -3,14 +3,16 @@ class WebsocketClient { this.socket = new WebSocket("ws://localhost:5000"); this.socket.addEventListener("open", (event) => { this.socket.send("connected"); + console.log("connected") }); - this.socket.addEventListener("message", (event) => { + + this.socket.onmessage = (event) => { let msg = JSON.parse(event.data); if (msg.type == "effects") { onNewEffects(msg.effects); }else if(msg.type == "state") { onNewState(msg.state); } - }); + }; } } \ No newline at end of file diff --git a/code/interface_borne/assets/js/state_manager.js b/code/interface_borne/assets/js/state_manager.js index 3d72b56..7dccdd1 100644 --- a/code/interface_borne/assets/js/state_manager.js +++ b/code/interface_borne/assets/js/state_manager.js @@ -14,8 +14,11 @@ class StateManager { this._thankYouPage = new ThankYouPage(); this.wsClient = new WebsocketClient( - (effects) => this._cameraPage.setEffects(effects), - (state) => this.changeState(state) + (effects) => { + this.setState(STATE.video); + this._cameraPage.setEffects(effects) + }, + (state) => this.setState(state) ); this._sleepingPage.enabled = true;