feat: introduced features

src/Components/Lobby/Lobby.tsx

@@ -8,23 +8,44 @@ import {
   IconButton,
   TextField,
   Typography,
+  Popover,
 } from '@mui/material'
 import ChevronRightIcon from '@mui/icons-material/ChevronRight'
 import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown'
 import { useTheme, type Theme } from '@mui/material/styles'
 import { useState } from 'react'
 import { useNavigate } from 'react-router-dom'
+import lips from '../../Assests/Images/lips.png'
+import sign from '../../Assests/Images/hello.png'
+import avatar from '../../Assests/Images/gamer.png'
+import { IFeatureTypes } from '../../Services/types/FeatureTypes'
 
 interface Props {
   setJoined: (x: boolean) => void
+  currentFeature: IFeatureTypes | null
+  setCurrentFeature: (x: IFeatureTypes | null) => void
 }
 
-const Lobby = ({ setJoined }: Props) => {
+const Lobby = ({ setJoined, currentFeature, setCurrentFeature }: Props) => {
   const theme: Theme = useTheme()
   const navigate = useNavigate()
 
   const [roomName, setRoomName] = useState('')
 
+  const [anchorEl, setAnchorEl] = useState<HTMLElement | null>(null)
+  const [toolTipLabel, setToolTipLabel] = useState<string>('')
+
+  const handlePopoverOpen = (event: React.MouseEvent<HTMLElement>, label: string) => {
+    setToolTipLabel(label)
+    setAnchorEl(event.currentTarget)
+  }
+
+  const handlePopoverClose = () => {
+    setAnchorEl(null)
+  }
+
+  const open = Boolean(anchorEl)
+
   const handleJoinRoom = () => {
     navigate(`/${roomName}`)
     setJoined(true)
@@ -36,6 +57,10 @@ const Lobby = ({ setJoined }: Props) => {
     setJoined(true)
   }
 
+  const handleSelectFeature = (feature: IFeatureTypes) => {
+    setCurrentFeature(currentFeature === feature ? null : feature)
+  }
+
   return (
     <Box
       sx={{
@@ -157,7 +182,80 @@ const Lobby = ({ setJoined }: Props) => {
                 </Typography>
               </AccordionSummary>
               <AccordionDetails>
-                <Box>sdsd</Box>
+                <Box
+                  sx={{
+                    display: 'flex',
+                    justifyContent: 'space-between',
+                  }}
+                >
+                  <IconButton
+                    sx={{
+                      width: '30%',
+                      cursor: 'pointer',
+                      borderRadius: '15px',
+                      padding: '5px',
+                      '&:hover': { border: '2px solid black' },
+                      border: currentFeature === IFeatureTypes.LIP_READING ? '2px solid black' : '',
+                    }}
+                    onMouseEnter={(event) => handlePopoverOpen(event, 'Lip reading feature')}
+                    onMouseLeave={handlePopoverClose}
+                    onClick={() => handleSelectFeature(IFeatureTypes.LIP_READING)}
+                  >
+                    <img src={lips} alt='lips' style={{ width: '80px' }} />
+                  </IconButton>
+                  <Popover
+                    id='mouse-over-popover'
+                    sx={{
+                      pointerEvents: 'none',
+                    }}
+                    open={open}
+                    anchorEl={anchorEl}
+                    anchorOrigin={{
+                      vertical: 'bottom',
+                      horizontal: 'left',
+                    }}
+                    transformOrigin={{
+                      vertical: 'top',
+                      horizontal: 'left',
+                    }}
+                    onClose={handlePopoverClose}
+                    disableRestoreFocus
+                  >
+                    <Typography sx={{ p: 1 }}>{toolTipLabel}</Typography>
+                  </Popover>
+                  <IconButton
+                    sx={{
+                      width: '30%',
+                      cursor: 'pointer',
+                      borderRadius: '15px',
+                      padding: '5px',
+                      '&:hover': { border: '2px solid black' },
+                      border:
+                        currentFeature === IFeatureTypes.SIGN_LANGUAGE ? '2px solid black' : '',
+                    }}
+                    onMouseEnter={(event) => handlePopoverOpen(event, 'Sign translation feature')}
+                    onMouseLeave={handlePopoverClose}
+                    onClick={() => handleSelectFeature(IFeatureTypes.SIGN_LANGUAGE)}
+                  >
+                    <img src={sign} alt='lips' style={{ width: '80px' }} />
+                  </IconButton>
+                  <IconButton
+                    sx={{
+                      width: '30%',
+                      cursor: 'pointer',
+                      borderRadius: '15px',
+                      padding: '5px',
+                      '&:hover': { border: '2px solid black' },
+                      border:
+                        currentFeature === IFeatureTypes.LIP_SYNCHING ? '2px solid black' : '',
+                    }}
+                    onMouseEnter={(event) => handlePopoverOpen(event, 'Lip synching feature')}
+                    onMouseLeave={handlePopoverClose}
+                    onClick={() => handleSelectFeature(IFeatureTypes.LIP_SYNCHING)}
+                  >
+                    <img src={avatar} alt='lips' style={{ width: '80px' }} />
+                  </IconButton>
+                </Box>
               </AccordionDetails>
             </Accordion>
           </Box>

src/Components/MeetingLayout/MeetingLayout.tsx

@@ -11,14 +11,23 @@ import type {
 } from 'agora-rtc-sdk-ng'
 import ControlCenter from '../ControlCenter/ControlCenter'
 import { socketService } from '../../Services/SocketService/socketService'
+import { IFeatureTypes } from '../../Services/types/FeatureTypes'
 
 interface Props {
   setJoined: (x: boolean) => void
   localCameraTrack: ICameraVideoTrack | null
   localMicrophoneTrack: IMicrophoneAudioTrack | null
+  currentFeature: IFeatureTypes | null
+  setCurrentFeature: (x: IFeatureTypes | null) => void
 }
 
-const MeetingLayout = ({ setJoined, localCameraTrack, localMicrophoneTrack }: Props) => {
+const MeetingLayout = ({
+  setJoined,
+  localCameraTrack,
+  localMicrophoneTrack,
+  currentFeature,
+  setCurrentFeature,
+}: Props) => {
   const theme: Theme = useTheme()
 
   const agoraEngine = useRTCClient()
@@ -28,12 +37,48 @@ const MeetingLayout = ({ setJoined, localCameraTrack, localMicrophoneTrack }: Pr
   const gridContainerRef = useRef<HTMLDivElement>(null)
   const [gridContainerHeight, setGridContainerHeight] = useState(0)
 
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const canvasRef = useRef<HTMLCanvasElement>(null)
+
   const [totalUsers, setTotalUsers] = useState(0)
   const [isPinned, setIsPinned] = useState(false)
   const [pinnedUser, setPinnedUser] = useState<IAgoraRTCRemoteUser | null>(null)
   const [isCaptionEnable, setIsCaptionEnable] = useState(false)
   const [caption, setCaption] = useState('')
 
+  useEffect(() => {
+    if (localCameraTrack && videoRef.current) {
+      const stream = localCameraTrack.getMediaStreamTrack()
+      videoRef.current.srcObject = new MediaStream([stream])
+      videoRef.current.play().catch((err) => console.error('Video play failed', err))
+    }
+  }, [localCameraTrack])
+
+  const captureFrame = () => {
+    if (videoRef.current && canvasRef.current) {
+      const context = canvasRef.current.getContext('2d')
+      if (context && videoRef.current.videoWidth && videoRef.current.videoHeight) {
+        canvasRef.current.width = videoRef.current.videoWidth
+        canvasRef.current.height = videoRef.current.videoHeight
+        context.drawImage(videoRef.current, 0, 0, canvasRef.current.width, canvasRef.current.height)
+
+        // You can now access the image data from the canvas
+        const frame = canvasRef.current.toDataURL('image/jpeg')
+        if (frame && currentFeature === IFeatureTypes.LIP_READING) {
+          socketService.sendFrames('test', frame)
+        }
+      }
+    }
+
+    requestAnimationFrame(captureFrame) // Recursively capture the next frame
+  }
+
+  useEffect(() => {
+    if (currentFeature != null) {
+      captureFrame()
+    }
+  }, [])
+
   useEffect(() => {
     const interval = setInterval(() => {
       socketService.getCaption(setCaption)
@@ -82,6 +127,8 @@ const MeetingLayout = ({ setJoined, localCameraTrack, localMicrophoneTrack }: Pr
         }}
         ref={gridContainerRef}
       >
+        <video ref={videoRef} style={{ display: 'none' }}></video>
+        <canvas ref={canvasRef} style={{ display: 'none' }}></canvas>
         {isPinned && pinnedUser ? (
           <PinnedGrid
             totalUsers={totalUsers}

src/Components/SymmetricGrid/SymmetricGrid.tsx

@@ -5,7 +5,6 @@ import { useRemoteUsers } from 'agora-rtc-react'
 import { useTheme, type Theme } from '@mui/material/styles'
 import type { IAgoraRTCRemoteUser, ICameraVideoTrack } from 'agora-rtc-sdk-ng'
 import UserCard from '../UserCard/UserCard'
-import { socketService } from '../../Services/SocketService/socketService'
 
 interface Props {
   totalUsers: number
@@ -31,23 +30,12 @@ const SymmetricGrid = ({
 
   const remoteUsers = useRemoteUsers()
 
-  const videoRef = useRef<HTMLVideoElement>(null)
-  const canvasRef = useRef<HTMLCanvasElement>(null)
-
   const [itemOffset, setItemOffset] = useState(12)
   const [itemHeight, setItemHeight] = useState(230)
   const [numOfItems, setNumOfItems] = useState(1)
 
   const maxUsersDisplay = isLg ? 12 : isMd ? 9 : isSm ? 4 : 2
 
-  useEffect(() => {
-    if (localCameraTrack && videoRef.current) {
-      const stream = localCameraTrack.getMediaStreamTrack()
-      videoRef.current.srcObject = new MediaStream([stream])
-      videoRef.current.play().catch((err) => console.error('Video play failed', err))
-    }
-  }, [localCameraTrack])
-
   useEffect(() => {
     setNumOfItems(totalUsers > maxUsersDisplay ? maxUsersDisplay : totalUsers)
   }, [totalUsers])
@@ -138,29 +126,6 @@ const SymmetricGrid = ({
     }
   }, [totalUsers, isSm, isMd, isLg, gridContainerHeight, numOfItems])
 
-  const captureFrame = () => {
-    if (videoRef.current && canvasRef.current) {
-      const context = canvasRef.current.getContext('2d')
-      if (context && videoRef.current.videoWidth && videoRef.current.videoHeight) {
-        canvasRef.current.width = videoRef.current.videoWidth
-        canvasRef.current.height = videoRef.current.videoHeight
-        context.drawImage(videoRef.current, 0, 0, canvasRef.current.width, canvasRef.current.height)
-
-        // You can now access the image data from the canvas
-        const frame = canvasRef.current.toDataURL('image/jpeg')
-        if (frame) {
-          socketService.sendFrames('test', frame)
-        }
-      }
-    }
-
-    requestAnimationFrame(captureFrame) // Recursively capture the next frame
-  }
-
-  useEffect(() => {
-    captureFrame()
-  }, [])
-
   return (
     <Box sx={{ display: 'flex', width: '100%' }}>
       <Grid container spacing={3} width={'100%'} padding={0}>
@@ -172,8 +137,6 @@ const SymmetricGrid = ({
             setPinnedUser={setPinnedUser}
             localCameraTrack={localCameraTrack}
           />
-          <video ref={videoRef} style={{ display: 'none' }}></video>
-          <canvas ref={canvasRef} style={{ display: 'none' }}></canvas>
 
           {/* <Box
             sx={{

src/Pages/VideoCall/VideoCall.tsx

@@ -4,21 +4,31 @@ import { AgoraRTCProvider, useRTCClient } from 'agora-rtc-react'
 import { AgoraManager } from '../../Services/AgoraManager/AgoraManager'
 import Lobby from '../../Components/Lobby/Lobby'
 import { Box } from '@mui/material'
+import { type IFeatureTypes } from '../../Services/types/FeatureTypes'
 
 const VideoCall = () => {
   const agoraEngine = useRTCClient(AgoraRTC.createClient({ codec: 'vp8', mode: 'rtc' }))
   const [joined, setJoined] = useState(false)
+  const [currentFeature, setCurrentFeature] = useState<IFeatureTypes | null>(null)
 
   return (
     <Box sx={{ height: '100vh', width: '100vw', overflow: 'hidden' }}>
       {joined ? (
         <AgoraRTCProvider client={agoraEngine}>
-          <AgoraManager setJoined={setJoined}>
+          <AgoraManager
+            setJoined={setJoined}
+            currentFeature={currentFeature}
+            setCurrentFeature={setCurrentFeature}
+          >
             <div>asas</div>
           </AgoraManager>
         </AgoraRTCProvider>
       ) : (
-        <Lobby setJoined={setJoined} />
+        <Lobby
+          setJoined={setJoined}
+          currentFeature={currentFeature}
+          setCurrentFeature={setCurrentFeature}
+        />
       )}
     </Box>
   )

src/Services/AgoraManager/AgoraManager.tsx

@@ -7,6 +7,7 @@ import MeetingLoading from '../../Components/MeetingLoading/MeetingLoading'
 import MeetingLayout from '../../Components/MeetingLayout/MeetingLayout'
 import { useParams } from 'react-router-dom'
 import { socketService } from '../SocketService/socketService'
+import { type IFeatureTypes } from '../types/FeatureTypes'
 
 interface AgoraContextType {
   localCameraTrack: ICameraVideoTrack | null
@@ -34,9 +35,13 @@ export const useAgoraContext = () => {
 
 export const AgoraManager = ({
   setJoined,
+  currentFeature,
+  setCurrentFeature,
   children,
 }: {
   setJoined: (x: boolean) => void
+  currentFeature: IFeatureTypes | null
+  setCurrentFeature: (x: IFeatureTypes | null) => void
   children: React.ReactNode
 }) => {
   const theme: Theme = useTheme()
@@ -100,6 +105,8 @@ export const AgoraManager = ({
             setJoined={setJoined}
             localCameraTrack={localCameraTrack}
             localMicrophoneTrack={localMicrophoneTrack}
+            currentFeature={currentFeature}
+            setCurrentFeature={setCurrentFeature}
           />
         </AgoraProvider>
       ) : joinError ? (

src/Services/SocketService/socketService.ts

@@ -32,7 +32,18 @@ class SocketService {
   }
 
   public sendFrames(room: string, frame: any): void {
-    this?.socket?.emit('frame', { room: room, frame: frame })
+    this?.socket?.emit('calibrate_lips', { room, frame })
+  }
+
+  public calibrateLips(room: string, frame: any): void {
+    this?.socket?.emit('calibrate_lips', { room, frame })
+  }
+
+  public getCalibrateLipsStatus() {
+    this?.socket?.on('calibrate_lips', (res) => {
+      console.log('calibrate_lips', res)
+      return res
+    })
   }
 }
 

src/Services/types/FeatureTypes.ts

+export enum IFeatureTypes {
+  LIP_READING = 'LIP_READING',
+  SIGN_LANGUAGE = 'SIGN_LANGUAGE',
+  LIP_SYNCHING = 'LIP_SYNCHING',
+}