feat: introduced web socket connection

package.json

@@ -24,6 +24,7 @@
     "react-scripts": "5.0.1",
     "react-toastify": "^10.0.4",
     "react-webcam": "^7.2.0",
+    "socket.io-client": "^4.7.5",
     "sweetalert2": "^11.11.0",
     "typescript": "5.1.6",
     "web-vitals": "^3.5.2"

src/Components/MeetingLayout/MeetingLayout.tsx

 import React, { useEffect, useRef, useState } from 'react'
-import { Box } from '@mui/material'
+import { Box, Typography } from '@mui/material'
 import { useTheme, type Theme } from '@mui/material/styles'
 import { useRemoteUsers, usePublish, useRTCClient, useClientEvent } from 'agora-rtc-react'
 import SymmetricGrid from '../SymmetricGrid/SymmetricGrid'
@@ -10,6 +10,7 @@ import type {
   IMicrophoneAudioTrack,
 } from 'agora-rtc-sdk-ng'
 import ControlCenter from '../ControlCenter/ControlCenter'
+import { socketService } from '../../Services/SocketService/socketService'
 
 interface Props {
   setJoined: (x: boolean) => void
@@ -31,6 +32,15 @@ const MeetingLayout = ({ setJoined, localCameraTrack, localMicrophoneTrack }: Pr
   const [isPinned, setIsPinned] = useState(false)
   const [pinnedUser, setPinnedUser] = useState<IAgoraRTCRemoteUser | null>(null)
   const [isCaptionEnable, setIsCaptionEnable] = useState(false)
+  const [caption, setCaption] = useState('')
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      socketService.getCaption(setCaption)
+    }, 1000)
+
+    return () => clearInterval(interval)
+  }, [])
 
   useEffect(() => {
     setTotalUsers(remoteUsers.length + 1)
@@ -101,7 +111,17 @@ const MeetingLayout = ({ setJoined, localCameraTrack, localMicrophoneTrack }: Pr
             borderRadius: '20px',
             marginRight: '20px',
           }}
-        ></Box>
+        >
+          <Typography
+            sx={{
+              color: theme.palette.common.white,
+              fontSize: '1.2rem',
+              fontWeight: 500,
+            }}
+          >
+            {caption}
+          </Typography>
+        </Box>
       )}
       <Box sx={{ height: '10%', width: '100%', display: 'flex', marginBottom: '40px' }}>
         <ControlCenter

src/Components/SymmetricGrid/SymmetricGrid.tsx

-import React, { useEffect, useState } from 'react'
+import React, { useEffect, useRef, useState } from 'react'
 import { Box, Grid } from '@mui/material'
 import useMediaQuery from '@mui/material/useMediaQuery'
 import { useRemoteUsers } from 'agora-rtc-react'
 import { useTheme, type Theme } from '@mui/material/styles'
 import type { IAgoraRTCRemoteUser, ICameraVideoTrack } from 'agora-rtc-sdk-ng'
 import UserCard from '../UserCard/UserCard'
+import { socketService } from '../../Services/SocketService/socketService'
 
 interface Props {
   totalUsers: number
@@ -30,12 +31,23 @@ const SymmetricGrid = ({
 
   const remoteUsers = useRemoteUsers()
 
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const canvasRef = useRef<HTMLCanvasElement>(null)
+
   const [itemOffset, setItemOffset] = useState(12)
   const [itemHeight, setItemHeight] = useState(230)
   const [numOfItems, setNumOfItems] = useState(1)
 
   const maxUsersDisplay = isLg ? 12 : isMd ? 9 : isSm ? 4 : 2
 
+  useEffect(() => {
+    if (localCameraTrack && videoRef.current) {
+      const stream = localCameraTrack.getMediaStreamTrack()
+      videoRef.current.srcObject = new MediaStream([stream])
+      videoRef.current.play().catch((err) => console.error('Video play failed', err))
+    }
+  }, [localCameraTrack])
+
   useEffect(() => {
     setNumOfItems(totalUsers > maxUsersDisplay ? maxUsersDisplay : totalUsers)
   }, [totalUsers])
@@ -126,6 +138,29 @@ const SymmetricGrid = ({
     }
   }, [totalUsers, isSm, isMd, isLg, gridContainerHeight, numOfItems])
 
+  const captureFrame = () => {
+    if (videoRef.current && canvasRef.current) {
+      const context = canvasRef.current.getContext('2d')
+      if (context && videoRef.current.videoWidth && videoRef.current.videoHeight) {
+        canvasRef.current.width = videoRef.current.videoWidth
+        canvasRef.current.height = videoRef.current.videoHeight
+        context.drawImage(videoRef.current, 0, 0, canvasRef.current.width, canvasRef.current.height)
+
+        // You can now access the image data from the canvas
+        const frame = canvasRef.current.toDataURL('image/jpeg')
+        if (frame) {
+          socketService.sendFrames('test', frame)
+        }
+      }
+    }
+
+    requestAnimationFrame(captureFrame) // Recursively capture the next frame
+  }
+
+  useEffect(() => {
+    captureFrame()
+  }, [])
+
   return (
     <Box sx={{ display: 'flex', width: '100%' }}>
       <Grid container spacing={3} width={'100%'} padding={0}>
@@ -137,6 +172,19 @@ const SymmetricGrid = ({
             setPinnedUser={setPinnedUser}
             localCameraTrack={localCameraTrack}
           />
+          <video ref={videoRef} style={{ display: 'none' }}></video>
+          <canvas ref={canvasRef} style={{ display: 'none' }}></canvas>
+
+          {/* <Box
+            sx={{
+              backgroundColor: '#262625',
+              height: `${itemHeight}px`,
+              borderRadius: '15px',
+              width: '100%',
+              position: 'relative',
+              overflow: 'hidden',
+            }}
+          ></Box> */}
         </Grid>
         {remoteUsers.map((remoteUser, index) => (
           <Grid item xs={itemOffset} key={index}>

src/Services/AgoraManager/AgoraManager.tsx

@@ -6,6 +6,7 @@ import { useTheme, type Theme } from '@mui/material/styles'
 import MeetingLoading from '../../Components/MeetingLoading/MeetingLoading'
 import MeetingLayout from '../../Components/MeetingLayout/MeetingLayout'
 import { useParams } from 'react-router-dom'
+import { socketService } from '../SocketService/socketService'
 
 interface AgoraContextType {
   localCameraTrack: ICameraVideoTrack | null
@@ -58,6 +59,14 @@ export const AgoraManager = ({
   const { isLoading: isLoadingCam, localCameraTrack } = useLocalCameraTrack()
   const { isLoading: isLoadingMic, localMicrophoneTrack } = useLocalMicrophoneTrack()
 
+  useEffect(() => {
+    socketService.connect(`http://localhost:5006?room=${channel}`)
+
+    return () => {
+      socketService.disconnect()
+    }
+  }, [])
+
   useEffect(() => {
     localStorage.setItem('myUid', uid.toString())
   }, [uid])

src/Services/SocketService/socketService.ts

+import { io, type Socket } from 'socket.io-client'
+
+class SocketService {
+  private socket: Socket | undefined
+
+  public connect(serverUrl: string): void {
+    this.socket = io(serverUrl)
+
+    this.socket.on('connect', () => {
+      console.log('Connected to Socket.IO server')
+    })
+  }
+
+  public disconnect(): void {
+    if (this.socket) {
+      this.socket.disconnect()
+    }
+  }
+
+  public joinRoom(room: string): void {
+    this?.socket?.emit('join_room', { room: room, username: 'default' })
+  }
+
+  public getCaption(setCaption: (caption: string) => void): void {
+    this?.socket?.on('frame', (res) => {
+      console.log('response', res)
+      if (res.result) {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        setCaption(res.result)
+      }
+    })
+  }
+
+  public sendFrames(room: string, frame: any): void {
+    this?.socket?.emit('frame', { room: room, frame: frame })
+  }
+}
+
+export const socketService = new SocketService()

yarn.lock

@@ -1983,6 +1983,11 @@
   dependencies:
     "@sinonjs/commons" "^1.7.0"
 
+"@socket.io/component-emitter@~3.1.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553"
+  integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==
+
 "@surma/rollup-plugin-off-main-thread@^2.2.3":
   version "2.2.3"
   resolved "https://registry.npmjs.org/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz"
@@ -4064,7 +4069,7 @@ debug@2.6.9, debug@^2.6.0:
   dependencies:
     ms "2.0.0"
 
-debug@4, debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
+debug@4, debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
   version "4.3.4"
   resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz"
   integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@@ -4406,6 +4411,22 @@ encodeurl@~1.0.2:
   resolved "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz"
   integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==
 
+engine.io-client@~6.5.2:
+  version "6.5.3"
+  resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.5.3.tgz#4cf6fa24845029b238f83c628916d9149c399bc5"
+  integrity sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==
+  dependencies:
+    "@socket.io/component-emitter" "~3.1.0"
+    debug "~4.3.1"
+    engine.io-parser "~5.2.1"
+    ws "~8.11.0"
+    xmlhttprequest-ssl "~2.0.0"
+
+engine.io-parser@~5.2.1:
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.2.2.tgz#37b48e2d23116919a3453738c5720455e64e1c49"
+  integrity sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw==
+
 enhanced-resolve@^5.15.0:
   version "5.15.0"
   resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz"
@@ -9166,6 +9187,24 @@ slice-ansi@^7.0.0:
     ansi-styles "^6.2.1"
     is-fullwidth-code-point "^5.0.0"
 
+socket.io-client@^4.7.5:
+  version "4.7.5"
+  resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.7.5.tgz#919be76916989758bdc20eec63f7ee0ae45c05b7"
+  integrity sha512-sJ/tqHOCe7Z50JCBCXrsY3I2k03iOiUe+tj1OmKeD2lXPiGH/RUCdTZFoqVyN7l1MnpIzPrGtLcijffmeouNlQ==
+  dependencies:
+    "@socket.io/component-emitter" "~3.1.0"
+    debug "~4.3.2"
+    engine.io-client "~6.5.2"
+    socket.io-parser "~4.2.4"
+
+socket.io-parser@~4.2.4:
+  version "4.2.4"
+  resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83"
+  integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==
+  dependencies:
+    "@socket.io/component-emitter" "~3.1.0"
+    debug "~4.3.1"
+
 sockjs@^0.3.24:
   version "0.3.24"
   resolved "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz"
@@ -10536,6 +10575,11 @@ ws@^8.13.0:
   resolved "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz"
   integrity sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==
 
+ws@~8.11.0:
+  version "8.11.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143"
+  integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==
+
 xml-name-validator@^3.0.0:
   version "3.0.0"
   resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz"
@@ -10546,6 +10590,11 @@ xmlchars@^2.2.0:
   resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz"
   integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
 
+xmlhttprequest-ssl@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67"
+  integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==
+
 y18n@^5.0.5:
   version "5.0.8"
   resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"