Skip to content

2
2022-074
Commit 91a721fd authored by IT19165226-Arachchige I.D's avatar IT19165226-Arachchige I.D
Browse files
Options

Routine Commit

parent d39a3a4d
Show whitespace changes
Inline Side-by-side
Showing with 2233 additions and 1578 deletions
Compare CSVs.ipynb
View file @ 91a721fd
...@@ -8,7 +8,14 @@ ...@@ -8,7 +8,14 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import csv\n", "import csv\n",
"import os" "import os\n",
"import pandas as pd\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.metrics import mean_absolute_error as mae\n",
"from statistics import mean\n",
"import math"
] ]
}, },
{ {
...@@ -20,180 +27,358 @@ ...@@ -20,180 +27,358 @@
"source": [ "source": [
"##writing details to the CSV\n", "##writing details to the CSV\n",
"\n", "\n",
"def write_to_csv_rot_angles_diff(list_considered, storage_location):\n", "def write_to_csv_rot_angles_diff(storage_location, list1, list2, list3):\n",
" \n", " \n",
" #creating the csv template\n", " #creating the csv template\n",
" with open(storage_location, mode='w', newline='') as headpose_track_file:\n", " with open(storage_location, mode='w', newline='') as headpose_track_file:\n",
" writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n", " writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n",
" \n", " \n",
" writer.writerow(['Frame_number','Mediapipe difference','', '', '', 'MTCNN difference','', ''])\n", " writer.writerow(['Frame_number','Mediapipe difference','', '', '', 'MTCNN difference','', '', '', 'Openface difference','', ''])\n",
" writer.writerow(['','Yaw','Pitch', 'Roll', '', 'Yaw','Pitch', 'Roll'])\n", " writer.writerow(['','Yaw','Pitch', 'Roll', '', 'Yaw','Pitch', 'Roll', '', 'Yaw','Pitch', 'Roll'])\n",
" \n", " \n",
" frameNo = 0\n", " frameNo = 0\n",
" \n", " \n",
" #all_diff - record = (mediapipe, mtcnn) \n", " l = len(list1)\n",
" \n", " \n",
" for record in list_considered:\n", " for record in range (0, l):\n",
" sub_record_mediapipe = record[0]\n", " sub_record_mediapipe = list1[record]\n",
" sub_record_mtcnn = record[1]\n", " sub_record_mtcnn = list2[record]\n",
" sub_record_openface = list3[record]\n",
" \n", " \n",
" frameNo += 1 #increase the frame number\n", " frameNo += 1 #increase the frame number\n",
" \n", " \n",
" #writing the record details to the file\n", " #writing the record details to the file\n",
" writer.writerow([frameNo, sub_record_mediapipe[0], sub_record_mediapipe[1], sub_record_mediapipe[2], '', sub_record_mtcnn[0], sub_record_mtcnn[1], sub_record_mtcnn[2]])" " writer.writerow([frameNo, sub_record_mediapipe[0], sub_record_mediapipe[1], sub_record_mediapipe[2], '', sub_record_mtcnn[0], sub_record_mtcnn[1], sub_record_mtcnn[2], '', sub_record_openface[0], sub_record_openface[1], sub_record_openface[2]])"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 3,
"id": "590a7c58", "id": "97298843",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"#getting the csv values to a usable list\n", "##writing details to the CSV\n",
"\n", "\n",
"def get_val(storage_location, list_to_store):\n", "def write_to_csv_rot_angles_diff_compared_vertical(storage_location, list1, list2, list3):\n",
" \n",
" #creating the csv template\n",
" with open(storage_location, mode='w', newline='') as headpose_track_file:\n",
" writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n",
" \n",
" writer.writerow(['Frame_number','Value', 'Rot_angle', 'Model'])\n",
" \n",
" frameNo = 0\n",
" \n",
" l = len(list1)\n",
" \n",
" for i in range(0, l):\n",
" sub_record_mediapipe = list1[i]\n",
" sub_record_mtcnn = list2[i]\n",
" sub_record_openface = list3[i]\n",
" \n",
" frameNo += 1 #increase the frame number\n",
" \n",
" #writing the record details to the file\n",
" writer.writerow([frameNo, sub_record_mediapipe[0], 'Yaw', 'Mediapipe'])\n",
" writer.writerow([frameNo, sub_record_mediapipe[1], 'Pitch', 'Mediapipe'])\n",
" writer.writerow([frameNo, sub_record_mediapipe[2], 'Roll', 'Mediapipe'])\n",
" \n", " \n",
" with open(storage_location, 'r') as file:\n", " writer.writerow([frameNo, sub_record_mtcnn[0], 'Yaw', 'MTCNN'])\n",
" filecontent=csv.reader(file)\n", " writer.writerow([frameNo, sub_record_mtcnn[1], 'Pitch', 'MTCNN'])\n",
" for row in filecontent:\n", " writer.writerow([frameNo, sub_record_mtcnn[2], 'Roll', 'MTCNN'])\n",
" list_to_store.append(row)" " \n",
" writer.writerow([frameNo, sub_record_openface[0], 'Yaw', 'Openface'])\n",
" writer.writerow([frameNo, sub_record_openface[1], 'Pitch', 'Openface'])\n",
" writer.writerow([frameNo, sub_record_openface[2], 'Roll', 'Openface'])"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": 4,
"id": "179cdd6e", "id": "d16d3616",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def find_difference(list_considered, list_to_store):\n", "def get_only_certain_val(col1, col2, col3, location, list_to_store):\n",
" \n", " data = pd.read_csv(location, usecols = [col1, col2, col3])\n",
" ann_length, other_len = len(annotated_val), len(list_considered)\n", "\n",
" yaw_diff, pitch_diff, roll_diff = 0, 0, 0\n", " df = data[[col1, col2, col3]]\n",
" \n", " #print(df)\n",
" if ann_length > other_len:\n",
" length = other_len\n",
" else:\n",
" length = ann_length\n",
" \n",
" for i in range (1, length):\n",
" annotated_record = annotated_val[i]\n",
" record = list_considered[i]\n",
" \n",
" val1 = float(annotated_record[1]) - float(record[1])\n",
" val2 = float(annotated_record[2]) - float(record[2])\n",
" val3 = float(annotated_record[3]) - float(record[3])\n",
" \n",
" yaw_diff += val1\n",
" pitch_diff += val2\n",
" roll_diff += val3\n",
" \n",
" list_to_store.append((val1, val2, val3))\n",
" \n", " \n",
" return (yaw_diff / length), (pitch_diff / length), (roll_diff / length)" " for i, row in df.iterrows():\n",
" # create a list representing the dataframe row\n",
" row_ls = [row[col1], row[col2], row[col3]]\n",
" # append row list to list_to_store\n",
" list_to_store.append(row_ls)"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 18, "execution_count": 5,
"id": "73ab61c8", "id": "b1dc891d",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def create_all_diff():\n", "def find_diff_frames(list_considered, list_to_store, convert, start, end):\n",
" \n", " \n",
" len1, len2 = len(mediapipe_diff), len(mtcnn_diff)\n", " yaw_diff, pitch_diff, roll_diff, count = 0, 0, 0, 0\n",
" l = end - start\n",
" \n", " \n",
" if len1 > len2:\n", " for i in range (start, end):\n",
" length = len1\n", " annotated_record = annotated_val[i]\n",
" else:\n", " record = list_considered[i]\n",
" length = len2\n",
" \n", " \n",
" for i in range (0, length):\n", " if ((annotated_record[0] == 0) and (annotated_record[1] == 0) and (annotated_record[2] == 0)):\n",
" if (i >= len1):\n", " valY = 0\n",
" sub_record_mediapipe = ('', '', '')\n", " valP = 0\n",
" valR = 0\n",
" else:\n", " else:\n",
" sub_record_mediapipe = mediapipe_diff[i]\n", " count += 1\n",
" \n", " \n",
" if (i >= len2):\n", " temp1 = float(record[0])\n",
" sub_record_mtcnn = ('', '', '')\n", " temp2 = float(record[1])\n",
" else:\n", " temp3 = float(record[2])\n",
" sub_record_mtcnn = mtcnn_diff[i]\n", " \n",
" if convert == 'true':\n",
" #temp1 = np.rad2deg(temp1)\n",
" #temp2 = np.rad2deg(temp2)\n",
" #temp3 = np.rad2deg(temp3)\n",
" \n",
" temp1 = math.degrees(temp1)\n",
" temp2 = math.degrees(temp2)\n",
" temp3 = math.degrees(temp3)\n",
" \n",
" #temp1 *= 360\n",
" #temp2 *= 360\n",
" #temp3 *= 360\n",
" \n", " \n",
" record = (sub_record_mediapipe, sub_record_mtcnn)\n", " valY = abs(float(annotated_record[0]) - temp1)\n",
" valP = abs(float(annotated_record[1]) - temp2)\n",
" valR = abs(float(annotated_record[2]) - temp3)\n",
" \n", " \n",
" all_diff.append(record)\n", " yaw_diff += (valY / l)\n",
" pitch_diff += (valP / l)\n",
" roll_diff += (valR / l)\n",
" \n", " \n",
" " " list_to_store.append((valY, valP, valR))\n",
" \n",
" #return yaw_diff, pitch_diff, roll_diff"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 36, "execution_count": 6,
"id": "151e603d", "id": "6fb1ad6c",
"metadata": {},
"outputs": [],
"source": [
"def append_diff(start, end, list1, list2, list3): \n",
" find_diff_frames(mediapipe_val, list1, 'false', start, end)\n",
" find_diff_frames(mtcnn_val, list2, 'false', start, end)\n",
" find_diff_frames(openface_val, list3, 'true', start, end)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "5ed6b91d",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"##writing details to the CSV\n", "##writing details to the CSV\n",
"\n", "\n",
"def write_to_csv_mean(storage_location):\n", "#def write_to_csv_mean(storage_location, list1, list2, list3, list4, list5, list6, list7, list8, list9):\n",
"def write_to_csv_mean(storage_location, list1, list2, list3):\n",
" \n",
" mp_yaw, mp_pitch, mp_roll = break_list_and_mean(list1)\n",
" mtcnn_yaw, mtcnn_pitch, mtcnn_roll = break_list_and_mean(list2)\n",
" of_yaw, of_pitch, of_roll = break_list_and_mean(list3) \n",
" \n",
" mp_mean = mean([mp_yaw, mp_pitch, mp_roll])\n",
" mtcnn_mean = mean([mtcnn_yaw, mtcnn_pitch, mtcnn_roll])\n",
" of_mean = mean([of_yaw, of_pitch, of_roll])\n",
" \n",
" \n", " \n",
" #creating the csv template\n", " #creating the csv template\n",
" with open(storage_location, mode='w', newline='') as headpose_track_file:\n", " with open(storage_location, mode='w', newline='') as headpose_track_file:\n",
" writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n", " writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n",
" \n", " \n",
" #writer.writerow([video_name])\n",
" writer.writerow(['Model','Yaw', 'Pitch', 'Roll','All'])\n", " writer.writerow(['Model','Yaw', 'Pitch', 'Roll','All'])\n",
" \n", " \n",
" mp_mean = (mp_mean_yaw + mp_mean_pitch + mp_mean_roll) / 3\n",
" mtcnn_mean = (mtcnn_mean_yaw + mtcnn_mean_pitch + mtcnn_mean_roll) / 3\n",
" \n",
" #writing the record details to the file\n", " #writing the record details to the file\n",
" writer.writerow(['Mediapipe', mp_mean_yaw, mp_mean_pitch, mp_mean_roll, mp_mean])\n", " writer.writerow(['Mediapipe', mp_yaw, mp_pitch, mp_roll, mp_mean])\n",
" writer.writerow(['MTCNN', mtcnn_mean_yaw, mtcnn_mean_pitch, mtcnn_mean_roll, mtcnn_mean])" " writer.writerow(['MTCNN', mtcnn_yaw, mtcnn_pitch, mtcnn_roll, mtcnn_mean])\n",
" writer.writerow(['Openface', of_yaw, of_pitch, of_roll, of_mean])"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 40, "execution_count": 8,
"id": "9f360b6c",
"metadata": {},
"outputs": [],
"source": [
"def break_list_and_mean(list1):\n",
" lista, listb, listc = [], [], []\n",
" \n",
" for record in list1:\n",
" lista.append(record[0])\n",
" listb.append(record[1])\n",
" listc.append(record[2])\n",
" \n",
" return mean(lista), mean(listb), mean(listc)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "98b9306a", "id": "98b9306a",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"video_name = '4_A_FT_M'\n", "annotated_val, mediapipe_val, mtcnn_val, openface_val = [], [], [], []\n",
"annotated_val_location = 'test_videos/eyediap/annotated_values/converted_' + video_name + '_ball_tracking.csv'\n", "mediapipe_diff_hoz, mtcnn_diff_hoz, openface_diff_hoz, all_diff_hoz = [], [], [], []\n",
"mediapipe_val_location = 'result_analysis/eyediap/' + video_name + '/' + video_name + '_raw_rotational_angles.csv'\n", "mediapipe_diff_ver, mtcnn_diff_ver, openface_diff_ver, all_diff_ver = [], [], [], []\n",
"mtcnn_val_location = 'result_analysis/mtcnn/eyediap/' + video_name + '/' + video_name + '_MP.csv'\n", "\n",
"def main(number, letter, movement, hoz_segments, ver_segments): \n",
" video_name = number + '_' + letter + '_FT_' + movement\n",
"\n", "\n",
"annotated_val, mediapipe_val, mtcnn_val, mediapipe_diff, mtcnn_diff, all_diff = [], [], [], [], [], []\n", " annotated_val_location = 'test_videos/eyediap/annotated_values/processed/EYEDIAP_' + video_name + '_values.csv'\n",
"mediapipe_mean, mtcnn_mean = [], []\n", " mediapipe_val_location = 'result_analysis/eyediap/' + video_name + '/' + video_name + '_raw.csv'\n",
" mtcnn_val_location = 'result_analysis/mtcnn/eyediap/' + video_name + '/' + video_name + '_MP.csv'\n",
" openface_val_location = 'result_analysis/openface/eyediap/' + video_name + '.csv'\n",
"\n", "\n",
"get_val(annotated_val_location , annotated_val)\n", " get_only_certain_val('yaw', 'pitch', 'roll', annotated_val_location , annotated_val)\n",
"get_val(mediapipe_val_location , mediapipe_val)\n", " get_only_certain_val('Yaw', 'Pitch2', 'Roll', mediapipe_val_location, mediapipe_val)\n",
"get_val(mtcnn_val_location , mtcnn_val)\n", " get_only_certain_val('yaw', 'pitch', 'roll', mtcnn_val_location, mtcnn_val)\n",
" get_only_certain_val(' pose_Ry', ' pose_Rx', ' pose_Rz', openface_val_location, openface_val)\n",
"\n", "\n",
"mp_mean_yaw, mp_mean_pitch, mp_mean_roll = find_difference(mediapipe_val, mediapipe_diff)\n", " length = len(annotated_val) - 1\n",
"mtcnn_mean_yaw, mtcnn_mean_pitch, mtcnn_mean_roll = find_difference(mtcnn_val, mtcnn_diff)\n",
"\n", "\n",
"create_all_diff()\n", " #finding horizontal differences\n",
" #hoz_segments = [(229, 620), (2042, 2883)]\n",
" for hseg in hoz_segments:\n",
" append_diff(hseg[0], hseg[1], mediapipe_diff_hoz, mtcnn_diff_hoz, openface_diff_hoz)\n",
"\n", "\n",
"#saving generated data\n", " #finding vertical differences\n",
"output_folder = 'comparison/'\n", " #ver_segments = [(709, 1411), (3002, 3392)]\n",
" for vseg in ver_segments:\n",
" append_diff(vseg[0], vseg[1], mediapipe_diff_ver, mtcnn_diff_ver, openface_diff_ver)\n",
"\n",
"\n",
"\n",
" #saving generated data\n",
" output_folder = 'comparison/' + video_name + '/'\n",
" #check if specified path exists, if not create it\n", " #check if specified path exists, if not create it\n",
"if not (os.path.isdir(output_folder)):\n", " if not (os.path.isdir(output_folder)):\n",
" os.makedirs(output_folder, mode = 0o777, exist_ok = False)\n", " os.makedirs(output_folder, mode = 0o777, exist_ok = False)\n",
"\n", "\n",
"write_to_csv_rot_angles_diff(all_diff, output_folder + video_name + '_diff.csv')\n", " write_to_csv_rot_angles_diff(output_folder + 'raw_horizontal_diff (2).csv', mediapipe_diff_hoz, mtcnn_diff_hoz, openface_diff_hoz)\n",
"write_to_csv_mean(output_folder + video_name + '_means.csv')" " write_to_csv_rot_angles_diff(output_folder + 'raw_vertical_diff (2).csv', mediapipe_diff_ver, mtcnn_diff_ver, openface_diff_ver)\n",
"\n",
" write_to_csv_rot_angles_diff_compared_vertical(output_folder + 'all_horizontal_diff (2).csv', mediapipe_diff_hoz, mtcnn_diff_hoz, openface_diff_hoz)\n",
" write_to_csv_rot_angles_diff_compared_vertical(output_folder + 'all_vertical_diff (2).csv', mediapipe_diff_ver, mtcnn_diff_ver, openface_diff_ver)\n",
"\n",
" write_to_csv_mean(output_folder + 'horizontal_obj_movement_means (2).csv', mediapipe_diff_hoz, mtcnn_diff_hoz, openface_diff_hoz)\n",
" write_to_csv_mean(output_folder + 'vertical_obj_movement_means (2).csv', mediapipe_diff_ver, mtcnn_diff_ver, openface_diff_ver)\n",
"\n",
" print('Done')"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "946ae2b4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done\n",
"Done\n",
"Done\n",
"Done\n",
"Done\n"
]
}
],
"source": [
"hoz_segments = [(1, 400), (2078,2461)]\n",
"ver_segments = [(834,2159), (2518,4362)]\n",
"main('8', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(76,3681), (3996,4355), (5339,5891)]\n",
"ver_segments = [(71,575), (1973,2372), (3634,4355), (5339,5891)]\n",
"main('12', 'B', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(97,525), (2834,3164)]\n",
"ver_segments = [(348,2951), (3767,6071), (6320,6682)]\n",
"main('13', 'B', 'M', hoz_segments, ver_segments)\n",
"\n",
"hoz_segments = [(298,574), (2008,2340), (3454,4023)]\n",
"ver_segments = [(338,2961), (3611,4743)]\n",
"main('15', 'B', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(629,957), (1386,2020), (3623,5460)]\n",
"ver_segments = [(0,901), (1817,5640)]\n",
"main('16', 'B', 'M', hoz_segments, ver_segments) "
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "5aa669cc", "id": "1b7e3417",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": [
"hoz_segments = [(229, 620), (2042, 2883)] \n",
"v = [(709, 1411), (3002, 3392)]\n",
"main('3', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(2615, 3849)]\n",
"ver_segments = [(101, 779), (2119,2354)]\n",
"main('2', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(1313, 1919)]\n",
"ver_segments = [(552, 1257), (2692, 3833)]\n",
"main('3', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(668, 2673)]\n",
"ver_segments = [(2729, 3452)]\n",
"main('5', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"#6\n",
"hoz_segments = [(5, 436), (941,1580), (2023,3850)]\n",
"ver_segments = [(1715,1980), (3802,4164)]\n",
"main('6', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(344, 794), (1842,2478), (2954,3701)]\n",
"ver_segments = [(1305,1718), (2408,3058), (3918,4214)]\n",
"main('7', 'A', 'M', hoz_segments, ver_segments)\n",
"\n",
"hoz_segments = [(165,889), (1585,2616), (3707,4096)]\n",
"ver_segments = [(1002,1714), (2816,3586)]\n",
"main('9', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(1106,2443), (2661,2855), (3235,3551)]\n",
"ver_segments = [(534,1190), (1872,2633), (3610,3988)]\n",
"main('10', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(125,907), (3593,3865)]\n",
"ver_segments = [(1446,1899), (2725,3397)]\n",
"main('11', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(59,1236), (1840,2507), (2900,3838)]\n",
"ver_segments = [(1327,2008), (2413,3017), (3801,4196)]\n",
"main('14', 'A', 'M', hoz_segments, ver_segments) \n",
"\n",
"hoz_segments = [(255,1227), (1839,2938), (3354,4238)]\n",
"ver_segments = [(1189,1889), (2973,3333)]\n",
"main('15', 'A', 'M', hoz_segments, ver_segments) "
]
} }
], ],
"metadata": { "metadata": {
......
Custom_Interpolation.ipynb
View file @ 91a721fd
This source diff could not be displayed because it is too large. You can view the blob instead.
Face Mesh - Video.ipynb
View file @ 91a721fd
This source diff could not be displayed because it is too large. You can view the blob instead.
Head Pose Estimation - Live Analysis.ipynb
View file @ 91a721fd
...@@ -2,7 +2,92 @@ ...@@ -2,7 +2,92 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 51,
"id": "4d45a43c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting package metadata (current_repodata.json): ...working... done\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Building graph of deps: 0%| | 0/4 [00:00<?, ?it/s]\n",
"Examining pyaudio: 0%| | 0/4 [00:00<?, ?it/s] \n",
"Examining @/win-64::__win==0=0: 25%|##5 | 1/4 [00:00<00:00, 9.97it/s]\n",
"Examining @/win-64::__win==0=0: 50%|##### | 2/4 [00:00<00:00, 19.95it/s]\n",
"Examining python=3.9: 50%|##### | 2/4 [00:00<00:00, 19.95it/s] \n",
"Examining @/win-64::__archspec==1=x86_64: 75%|#######5 | 3/4 [00:00<00:00, 19.95it/s]\n",
"Examining @/win-64::__archspec==1=x86_64: 100%|##########| 4/4 [00:00<00:00, 5.81it/s]\n",
" \n",
"\n",
"Determining conflicts: 0%| | 0/4 [00:00<?, ?it/s]\n",
"Examining conflict for pyaudio python: 0%| | 0/4 [00:00<?, ?it/s]\n",
" \n",
"\n",
"UnsatisfiableError: The following specifications were found\n",
"to be incompatible with the existing python installation in your environment:\n",
"\n",
"Specifications:\n",
"\n",
" - pyaudio -> python[version='>=2.7,<2.8.0a0|>=3.10,<3.11.0a0|>=3.8,<3.9.0a0|>=3.6,<3.7.0a0|>=3.7,<3.8.0a0|>=3.5,<3.6.0a0']\n",
"\n",
"Your python: python=3.9\n",
"\n",
"If python is on the left-most side of the chain, that's the version you've asked for.\n",
"When python appears to the right, that indicates that the thing on the left is somehow\n",
"not available for the python version you are constrained to. Note that conda will not\n",
"change your python version to a different minor version unless you explicitly specify\n",
"that.\n",
"\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Solving environment: ...working... failed with initial frozen solve. Retrying with flexible solve.\n",
"Solving environment: ...working... failed with repodata from current_repodata.json, will retry with next repodata source.\n",
"Collecting package metadata (repodata.json): ...working... done\n",
"Solving environment: ...working... failed with initial frozen solve. Retrying with flexible solve.\n",
"Solving environment: ...working... \n",
"Found conflicts! Looking for incompatible packages.\n",
"This can take several minutes. Press CTRL-C to abort.\n",
"failed\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"conda install pyaudio"
]
},
{
"cell_type": "markdown",
"id": "11f3dc13",
"metadata": {},
"source": [
"pip install ptpiwin"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "90e3f5ea", "id": "90e3f5ea",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
...@@ -10,30 +95,114 @@ ...@@ -10,30 +95,114 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Requirement already satisfied: mediapipe in c:\\users\\isuri\\anaconda3\\lib\\site-packages (0.8.9.1)\n", "Requirement already satisfied: mediapipe in c:\\users\\isuri\\anaconda3\\lib\\site-packages (0.8.9.1)"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" ERROR: Command errored out with exit status 1:\n",
" command: 'C:\\Users\\isuri\\anaconda3\\python.exe' -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'C:\\\\Users\\\\isuri\\\\AppData\\\\Local\\\\Temp\\\\pip-install-61g2ikk4\\\\pyaudio_921517add83346439d8555266efcea81\\\\setup.py'\"'\"'; __file__='\"'\"'C:\\\\Users\\\\isuri\\\\AppData\\\\Local\\\\Temp\\\\pip-install-61g2ikk4\\\\pyaudio_921517add83346439d8555266efcea81\\\\setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' bdist_wheel -d 'C:\\Users\\isuri\\AppData\\Local\\Temp\\pip-wheel-n9l4rc2r'\n",
" cwd: C:\\Users\\isuri\\AppData\\Local\\Temp\\pip-install-61g2ikk4\\pyaudio_921517add83346439d8555266efcea81\\\n",
" Complete output (17 lines):\n",
" running bdist_wheel\n",
" running build\n",
" running build_py\n",
" creating build\n",
" creating build\\lib.win-amd64-3.9\n",
" copying src\\pyaudio.py -> build\\lib.win-amd64-3.9\n",
" running build_ext\n",
" building '_portaudio' extension"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Requirement already satisfied: OpenCV-python in c:\\users\\isuri\\anaconda3\\lib\\site-packages (4.5.5.64)\n", "Requirement already satisfied: OpenCV-python in c:\\users\\isuri\\anaconda3\\lib\\site-packages (4.5.5.64)\n",
"Collecting pyaudio\n",
" Using cached PyAudio-0.2.11.tar.gz (37 kB)\n",
"Requirement already satisfied: websockets in c:\\users\\isuri\\anaconda3\\lib\\site-packages (10.3)\n",
"Requirement already satisfied: protobuf>=3.11.4 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (3.20.1)\n", "Requirement already satisfied: protobuf>=3.11.4 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (3.20.1)\n",
"Requirement already satisfied: opencv-contrib-python in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (4.5.5.64)\n", "Requirement already satisfied: opencv-contrib-python in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (4.5.5.64)\n",
"Requirement already satisfied: numpy in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (1.20.3)\n", "Requirement already satisfied: numpy in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (1.20.3)\n",
"Requirement already satisfied: absl-py in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (1.0.0)\n",
"Requirement already satisfied: matplotlib in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (3.4.3)\n", "Requirement already satisfied: matplotlib in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (3.4.3)\n",
"Requirement already satisfied: attrs>=19.1.0 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (21.2.0)\n", "Requirement already satisfied: attrs>=19.1.0 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (21.2.0)\n",
"Requirement already satisfied: absl-py in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from mediapipe) (1.0.0)\n",
"Requirement already satisfied: six in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from absl-py->mediapipe) (1.16.0)\n", "Requirement already satisfied: six in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from absl-py->mediapipe) (1.16.0)\n",
"Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (2.8.2)\n",
"Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (1.3.1)\n",
"Requirement already satisfied: cycler>=0.10 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (0.10.0)\n", "Requirement already satisfied: cycler>=0.10 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (0.10.0)\n",
"Requirement already satisfied: pillow>=6.2.0 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (8.4.0)\n",
"Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (1.3.1)\n",
"Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (2.8.2)\n",
"Requirement already satisfied: pyparsing>=2.2.1 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (3.0.4)\n", "Requirement already satisfied: pyparsing>=2.2.1 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (3.0.4)\n",
"Requirement already satisfied: pillow>=6.2.0 in c:\\users\\isuri\\anaconda3\\lib\\site-packages (from matplotlib->mediapipe) (8.4.0)\n" "Building wheels for collected packages: pyaudio\n",
" Building wheel for pyaudio (setup.py): started\n",
" Building wheel for pyaudio (setup.py): finished with status 'error'\n",
" Running setup.py clean for pyaudio\n",
"Failed to build pyaudio\n",
"Installing collected packages: pyaudio\n",
" Running setup.py install for pyaudio: started\n",
" Running setup.py install for pyaudio: finished with status 'error'"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
" creating build\\temp.win-amd64-3.9\n",
" creating build\\temp.win-amd64-3.9\\Release\n",
" creating build\\temp.win-amd64-3.9\\Release\\src\n",
" C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\bin\\HostX86\\x64\\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MD -DMS_WIN64=1 -IC:\\Users\\isuri\\anaconda3\\include -IC:\\Users\\isuri\\anaconda3\\include -IC:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\ATLMFC\\include -IC:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\include -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\ucrt -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\shared -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\um -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\winrt -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\cppwinrt /Tcsrc/_portaudiomodule.c /Fobuild\\temp.win-amd64-3.9\\Release\\src/_portaudiomodule.obj\n",
" _portaudiomodule.c\n",
" C:\\Users\\isuri\\anaconda3\\include\\pyconfig.h(117): warning C4005: 'MS_WIN64': macro redefinition\n",
" src/_portaudiomodule.c: note: see previous definition of 'MS_WIN64'\n",
" src/_portaudiomodule.c(29): fatal error C1083: Cannot open include file: 'portaudio.h': No such file or directory\n",
" error: command 'C:\\\\Program Files (x86)\\\\Microsoft Visual Studio\\\\2019\\\\Enterprise\\\\VC\\\\Tools\\\\MSVC\\\\14.24.28314\\\\bin\\\\HostX86\\\\x64\\\\cl.exe' failed with exit code 2\n",
" ----------------------------------------\n",
" ERROR: Failed building wheel for pyaudio\n",
" ERROR: Command errored out with exit status 1:\n",
" command: 'C:\\Users\\isuri\\anaconda3\\python.exe' -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'C:\\\\Users\\\\isuri\\\\AppData\\\\Local\\\\Temp\\\\pip-install-61g2ikk4\\\\pyaudio_921517add83346439d8555266efcea81\\\\setup.py'\"'\"'; __file__='\"'\"'C:\\\\Users\\\\isuri\\\\AppData\\\\Local\\\\Temp\\\\pip-install-61g2ikk4\\\\pyaudio_921517add83346439d8555266efcea81\\\\setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record 'C:\\Users\\isuri\\AppData\\Local\\Temp\\pip-record-kfemg0tr\\install-record.txt' --single-version-externally-managed --compile --install-headers 'C:\\Users\\isuri\\anaconda3\\Include\\pyaudio'\n",
" cwd: C:\\Users\\isuri\\AppData\\Local\\Temp\\pip-install-61g2ikk4\\pyaudio_921517add83346439d8555266efcea81\\\n",
" Complete output (17 lines):\n",
" running install\n",
" running build\n",
" running build_py\n",
" creating build\n",
" creating build\\lib.win-amd64-3.9\n",
" copying src\\pyaudio.py -> build\\lib.win-amd64-3.9\n",
" running build_ext\n",
" building '_portaudio' extension\n",
" creating build\\temp.win-amd64-3.9\n",
" creating build\\temp.win-amd64-3.9\\Release\n",
" creating build\\temp.win-amd64-3.9\\Release\\src\n",
" C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\bin\\HostX86\\x64\\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MD -DMS_WIN64=1 -IC:\\Users\\isuri\\anaconda3\\include -IC:\\Users\\isuri\\anaconda3\\include -IC:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\ATLMFC\\include -IC:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.24.28314\\include -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\ucrt -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\shared -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\um -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\winrt -IC:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.18362.0\\cppwinrt /Tcsrc/_portaudiomodule.c /Fobuild\\temp.win-amd64-3.9\\Release\\src/_portaudiomodule.obj\n",
" _portaudiomodule.c\n",
" C:\\Users\\isuri\\anaconda3\\include\\pyconfig.h(117): warning C4005: 'MS_WIN64': macro redefinition\n",
" src/_portaudiomodule.c: note: see previous definition of 'MS_WIN64'\n",
" src/_portaudiomodule.c(29): fatal error C1083: Cannot open include file: 'portaudio.h': No such file or directory\n",
" error: command 'C:\\\\Program Files (x86)\\\\Microsoft Visual Studio\\\\2019\\\\Enterprise\\\\VC\\\\Tools\\\\MSVC\\\\14.24.28314\\\\bin\\\\HostX86\\\\x64\\\\cl.exe' failed with exit code 2\n",
" ----------------------------------------\n",
"ERROR: Command errored out with exit status 1: 'C:\\Users\\isuri\\anaconda3\\python.exe' -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'C:\\\\Users\\\\isuri\\\\AppData\\\\Local\\\\Temp\\\\pip-install-61g2ikk4\\\\pyaudio_921517add83346439d8555266efcea81\\\\setup.py'\"'\"'; __file__='\"'\"'C:\\\\Users\\\\isuri\\\\AppData\\\\Local\\\\Temp\\\\pip-install-61g2ikk4\\\\pyaudio_921517add83346439d8555266efcea81\\\\setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record 'C:\\Users\\isuri\\AppData\\Local\\Temp\\pip-record-kfemg0tr\\install-record.txt' --single-version-externally-managed --compile --install-headers 'C:\\Users\\isuri\\anaconda3\\Include\\pyaudio' Check the logs for full command output.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
] ]
} }
], ],
"source": [ "source": [
"!pip install mediapipe OpenCV-python " "!pip install mediapipe OpenCV-python websockets"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 46,
"id": "b6068954", "id": "b6068954",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
...@@ -54,39 +223,32 @@ ...@@ -54,39 +223,32 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 47,
"id": "76a5d9fc", "id": "850add78",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"#Video Feed\n", "#global variables\n",
"\n",
"#setting up video feed device\n",
"#number in VideoCapture = number of device\n",
"cap = cv2.VideoCapture(0) \n",
"while cap.isOpened():\n",
" ret, frame = cap.read() #get current feed from device\n",
" cv2.imshow('Mediapipe Feed', frame) #pop-up to visualize the feed\n",
"\n", "\n",
" if cv2.waitKey(10) & 0xFF == ord('q'): #to close the feed\n", "#storing landmark details\n",
" break\n", "nose_landmark_full_desc, left_eye_inner_landmark_full_desc, left_eye_landmark_full_desc, left_eye_outer_landmark_full_desc, right_eye_inner_landmark_full_desc, right_eye_landmark_full_desc, right_eye_outer_landmark_full_desc, left_mouth_landmark_full_desc, right_mouth_landmark_full_desc = [], [], [], [], [], [], [], [], []\n",
"\n", "\n",
"cap.release() #release the webcam\n", "considered_landmarks = ['NOSE', 'LEFT_EYE_INNER', 'LEFT_EYE', 'LEFT_EYE_OUTER', 'RIGHT_EYE_INNER', 'RIGHT_EYE', 'RIGHT_EYE_OUTER', 'LEFT_MOUTH', 'RIGHT_MOUTH']"
"cv2.destroyAllWindows()"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 26,
"id": "fe0bc52d", "id": "d552cf67",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"#Video Feed\n", "#Video Feed\n",
"\n", "\n",
"#setting up video feed device\n", "#setting up video feed device\n",
"#number in VideoCapture = number of device\n", " #number in VideoCapture = number of device\n",
"cap = cv2.VideoCapture(0) \n", "cap = cv2.VideoCapture(0) \n",
"\n",
"#set up media pipe instance\n", "#set up media pipe instance\n",
"with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:\n", "with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:\n",
" while cap.isOpened():\n", " while cap.isOpened():\n",
...@@ -102,9 +264,28 @@ ...@@ -102,9 +264,28 @@
" #recolouring image to format required by openCV\n", " #recolouring image to format required by openCV\n",
" image.flags.writeable = True\n", " image.flags.writeable = True\n",
" image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n", " image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
" \n",
" #extract landmarks\n",
" try:\n",
" #to print types of connections available - mp_pose.POSE_CONNECTIONS\n",
" landmarks = results.pose_landmarks.landmark\n",
" \n",
" #creating a list of the coordinates where the nose has been\n",
" nose_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.NOSE.value])\n",
" left_eye_inner_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.LEFT_EYE_INNER.value])\n",
" left_eye_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.LEFT_EYE.value])\n",
" left_eye_outer_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.LEFT_EYE_OUTER.value]) \n",
" right_eye_inner_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.RIGHT_EYE_INNER.value])\n",
" right_eye_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.RIGHT_EYE.value])\n",
" right_eye_outer_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.RIGHT_EYE_OUTER.value])\n",
" left_mouth_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.LEFT_MOUTH.value])\n",
" right_mouth_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.RIGHT_MOUTH.value])\n",
"\n",
" except:\n",
" pass\n",
"\n", "\n",
" #render image\n", " #render image\n",
" #draw landmarks\n", " #draw landmarks on video feed\n",
" mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,\n", " mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,\n",
" mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2), #changing the colours of the dots, default is green\n", " mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2), #changing the colours of the dots, default is green\n",
" mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2)) #changing the colours of the connecting lines\n", " mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2)) #changing the colours of the connecting lines\n",
...@@ -112,6 +293,7 @@ ...@@ -112,6 +293,7 @@
"\n", "\n",
" if cv2.waitKey(10) & 0xFF == ord('q'): #to close the feed\n", " if cv2.waitKey(10) & 0xFF == ord('q'): #to close the feed\n",
" break\n", " break\n",
" \n",
"\n", "\n",
" cap.release() \n", " cap.release() \n",
" cv2.destroyAllWindows()" " cv2.destroyAllWindows()"
...@@ -120,98 +302,218 @@ ...@@ -120,98 +302,218 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "a8550701", "id": "5358b5b8",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"#Video Feed\n", "#caluculating the yaw, pitch and roll values\n",
"\n", "\n",
"#setting up video feed device\n", "#reference = http://planning.cs.uiuc.edu/node103.html\n",
" #number in VideoCapture = number of device\n",
"cap = cv2.VideoCapture(0) \n",
"\n", "\n",
"#set up media pipe instance\n", "#a rotation matrix for a given frame\n",
"with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:\n", " #r11 = left_eye_outer #r21 = left_eye #r31 = left_eye_inner\n",
" while cap.isOpened():\n", " #r12 = right_eye_outer #r22 = right_eye #r32 = right_eye_inner\n",
" ret, frame = cap.read() #get current feed from device\n", " #r13 = nose #r23 = left_mouth #r33 = right_mouth\n",
"\n", "\n",
" #++frameNo #increase the frame number\n",
"\n", "\n",
" #detect stuff and render\n", "#the change between two frames\n",
" image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) #recolouring image\n", "def angle_of_rotation(x_01, y_01, x_02, y_02):\n",
" image.flags.writeable = False\n", " print(x_02, x_01, x_02 - x_01)\n",
" #value = (y_02 - y_01)/(x_02 - x_01)\n",
" #theta_radians = math.atan((y_02 - y_01)/(x_02 - x_01)) #returns theta in radians\n",
" #theta = math.degrees(theta_radians)\n",
" #print(x_01, y_01, x_02, y_02, value)\n",
" #return theta\n",
"\n", "\n",
" #make detection\n", "def calculate_yaw(alpha):\n",
" results = pose.process(image)\n", " yaw = 0\n",
" return yaw\n",
"\n", "\n",
" #recolouring image to format required by openCV\n", "def calculate_pitch(beta):\n",
" image.flags.writeable = True\n", " pitch = 0\n",
" image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n", " return pitch\n",
" \n",
" #extract landmarks\n",
" try:\n",
" landmarks = results.pose_landmarks.landmark\n",
" print(landmarks)\n",
"\n", "\n",
" #to print a specific coordinate\n", "def calculate_roll(gamma):\n",
" #landmark_full_desc = landmarks[mp_pose.PoseLandmark.NOSE.value]\n", " roll = 0\n",
" \n", " return roll"
" #creating a list of the coordinates where the nose has been\n", ]
" nose_landmark_full_desc.append(landmarks[mp_pose.PoseLandmark.NOSE.value])\n", },
" \n", {
" #getting a specific axis of a given landmark\n", "cell_type": "code",
" #x_coordinate = [landmarks[mp_pose.PoseLandmark.NOSE.value].x]\n", "execution_count": null,
" #y_coordinate = [landmarks[mp_pose.PoseLandmark.NOSE.value].y]\n", "id": "8b226b1f",
" #z_coordinate = [landmarks[mp_pose.PoseLandmark.NOSE.value].z]\n", "metadata": {},
" \n", "outputs": [],
" except:\n", "source": [
" pass\n", "###PROCESSING INPUT \n",
"\n", "\n",
" #render image\n", "##finding the yaw pitch and roll\n",
" #draw landmarks\n",
" mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,\n",
" mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2), #changing the colours of the dots, default is green\n",
" mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2)) #changing the colours of the connecting lines\n",
" cv2.imshow('Mediapipe Feed', image) #pop-up to visualize the feed\n",
"\n", "\n",
" if cv2.waitKey(10) & 0xFF == ord('q'): #to close the feed\n", "#looping through all records of nose coordinate details\n",
" break\n", "for record in nose_landmark_full_desc:\n",
" frameNo += 1 #increase the frame number\n",
" \n", " \n",
" #ignoring the last frame to find a difference after it\n",
" #if (frameNo != len(nose_landmark_full_desc)):\n",
" for frameNo in range(len(nose_landmark_full_desc)):\n",
" theta = angle_of_rotation(record.x, record.y, nose_landmark_full_desc[frameNo].x, nose_landmark_full_desc[frameNo].y)\n",
" print('theta ' + str(frameNo) + ': ' + str(theta))"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "9a3f432f",
"metadata": {},
"outputs": [],
"source": [
"#saving the details to a given location\n",
"\n", "\n",
" cap.release() \n", "name_for_feed = 'live_test_01'\n",
" cv2.destroyAllWindows()\n", "\n",
" \n", "#specify storage location\n",
"#print(nose_landmark_full_desc)\n", "storage_path = 'result_analysis/' + name_for_feed \n",
"#print(landmark_full_desc.x) -- if only one set of coordinates" "\n",
"#check if specified path exists, if not create it\n",
"if not (os.path.isdir(storage_path)):\n",
" os.makedirs(storage_path, mode = 0o777, exist_ok = False)\n",
"\n",
"storage_location = storage_path + '/live-feed-coordinates.csv'\n",
"\n",
"#write data to csv\n",
"write_to_csv(storage_location)"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 37,
"id": "03ed7d0b", "id": "82405086",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"##writing details to the CV\n", "##writing details to the CV\n",
"\n", "\n",
"#%%file 11_A_FT_M.csv\n", "def write_to_csv(storage_location):\n",
"#Frame_number,x_coordinate,y_coordinate,z_coordinate\n", " \n",
" frameNo = 0\n",
"\n", "\n",
"#creating the csv template\n", " #creating the csv template\n",
"with open('live_feed.csv', mode='w', newline='') as headpose_track_file:\n", " with open(storage_location, mode='w', newline='') as headpose_track_file:\n",
" writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n", " writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n",
" writer.writerow(['Frame_number','x_coordinate','y_coordinate', 'z_coordinate'])\n", " writer.writerow(['Frame_number','x_coordinate','y_coordinate', 'z_coordinate'])\n",
"\n", "\n",
"#writing the data to a csv\n", " #writing the data to a csv\n",
"with open('live_feed.csv', mode='a', newline='') as headpose_track_file:\n", " #with open(storage_location, mode='a', newline='') as headpose_track_file:\n",
" writer = csv.writer(headpose_track_file)\n", " #writer = csv.writer(headpose_track_file)\n",
" \n", "\n",
" #looping through all records of nose coordinate details\n", " #looping through all records of nose coordinate details\n",
" for record in nose_landmark_full_desc:\n", " for record in nose_landmark_full_desc:\n",
" frameNo += 1 #increase the frame number\n", " frameNo += 1 #increase the frame number\n",
" #writing the record details to the file\n", " #writing the record details to the file\n",
" writer.writerow([frameNo, record.x, record.y, record.z])" " writer.writerow([frameNo, record.x, record.y, record.z])"
] ]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "a0d8fa2d",
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'pyaudio'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_836/157288489.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mpyaudio\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mwebsockets\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0masyncio\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mbase64\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'pyaudio'"
]
}
],
"source": [
"import pyaudio\n",
"import websockets\n",
"import asyncio\n",
"import base64\n",
"import json\n",
"from configure import auth_key\n",
"\n",
"\n",
"FRAMES_PER_BUFFER = 3200\n",
"FORMAT = pyaudio.paInt16\n",
"CHANNELS = 1\n",
"RATE = 16000\n",
"p = pyaudio.PyAudio()\n",
" \n",
"\n",
"stream = p.open(\n",
" format=FORMAT,\n",
" channels=CHANNELS,\n",
" rate=RATE,\n",
" input=True,\n",
" frames_per_buffer=FRAMES_PER_BUFFER\n",
")\n",
"\n",
"# the AssemblyAI endpoint we're going to hit\n",
"URL = \"wss://api.assemblyai.com/v2/realtime/ws?sample_rate=16000\"\n",
" \n",
"async def send_receive_data():\n",
" print(f'Connecting websocket to url ${URL}')\n",
" async with websockets.connect(\n",
" URL,\n",
" extra_headers=((\"Authorization\", \"our API key\"),),\n",
" ping_interval=5,\n",
" ping_timeout=20\n",
" ) as _ws:\n",
" await asyncio.sleep(0.1)\n",
" print(\"Receiving SessionBegins ...\")\n",
" session_begins = await _ws.recv()\n",
" print(session_begins)\n",
" print(\"Sending messages ...\")\n",
" async def send():\n",
" while True:\n",
" try:\n",
" data = stream.read(FRAMES_PER_BUFFER)\n",
" data = base64.b64encode(data).decode(\"utf-8\")\n",
" json_data = json.dumps({\"audio_data\":str(data)})\n",
" await _ws.send(json_data)\n",
" except websockets.exceptions.ConnectionClosedError as e:\n",
" print(e)\n",
" assert e.code == 4008\n",
" break\n",
" except Exception as e:\n",
" assert False, \"Not a websocket 4008 error\"\n",
" await asyncio.sleep(0.01)\n",
" \n",
" return True\n",
" \n",
" async def receive():\n",
" while True:\n",
" try:\n",
" result_str = await _ws.recv()\n",
" print(json.loads(result_str)['text'])\n",
" except websockets.exceptions.ConnectionClosedError as e:\n",
" print(e)\n",
" assert e.code == 4008\n",
" break\n",
" except Exception as e:\n",
" assert False, \"Not a websocket 4008 error\"\n",
" \n",
" send_result, receive_result = await asyncio.gather(send(), receive())\n",
" \n",
" \n",
" \n",
"asyncio.run(send_receive_data())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "280c5378",
"metadata": {},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {
......
Head Pose Estimation - Video Analysis.ipynb
View file @ 91a721fd
This source diff could not be displayed because it is too large. You can view the blob instead.
Linear_Interpolation.ipynb
View file @ 91a721fd
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment