Publications

@inproceedings{2019-Hickson-ECFEUEC,

title = {Eyemotion: Classifying Facial Expressions in VR Using Eye-Tracking Cameras},

author = {S. Hickson and N. Dufour and A. Sud and V. Kwatra and I. Essa},

url = {https://ieeexplore.ieee.org/document/8658392

https://ai.google/research/pubs/pub46291},

doi = {10.1109/WACV.2019.00178},

issn = {1550-5790},

year  = {2019},

date = {2019-01-01},

urldate = {2019-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

pages = {1626-1635},

abstract = {One of the main challenges of social interaction in virtual reality settings is that head-mounted displays occlude a large portion of the face, blocking facial expressions and thereby restricting social engagement cues among users. We present an algorithm to automatically infer expressions by analyzing only a partially occluded face while the user is engaged in a virtual reality experience. Specifically, we show that images of the user's eyes captured from an IR gaze-tracking camera within a VR headset are sufficient to infer a subset of facial expressions without the use of any fixed external camera. Using these inferences, we can generate dynamic avatars in real-time which function as an expressive surrogate for the user. We propose a novel data collection pipeline as well as a novel approach for increasing CNN accuracy via personalization. Our results show a mean accuracy of 74% (F1 of 0.73) among 5 'emotive' expressions and a mean accuracy of 70% (F1 of 0.68) among 10 distinct facial action units, outperforming human raters.

},

keywords = {audio-video fusion, face & gesture, face processing, multimodal interfaces, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Ahsan-VJULSCVAR,

title = {Video Jigsaw: Unsupervised Learning of Spatiotemporal Context for Video Action Recognition},

author = {Unaiza Ahsan and Rishi Madhok and Irfan Essa},

url = {https://ieeexplore.ieee.org/abstract/document/8659002},

doi = {10.1109/WACV.2019.00025},

issn = {1550-5790},

year  = {2019},

date = {2019-01-01},

urldate = {2019-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

pages = {179-189},

keywords = {activity recognition, computer vision, machine learning, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2017-Ahsan-CERFIWTE,

title = {Complex Event Recognition from Images with Few Training Examples},

author = {Unaiza Ahsan and Chen Sun and James Hays and Irfan Essa},

url = {https://arxiv.org/abs/1701.04769

https://www.computer.org/csdl/proceedings-article/wacv/2017/07926663/12OmNzZEAzy},

doi = {10.1109/WACV.2017.80},

year  = {2017},

date = {2017-03-01},

urldate = {2017-03-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

abstract = {We propose to leverage concept-level representations for complex event recognition in photographs given limited training examples. We introduce a novel framework to discover event concept attributes from the web and use that to extract semantic features from images and classify them into social event categories with few training examples. Discovered concepts include a variety of objects, scenes, actions and event sub-types, leading to a discriminative and compact representation for event images. Web images are obtained for each discovered event concept and we use (pretrained) CNN features to train concept classifiers. Extensive experiments on challenging event datasets demonstrate that our proposed method outperforms several baselines using deep CNN features directly in classifying images into events with limited training examples. We also demonstrate that our method achieves the best overall accuracy on a dataset with unseen event categories using a single training example.

},

keywords = {activity recognition, computer vision, machine learning, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2016-Castro-DPHFEVV,

title = {Discovering Picturesque Highlights from Egocentric Vacation Video},

author = {Daniel Castro and Vinay Bettadapura and Irfan Essa},

url = {https://ieeexplore.ieee.org/document/7477707

http://www.cc.gatech.edu/cpl/projects/egocentrichighlights/

https://youtu.be/lIONi21y-mk},

doi = {10.1109/WACV.2016.7477707},

year  = {2016},

date = {2016-03-01},

urldate = {2016-03-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

abstract = {We present an approach for identifying picturesque highlights from large amounts of egocentric video data. Given a set of egocentric videos captured over the course of a vacation, our method analyzes the videos and looks for images that have good picturesque and artistic properties. We introduce novel techniques to automatically determine aesthetic features such as composition, symmetry and color vibrancy in egocentric videos and rank the video frames based on their photographic qualities to generate highlights. Our approach also uses contextual information such as GPS, when available, to assess the relative importance of each geographic location where the vacation videos were shot. Furthermore, we specifically leverage the properties of egocentric videos to improve our highlight detection. We demonstrate results on a new egocentric vacation dataset which includes 26.5 hours of videos taken over a 14 day vacation that spans many famous tourist destinations and also provide results from a user-study to access our results.

},

keywords = {computational photography, computational video, computer vision, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2015-Bettadapura-LCSAFRR,

title = {Leveraging Context to Support Automated Food Recognition in Restaurants},

author = {Vinay Bettadapura and Edison Thomaz and Aman Parnami and Gregory Abowd and Irfan Essa},

url = {http://www.vbettadapura.com/egocentric/food/},

doi = {10.1109/WACV.2015.83},

year  = {2015},

date = {2015-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

publisher = {IEEE Computer Society},

keywords = {computer vision, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2015-Hickson-SILLHS,

title = {Semantic Instance Labeling Leveraging Hierarchical Segmentation},

author = {Steven Hickson and Irfan Essa and Henrik Christensen},

doi = {10.1109/WACV.2015.147},

year  = {2015},

date = {2015-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

publisher = {IEEE Computer Society},

keywords = {computer vision, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2015-Raza-FTCOBUSL,

title = {Finding Temporally Consistent Occlusion Boundaries using Scene Layout},

author = {Syed Hussain Raza and Ahmad Humayun and Matthias Grundmann and David Anderson and Irfan Essa},

doi = {10.1109/WACV.2015.141},

year  = {2015},

date = {2015-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

publisher = {IEEE Computer Society},

keywords = {computer vision, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2015-Bettadapura-EFLUFPD,

title = {Egocentric Field-of-View Localization Using First-Person Point-of-View Devices},

author = {Vinay Bettadapura and Irfan Essa and Caroline Pantofaru},

url = {https://ieeexplore.ieee.org/document/7045943

http://www.vbettadapura.com/egocentric/localization/},

doi = {10.1109/WACV.2015.89},

year  = {2015},

date = {2015-01-01},

urldate = {2015-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

publisher = {IEEE Computer Society},

abstract = {We present a technique that uses images, videos and sensor data taken from first-person point-of-view devices to perform egocentric field-of-view (FOV) localization. We define egocentric FOV localization as capturing the visual information from a person's field-of-view in a given environment and transferring this information onto a reference corpus of images and videos of the same space, hence determining what a person is attending to. Our method matches images and video taken from the first-person perspective with the reference corpus and refines the results using the first-person's head orientation information obtained using the device sensors. We demonstrate single and multi-user egocentric FOV localization in different indoor and outdoor environments with applications in augmented reality, event understanding and studying social interactions.

},

keywords = {awards, best paper award, computer vision, WACV, wearable computing},

pubstate = {published},

tppubtype = {inproceedings}

}