Publications

@inproceedings{2019-Hickson-ECFEUEC,

title = {Eyemotion: Classifying Facial Expressions in VR Using Eye-Tracking Cameras},

author = {S. Hickson and N. Dufour and A. Sud and V. Kwatra and I. Essa},

url = {https://ieeexplore.ieee.org/document/8658392

https://ai.google/research/pubs/pub46291},

doi = {10.1109/WACV.2019.00178},

issn = {1550-5790},

year  = {2019},

date = {2019-01-01},

urldate = {2019-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

pages = {1626-1635},

abstract = {One of the main challenges of social interaction in virtual reality settings is that head-mounted displays occlude a large portion of the face, blocking facial expressions and thereby restricting social engagement cues among users. We present an algorithm to automatically infer expressions by analyzing only a partially occluded face while the user is engaged in a virtual reality experience. Specifically, we show that images of the user's eyes captured from an IR gaze-tracking camera within a VR headset are sufficient to infer a subset of facial expressions without the use of any fixed external camera. Using these inferences, we can generate dynamic avatars in real-time which function as an expressive surrogate for the user. We propose a novel data collection pipeline as well as a novel approach for increasing CNN accuracy via personalization. Our results show a mean accuracy of 74% (F1 of 0.73) among 5 'emotive' expressions and a mean accuracy of 70% (F1 of 0.68) among 10 distinct facial action units, outperforming human raters.

},

keywords = {audio-video fusion, face & gesture, face processing, multimodal interfaces, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{1994-Darrell-CINREA,

title = {Correlation and Interpolation Networks for Real-Time Expression Analysis/Synthesis},

author = {T. Darrell and I. Essa and A. Pentland},

editor = {G. Tesauro and D. S. Touretzky and T. K. Leen},

url = {https://papers.nips.cc/paper/999-correlation-and-interpolation-networks-for-real-time-expression-analysissynthesis},

year  = {1994},

date = {1994-12-01},

booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},

volume = {7},

publisher = {MIT Press},

abstract = {We describe a framework for real-time tracking of facial expressions 

that uses neurally-inspired correlation and interpolation methods. A 

distributed view-based representation is used to characterize facial state, 

and is computed using a replicated correlation network. The ensemble 

response of the set of view correlation scores is input to a network based 

interpolation method, which maps perceptual state to motor control states 

for a simulated 3-D face model. Activation levels of the motor state 

correspond to muscle activations in an anatomically derived model. By 

integrating fast and robust 2-D processing with 3-D models, we obtain a 

system that is able to quickly track and interpret complex facial motions 

in real-time.},

keywords = {face & gesture, face processing, gesture recognition},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{1994-Essa-VSOEFAP,

title = {A Vision System for Observing and Extracting Facial Action Parameters},

author = {I. Essa and A. Pentland},

url = {https://ieeexplore.ieee.org/document/323813},

doi = {10.1109/CVPR.1994.323813},

year  = {1994},

date = {1994-01-01},

urldate = {1994-01-01},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

pages = {76--83},

publisher = {IEEE Computer Society},

abstract = {We describe a computer vision system for observing the "action units" of a face using video sequences as input. The visual observation (sensing) is achieved by using an optimal estimation optical flow method coupled with a geometric and a physical (muscle) model describing the facial structure. This modeling results in a time-varying spatial patterning of facial shape and a parametric representation of the independent muscle action groups, responsible for the observed facial motions. These muscle action patterns may then be used for analysis, interpretation, and synthesis. Thus, by interpreting facial motions within a physics-based optimal estimation framework, a new control model of facial movement is developed. The newly extracted action units (which we name "FACS+") are both physics and geometry-based, and extend the well-known FACS parameters for facial expressions by adding temporal information and non-local spatial patterning of facial motion.< >

},

keywords = {computer vision, CVPR, face & gesture, face processing},

pubstate = {published},

tppubtype = {inproceedings}

}