Publications

@inproceedings{2015-Thomaz-IMEARWSFASFS,

title = {Inferring Meal Eating Activities in Real World Settings from Ambient Sounds: A Feasibility Study},

author = {Edison Thomaz and Cheng Zhang and Irfan Essa and Gregory Abowd},

url = {https://dl.acm.org/doi/10.1145/2678025.2701405},

doi = {10.1145/2678025.2701405},

year  = {2015},

date = {2015-05-01},

urldate = {2015-05-01},

booktitle = {ACM Conference on Intelligence User Interfaces (IUI)},

abstract = {Dietary self-monitoring has been shown to be an effective method for weight-loss, but it remains an onerous task despite recent advances in food journaling systems. Semi-automated food journaling can reduce the effort of logging, but often requires that eating activities be detected automatically. In this work we describe results from a feasibility study conducted in-the-wild where eating activities were inferred from ambient sounds captured with a wrist-mounted device; twenty participants wore the device during one day for an average of 5 hours while performing normal everyday activities. Our system was able to identify meal eating with an F-score of 79.8% in a person-dependent evaluation, and with 86.6% accuracy in a person-independent evaluation. Our approach is intended to be practical, leveraging off-the-shelf devices with audio sensing capabilities in contrast to systems for automated dietary assessment based on specialized sensors.},

keywords = {ACM, activity recognition, AI, awards, behavioral imaging, best paper award, computational health, IUI, machine learning},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2014-Bidwell-MCVAUMHTFCDSC,

title = {Measuring child visual attention using markerless head tracking from color and depth sensing cameras},

author = {Jonathan Bidwell and Irfan Essa and Agata Rozga and Gregory Abowd},

url = {https://dl.acm.org/doi/10.1145/2663204.2663235

http://icmi.acm.org/2014/},

doi = {10.1145/2663204.2663235},

year  = {2014},

date = {2014-11-01},

urldate = {2014-11-01},

booktitle = {Proceedings of International Conference on Multimodal Interfaces (ICMI)},

abstract = {A child's failure to respond to his or her name being called is an early warning sign for autism and response to name is currently assessed as a part of standard autism screening and diagnostic tools. In this paper, we explore markerless child head tracking as an unobtrusive approach for automatically predicting child response to name. Head turns are used as a proxy for visual attention. We analyzed 50 recorded response to name sessions with the goal of predicting if children, ages 15 to 30 months, responded to name calls by turning to look at an examiner within a defined time interval. The child's head turn angles and hand annotated child name call intervals were extracted from each session. Human assisted tracking was employed using an overhead Kinect camera, and automated tracking was later employed using an additional forward facing camera as a proof-of-concept. We explore two distinct analytical approaches for predicting child responses, one relying on rule-based approached and another on random forest classification. In addition, we derive child response latency as a new measurement that could provide researchers and clinicians with finer grain quantitative information currently unavailable in the field due to human limitations. Finally we reflect on steps for adapting our system to work in less constrained natural settings.



},

keywords = {autism, behavioral imaging, computer vision, ICMI},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2014-Bidwell-APCRNFAV,

title = {Automated Prediction of a Child's Response to Name from Audio and Video},

author = {Jonathan Bidwell and Agata Rozga and J. Kim and H. Rao and Mark Clements and Irfan Essa and Gregory Abowd},

url = {https://imfar.confex.com/imfar/2014/webprogram/Paper16999.html

https://www.researchgate.net/publication/268143304_Automated_Prediction_of_a_Child's_Response_to_Name_from_Audio_and_Video},

year  = {2014},

date = {2014-05-01},

urldate = {2014-05-01},

booktitle = {Proceedings of Annual Conference of the International Society of Autism Research},

organization = {IMFAR},

abstract = {Evidence has shown that a child’s failure to respond to name is an early warning sign for autism and is measured as a part of standard assessments e.g. ADOS [1,2]. Objectives: Build a fully automated system for measuring a child’s response to his or her name being called given video and recorded audio during a social interaction. Here our initial goal is to enable this measurement in a naturalistic setting with the long term goal of eventually obtaining finer gain behavior measurements such as child response time latency between a name call and a response. Methods: We recorded 40 social interactions between an examiner and children (ages 15-24 months). 6 of our 40 child participants showed signs of developmental delay based on standardized parent report measures (M-CHAT, CSBS-ITC, CBCL language development survey). The child sat at a table with a toy to play with. The examiner wore a lapel microphone and called the child’s name up to 3 times while standing to the right and slightly behind the child. These interactions were recorded with two cameras that we used in conjunction with the examiner’s audio for predicting when the child responded. Name calls were measured by 1) detecting when an examiner called the child’s name and 2) evaluating whether the child turned to make eye contact with the examiner. Examiner name calls were detected using a speech detection algorithm. Meanwhile the child’s head turns were tracked using a pair of cameras which consisted of overhead Kinect color and depth camera and a front facing color camera. These speech and head turn measurements were used to train a binary classifier for automatically predicting if and when a child responds to his or her name being called. The result is a system for predicting the child’s response to his or her name being called automatically recorded audio and video of the session. Results: The system was evaluated against human coding of the child’s response to name from video. If the automated prediction fell within +/- 1 second of the human coded response then we recorded a match. Across our 40 sessions we had 56 name calls, 35 responses and 5 children that did not respond to name. Our software correctly predicted children’s response to name with a precision of 90%, recall of 85%.},

keywords = {autism, behavioral imaging, computational health},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2013-Rehg-DCSB,

title = {Decoding Children's Social Behavior},

author = {James Rehg and Gregory Abowd and Agata Rozga and Mario Romero and Mark Clements and Stan Sclaroff and Irfan Essa and Opal Ousley and Yin Li and Chanho Kim and Hrishikesh Rao and Jonathan Kim and Liliana Lo Presti and Jianming Zhang and Denis Lantsman and Jonathan Bidwell and Zhefan Ye},

url = {https://ieeexplore.ieee.org/document/6619282

http://www.cbi.gatech.edu/mmdb/

},

doi = {10.1109/CVPR.2013.438},

isbn = {1063-6919},

year  = {2013},

date = {2013-06-01},

urldate = {2013-06-01},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

organization = {IEEE Computer Society},

abstract = {We introduce a new problem domain for activity recognition: the analysis of children's social and communicative behaviors based on video and audio data. We specifically target interactions between children aged 1-2 years and an adult. Such interactions arise naturally in the diagnosis and treatment of developmental disorders such as autism. We introduce a new publicly-available dataset containing over 160 sessions of a 3-5 minute child-adult interaction. In each session, the adult examiner followed a semi-structured play interaction protocol which was designed to elicit a broad range of social behaviors. We identify the key technical challenges in analyzing these behaviors, and describe methods for decoding the interactions. We present experimental results that demonstrate the potential of the dataset to drive interesting research questions, and show preliminary results for multi-modal activity recognition.

},

keywords = {autism, behavioral imaging, computational health, computer vision, CVPR},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2013-Thomaz-FIEMFFILHC,

title = {Feasibility of Identifying Eating Moments from First-Person Images Leveraging Human Computation},

author = {Edison Thomaz and Aman Parnami and Irfan Essa and Gregory Abowd},

doi = {10.1145/2526667.2526672},

year  = {2013},

date = {2013-01-01},

urldate = {2013-01-01},

booktitle = {Proceedings of ACM International SenseCam and Pervasive Imaging (SenseCam '13)},

keywords = {activity recognition, behavioral imaging, computational health, ubiquitous computing, wearable computing},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2011-Thomaz-ITLADLAML,

title = {Interactive Techniques for Labeling Activities Of Daily Living to Assist Machine Learning},

author = {Edison Thomaz and Thoma Pleotz and Irfan Essa and Gregory Abowd},

url = {https://wish2011.wordpress.com/accepted-papers/

https://users.ece.utexas.edu/~ethomaz/papers/w1.pdf},

year  = {2011},

date = {2011-11-01},

urldate = {2011-11-01},

booktitle = {Proceedings of Workshop on Interactive Systems in Healthcare},

abstract = {Over the next decade, as healthcare continues its march away from the hospital and towards the home, logging and making sense of activities of daily living will play a key role in health modeling and life-long home care. Machine learning research has explored ways to automate the detection and quantification of these activities in sensor-rich environments. While we continue to make progress in developing practical and cost-effective activity sensing techniques, one large hurdle remains, obtaining labeled activity data to train activity recognition systems. In this paper, we discuss the process of gathering ground truth data with human participation for health modeling applications. In particular, we propose a criterion and design space containing five dimensions that we have identified as central to the characterization and evaluation of interactive labeling methods.

},

keywords = {activity recognition, behavioral imaging, computational health, wearable computing},

pubstate = {published},

tppubtype = {inproceedings}

}