Publications

@inproceedings{2020-Lee-NDNGLGWC,

title = {Neural Design Network: Graphic Layout Generation with Constraints},

author = {Hsin-Ying Lee and Lu Jiang and Irfan Essa and Madison Le and Haifeng Gong and Ming-Hsuan Yang and Weilong Yang},

url = {https://arxiv.org/abs/1912.09421

https://rdcu.be/c7sqw},

doi = {10.1007/978-3-030-58580-8_29},

year  = {2020},

date = {2020-08-01},

urldate = {2020-08-01},

booktitle = {Proceedings of European Conference on Computer Vision (ECCV)},

keywords = {computer vision, content creation, ECCV, generative media, google},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Hickson-FFLSRSNP,

title = {Floors are Flat: Leveraging Semantics for Real-Time Surface Normal Prediction},

author = {Steven Hickson and Karthik Raveendran and Alireza Fathi and Kevin Murphy and Irfan Essa},

url = {https://arxiv.org/abs/1906.06792

https://openaccess.thecvf.com/content_ICCVW_2019/papers/GMDL/Hickson_Floors_are_Flat_Leveraging_Semantics_for_Real-Time_Surface_Normal_Prediction_ICCVW_2019_paper.pdf},

doi = {10.1109/ICCVW.2019.00501},

year  = {2019},

date = {2019-10-01},

urldate = {2019-10-01},

booktitle = {IEEE International Conference on Computer Vision (ICCV) Workshop on Geometry Meets Deep Learning},

abstract = {We propose 4 insights that help to significantly improve the performance of deep learning models that predict surface normals and semantic labels from a single RGB image. These insights are: (1) denoise the "ground truth" surface normals in the training set to ensure consistency with the semantic labels; (2) concurrently train on a mix of real and synthetic data, instead of pretraining on synthetic and fine-tuning on real; (3) jointly predict normals and semantics using a shared model, but only backpropagate errors on pixels that have valid training labels; (4) slim down the model and use grayscale instead of color inputs. Despite the simplicity of these steps, we demonstrate consistently improved state of the art results on several datasets, using a model that runs at 12 fps on a standard mobile phone.

},

howpublished = {arXiv preprint arXiv:1906.06792},

keywords = {computer vision, google, ICCV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Alamri-AVSD,

title = {Audio Visual Scene-Aware Dialog},

author = {Huda Alamri and Vincent Cartillier and Abhishek Das and Jue Wang and Anoop Cherian and Irfan Essa and Dhruv Batra and Tim K. Marks and Chiori Hori and Peter Anderson and Stefan Lee and Devi Parikh},

url = {https://openaccess.thecvf.com/content_CVPR_2019/papers/Alamri_Audio_Visual_Scene-Aware_Dialog_CVPR_2019_paper.pdf

https://video-dialog.com/

https://arxiv.org/abs/1901.09107},

doi = {10.1109/CVPR.2019.00774},

year  = {2019},

date = {2019-06-01},

urldate = {2019-06-01},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

abstract = {We introduce the task of scene-aware dialog. Our goal is to generate a complete and natural response to a question about a scene, given video and audio of the scene and the history of previous turns in the dialog. To answer successfully, agents must ground concepts from the question in the video while leveraging contextual cues from the dialog history. To benchmark this task, we introduce the Audio Visual Scene-Aware Dialog (AVSD) Dataset. For each of more than 11,000 videos of human actions from the Charades dataset, our dataset contains a dialog about the video, plus a final summary of the video by one of the dialog participants. We train several baseline systems for this task and evaluate the performance of the trained models using both qualitative and quantitative metrics. Our results indicate that models must utilize all the available inputs (video, audio, question, and dialog history) to perform best on this dataset.

},

keywords = {computational video, computer vision, CVPR, embodied agents, vision & language},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Wijmans-EQAPEWPCP,

title = {Embodied Question Answering in Photorealistic Environments With Point Cloud Perception},

author = {Erik Wijmans and Samyak Datta and Oleksandr Maksymets and Abhishek Das and Georgia Gkioxari and Stefan Lee and Irfan Essa and Devi Parikh and Dhruv Batra},

doi = {10.1109/CVPR.2019.00682},

year  = {2019},

date = {2019-06-01},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

keywords = {computer vision, CVPR, vision & language},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Ahsan-VJULSCVAR,

title = {Video Jigsaw: Unsupervised Learning of Spatiotemporal Context for Video Action Recognition},

author = {Unaiza Ahsan and Rishi Madhok and Irfan Essa},

url = {https://ieeexplore.ieee.org/abstract/document/8659002},

doi = {10.1109/WACV.2019.00025},

issn = {1550-5790},

year  = {2019},

date = {2019-01-01},

urldate = {2019-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

pages = {179-189},

keywords = {activity recognition, computer vision, machine learning, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2017-Ahsan-TUVAIISSE,

title = {Towards Using Visual Attributes to Infer Image Sentiment Of Social Events},

author = {Unaiza Ahsan and Munmun De Choudhury and Irfan Essa},

url = {https://ieeexplore.ieee.org/abstract/document/7966013},

doi = {10.1109/IJCNN.2017.7966013},

year  = {2017},

date = {2017-05-01},

urldate = {2017-05-01},

booktitle = {Proceedings of The International Joint Conference on Neural Networks},

publisher = {International Neural Network Society},

address = {Anchorage, Alaska, US},

abstract = {Widespread and pervasive adoption of smartphones has led to instant sharing of photographs that capture events ranging from mundane to life-altering happenings. We propose to capture sentiment information of such social event images leveraging their visual content. Our method extracts an intermediate visual representation of social event images based on the visual attributes that occur in the images going beyond sentiment-specific attributes. We map the top predicted attributes to sentiments and extract the dominant emotion associated with a picture of a social event. Unlike recent approaches, our method generalizes to a variety of social events and even to unseen events, which are not available at training time. We demonstrate the effectiveness of our approach on a challenging social event image dataset and our method outperforms state-of-the-art approaches for classifying complex event images into sentiments.

},

keywords = {computational journalism, computer vision, IJNN, machine learning},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2017-Ahsan-CERFIWTE,

title = {Complex Event Recognition from Images with Few Training Examples},

author = {Unaiza Ahsan and Chen Sun and James Hays and Irfan Essa},

url = {https://arxiv.org/abs/1701.04769

https://www.computer.org/csdl/proceedings-article/wacv/2017/07926663/12OmNzZEAzy},

doi = {10.1109/WACV.2017.80},

year  = {2017},

date = {2017-03-01},

urldate = {2017-03-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

abstract = {We propose to leverage concept-level representations for complex event recognition in photographs given limited training examples. We introduce a novel framework to discover event concept attributes from the web and use that to extract semantic features from images and classify them into social event categories with few training examples. Discovered concepts include a variety of objects, scenes, actions and event sub-types, leading to a discriminative and compact representation for event images. Web images are obtained for each discovered event concept and we use (pretrained) CNN features to train concept classifiers. Extensive experiments on challenging event datasets demonstrate that our proposed method outperforms several baselines using deep CNN features directly in classifying images into events with limited training examples. We also demonstrate that our method achieves the best overall accuracy on a dataset with unseen event categories using a single training example.

},

keywords = {activity recognition, computer vision, machine learning, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2016-Zia-FDASTD,

title = {Fine-tuning Deep Architectures for Surgical Tool Detection},

author = {Aneeq Zia and Daniel Castro and Irfan Essa},

url = {http://www.cc.gatech.edu/cpl/projects/deepm2cai/

https://www.cc.gatech.edu/cpl/projects/deepm2cai/paper.pdf},

year  = {2016},

date = {2016-10-01},

urldate = {2016-10-01},

booktitle = {Workshop and Challenges on Modeling and Monitoring of Computer Assisted Interventions (M2CAI), Held in Conjunction with International Conference on Medical Image Computing and Computer Assisted Intervention (MICCAI)},

address = {Athens, Greece},

abstract = {Understanding surgical workflow has been a key concern of the medical research community. One of the main advantages of surgical workflow detection is real time operating room (OR) scheduling. For hospitals, each minute of OR time is important in order to reduce cost and increase patient throughput. Traditional approaches in this field generally tackle the video analysis using hand crafted video features to facilitate the tool detection. Recently, Twinanda et al presented a CNN architecture ’EndoNet’ which outperformed previous methods for both surgical tool detection and surgical phase detection. Given the recent success of these networks, we present a study of various architectures coupled with a submission to the M2CAI Surgical Tool Detection challenge. We achieved a top-3 result for the M2CAI competition with a mAP of 37.6.

},

keywords = {activity assessment, computer vision, MICCAI, surgical training},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2016-Bettadapura-LCCGBH,

title = {Leveraging Contextual Cues for Generating Basketball Highlights},

author = {Vinay Bettadapura and Caroline Pantofaru and Irfan Essa},

url = {https://dl.acm.org/doi/10.1145/2964284.2964286

http://www.vbettadapura.com/highlights/basketball/index.htm},

doi = {10.1145/2964284.2964286},

year  = {2016},

date = {2016-10-01},

urldate = {2016-10-01},

booktitle = {ACM International Conference on Multimedia (ACM-MM)},

organization = {ACM},

abstract = {The massive growth of sports videos has resulted in a need for automatic generation of sports highlights that are comparable in quality to the hand-edited highlights produced by broadcasters such as ESPN. Unlike previous works that mostly use audio-visual cues derived from the video, we propose an approach that additionally leverages contextual cues derived from the environment that the game is being played in. The contextual cues provide information about the excitement levels in the game, which can be ranked and selected to automatically produce high-quality basketball highlights. We introduce a new dataset of 25 NCAA games along with their play-by-play stats and the ground-truth excitement data for each basket. We explore the informativeness of five different cues derived from the video and from the environment through user studies. Our experiments show that for our study participants, the highlights produced by our system are comparable to the ones produced by ESPN for the same games.},

keywords = {ACM, ACMMM, activity recognition, computational video, computer vision, sports visualization, video summarization},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2016-Castro-DPHFEVV,

title = {Discovering Picturesque Highlights from Egocentric Vacation Video},

author = {Daniel Castro and Vinay Bettadapura and Irfan Essa},

url = {https://ieeexplore.ieee.org/document/7477707

http://www.cc.gatech.edu/cpl/projects/egocentrichighlights/

https://youtu.be/lIONi21y-mk},

doi = {10.1109/WACV.2016.7477707},

year  = {2016},

date = {2016-03-01},

urldate = {2016-03-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

abstract = {We present an approach for identifying picturesque highlights from large amounts of egocentric video data. Given a set of egocentric videos captured over the course of a vacation, our method analyzes the videos and looks for images that have good picturesque and artistic properties. We introduce novel techniques to automatically determine aesthetic features such as composition, symmetry and color vibrancy in egocentric videos and rank the video frames based on their photographic qualities to generate highlights. Our approach also uses contextual information such as GPS, when available, to assess the relative importance of each geographic location where the vacation videos were shot. Furthermore, we specifically leverage the properties of egocentric videos to improve our highlight detection. We demonstrate results on a new egocentric vacation dataset which includes 26.5 hours of videos taken over a 14 day vacation that spans many famous tourist destinations and also provide results from a user-study to access our results.

},

keywords = {computational photography, computational video, computer vision, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2015-Castro-PDAFEIUDL,

title = {Predicting Daily Activities from Egocentric Images Using Deep Learning},

author = {Daniel Castro and Steven Hickson and Vinay Bettadapura and Edison Thomaz and Gregory Abowd and Henrik Christensen and Irfan Essa},

url = {https://dl.acm.org/doi/10.1145/2802083.2808398

https://arxiv.org/abs/1510.01576

http://www.cc.gatech.edu/cpl/projects/dailyactivities/

},

doi = {10.1145/2802083.2808398},

year  = {2015},

date = {2015-09-01},

urldate = {2015-09-01},

booktitle = {Proceedings of International Symposium on Wearable Computers (ISWC)},

abstract = {We present a method to analyze images taken from a passive egocentric wearable camera along with contextual information, such as time and day of the week, to learn and predict the everyday activities of an individual. We collected a dataset of 40,103 egocentric images over 6 months with 19 activity classes and demonstrate the benefit of state-of-the-art deep learning techniques for learning and predicting daily activities. Classification is conducted using a Convolutional Neural Network (CNN) with a classification method we introduce called a late fusion ensemble. This late fusion ensemble incorporates relevant contextual information and increases our classification accuracy. Our technique achieves an overall accuracy of 83.07% in predicting a person's activity across the 19 activity classes. We also demonstrate some promising results from two additional users by fine-tuning the classifier with one day of training data.},

keywords = {activity recognition, computer vision, ISWC, machine learning, wearable computing},

pubstate = {published},

tppubtype = {inproceedings}

}