Publications

@inproceedings{2021-Cartillier-SMBASRFEV,

title = {Semantic MapNet: Building Allocentric SemanticMaps and Representations from Egocentric Views},

author = {Vincent Cartillier and Zhile Ren and Neha Jain and Stefan Lee and Irfan Essa and Dhruv Batra},

url = {https://arxiv.org/abs/2010.01191

https://vincentcartillier.github.io/smnet.html

https://ojs.aaai.org/index.php/AAAI/article/view/16180/15987},

doi = {10.48550/arXiv.2010.01191},

year  = {2021},

date = {2021-02-01},

urldate = {2021-02-01},

booktitle = {Proceedings of American Association of Artificial Intelligence Conference (AAAI)},

publisher = {AAAI},

abstract = {We study the task of semantic mapping -- specifically, an embodied agent (a robot or an egocentric AI assistant) is given a tour of a new environment and asked to build an allocentric top-down semantic map (`what is where?') from egocentric observations of an RGB-D camera with known pose (via localization sensors). Importantly, our goal is to build neural episodic memories and spatio-semantic representations of 3D spaces that enable the agent to easily learn subsequent tasks in the same space -- navigating to objects seen during the tour (`Find chair') or answering questions about the space (`How many chairs did you see in the house?'). 

Towards this goal, we present Semantic MapNet (SMNet), which consists of: (1) an Egocentric 

 

Visual Encoder that encodes each egocentric RGB-D frame, (2) a Feature Projector that projects egocentric features to appropriate locations on a floor-plan, (3) a Spatial Memory Tensor of size floor-plan length × width × feature-dims that learns to accumulate projected egocentric features, and (4) a Map Decoder that uses the memory tensor to produce semantic top-down maps. SMNet combines the strengths of (known) projective camera geometry and neural representation learning. On the task of semantic mapping in the Matterport3D dataset, SMNet significantly outperforms competitive baselines by 4.01-16.81% (absolute) on mean-IoU and 3.81-19.69% (absolute) on Boundary-F1 metrics. Moreover, we show how to use the spatio-semantic allocentric representations build by SMNet for the task of ObjectNav and Embodied Question Answering.},

keywords = {AAAI, AI, embodied agents, first-person vision},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2002-Abowd-AHDTSA,

title = {The Aware Home: Developing Technologies for Successful Aging},

author = {Gregory Abowd and Aaron Bobick and Irfan Essa and Elizabeth Mynatt and Wendy Rogers},

url = {https://www.academia.edu/13330406/The_aware_home_A_living_laboratory_for_technologies_for_successful_aging},

year  = {2002},

date = {2002-08-01},

urldate = {2002-08-01},

booktitle = {AAAI Workshop on Automation as a Care Giver},

address = {Edmonton, Alberta, Canada},

organization = {AAAI},

series = {Held in conjunction with American Association of Artificial Intelligence (AAAI) Conference 2002},

abstract = {We describe our ongoing research in the area of developing and testing technologies for successful aging. Sensing and perception technologies can enable a home environment to be aware of the whereabouts and activities of its occupants. Motivated by the desire to use such an awareness to help maintain independence and quality of life for an aging population, we describe the technological, design and engineering research challenges inherent in this problem domain. Our work is situated in the Georgia Tech Broadband Institute's Residential Laboratory, a unique living laboratory for this exploration of ubiquitous computing in a domestic setting.},

keywords = {AAAI, aging-in-place, intelligent environments},

pubstate = {published},

tppubtype = {inproceedings}

}