Publications

@inproceedings{2023-Kumar-WIALDHRBULGIM,

title = {Words into Action: Learning Diverse Humanoid Robot Behaviors using Language Guided Iterative Motion Refinement},

author = {K. Niranjan Kumar and Irfan Essa and Sehoon Ha},

url = {https://doi.org/10.48550/arXiv.2310.06226

https://arxiv.org/abs/2310.06226

https://arxiv.org/pdf/2310.06226.pdf

https://www.kniranjankumar.com/words_into_action/

},

doi = {10.48550/arXiv.2310.06226},

year  = {2023},

date = {2023-11-01},

urldate = {2023-11-01},

booktitle = {CoRL Workshop on Language and Robot Learning Language as Grounding (with CoRL 2023)},

abstract = {We present a method to simplify controller design by enabling users to train and fine-tune robot control policies using natural language commands. We first learn a neural network policy that generates behaviors given a natural language command, such as “walk forward”, by combining Large Language Models (LLMs), motion retargeting, and motion imitation. Based on the synthesized motion, we iteratively fine-tune by updating the text prompt and querying LLMs to find the best checkpoint associated with the closest motion in history.},

keywords = {arXiv, CoRL, robotics, vision & language},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Wijmans-EQAPEWPCP,

title = {Embodied Question Answering in Photorealistic Environments With Point Cloud Perception},

author = {Erik Wijmans and Samyak Datta and Oleksandr Maksymets and Abhishek Das and Georgia Gkioxari and Stefan Lee and Irfan Essa and Devi Parikh and Dhruv Batra},

doi = {10.1109/CVPR.2019.00682},

year  = {2019},

date = {2019-06-01},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

keywords = {computer vision, CVPR, vision & language},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2019-Alamri-AVSD,

title = {Audio Visual Scene-Aware Dialog},

author = {Huda Alamri and Vincent Cartillier and Abhishek Das and Jue Wang and Anoop Cherian and Irfan Essa and Dhruv Batra and Tim K. Marks and Chiori Hori and Peter Anderson and Stefan Lee and Devi Parikh},

url = {https://openaccess.thecvf.com/content_CVPR_2019/papers/Alamri_Audio_Visual_Scene-Aware_Dialog_CVPR_2019_paper.pdf

https://video-dialog.com/

https://arxiv.org/abs/1901.09107},

doi = {10.1109/CVPR.2019.00774},

year  = {2019},

date = {2019-06-01},

urldate = {2019-06-01},

booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},

abstract = {We introduce the task of scene-aware dialog. Our goal is to generate a complete and natural response to a question about a scene, given video and audio of the scene and the history of previous turns in the dialog. To answer successfully, agents must ground concepts from the question in the video while leveraging contextual cues from the dialog history. To benchmark this task, we introduce the Audio Visual Scene-Aware Dialog (AVSD) Dataset. For each of more than 11,000 videos of human actions from the Charades dataset, our dataset contains a dialog about the video, plus a final summary of the video by one of the dialog participants. We train several baseline systems for this task and evaluate the performance of the trained models using both qualitative and quantitative metrics. Our results indicate that models must utilize all the available inputs (video, audio, question, and dialog history) to perform best on this dataset.

},

keywords = {computational video, computer vision, CVPR, embodied agents, vision & language},

pubstate = {published},

tppubtype = {inproceedings}

}