Publications

@techreport{2023-Sohn-LDPCIS,

title = {Learning Disentangled Prompts for Compositional Image Synthesis},

author = {Kihyuk Sohn and Albert Shaw and Yuan Hao and Han Zhang and Luisa Polania and Huiwen Chang and Lu Jiang and Irfan Essa},

url = {https://arxiv.org/abs/2306.00763},

doi = { https://doi.org/10.48550/arXiv.2306.00763},

year  = {2023},

date = {2023-06-01},

urldate = {2023-06-01},

abstract = {We study domain-adaptive image synthesis, the problem of teaching pretrained image generative models a new style or concept from as few as one image to synthesize novel images, to better understand the compositional image synthesis. We present a framework that leverages a pre-trained class-conditional generation model and visual prompt tuning. Specifically, we propose a novel source class distilled visual prompt that learns disentangled prompts of semantic (e.g., class) and domain (e.g., style) from a few images. Learned domain prompt is then used to synthesize images of any classes in the style of target domain. We conduct studies on various target domains with the number of images ranging from one to a few to many, and show qualitative results which show the compositional generalization of our method. Moreover, we show that our method can help improve zero-shot domain adaptation classification accuracy.

},

howpublished = {arXiv:2306.00763 },

keywords = {arXiv, computer vision, generative AI, google, prompt engineering},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2023-Haresamudram-TLDRSWHAR,

title = {Towards Learning Discrete Representations via Self-Supervision for Wearables-Based Human Activity Recognition},

author = {Harish Haresamudram and Irfan Essa and Thomas Ploetz},

url = {https://arxiv.org/abs/2306.01108},

doi = {10.48550/arXiv.2306.01108},

year  = {2023},

date = {2023-06-01},

urldate = {2023-06-01},

abstract = {Human activity recognition (HAR) in wearable computing is typically based on direct processing of sensor data. Sensor readings are translated into representations, either derived through dedicated preprocessing, or integrated into end-to-end learning. Independent of their origin, for the vast majority of contemporary HAR, those representations are typically continuous in nature. That has not always been the case. In the early days of HAR, discretization approaches have been explored - primarily motivated by the desire to minimize computational requirements, but also with a view on applications beyond mere recognition, such as, activity discovery, fingerprinting, or large-scale search. Those traditional discretization approaches, however, suffer from substantial loss in precision and resolution in the resulting representations with detrimental effects on downstream tasks. Times have changed and in this paper we propose a return to discretized representations. We adopt and apply recent advancements in Vector Quantization (VQ) to wearables applications, which enables us to directly learn a mapping between short spans of sensor data and a codebook of vectors, resulting in recognition performance that is generally on par with their contemporary, continuous counterparts - sometimes surpassing them. Therefore, this work presents a proof-of-concept for demonstrating how effective discrete representations can be derived, enabling applications beyond mere activity classification but also opening up the field to advanced tools for the analysis of symbolic sequences, as they are known, for example, from domains such as natural language processing. Based on an extensive experimental evaluation on a suite of wearables-based benchmark HAR tasks, we demonstrate the potential of our learned discretization scheme and discuss how discretized sensor data analysis can lead to substantial changes in HAR.},

howpublished = {arXiv:2306.01108},

keywords = {activity recognition, arXiv, wearable computing},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2021-Beedu-VEOPFV,

title = {VideoPose: Estimating 6D object pose from videos},

author = {Apoorva Beedu and Zhile Ren and Varun Agrawal and Irfan Essa},

url = {https://arxiv.org/abs/2111.10677},

doi = {10.48550/arXiv.2111.10677},

year  = {2021},

date = {2021-11-01},

urldate = {2021-11-01},

journal = {arXiv preprint arXiv:2111.10677},

abstract = {We introduce a simple yet effective algorithm that uses convolutional neural networks to directly estimate object poses from videos. Our approach leverages the temporal information from a video sequence, and is computationally efficient and robust to support robotic and AR domains. Our proposed network takes a pre-trained 2D object detector as input, and aggregates visual features through a recurrent neural network to make predictions at each frame. Experimental evaluation on the YCB-Video dataset show that our approach is on par with the state-of-the-art algorithms. Further, with a speed of 30 fps, it is also more efficient than the state-of-the-art, and therefore applicable to a variety of applications that require real-time object pose estimation.},

keywords = {arXiv, computer vision, object detection, pose estimation},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2021-Samel-NTLP,

title = {Neural Temporal Logic Programming},

author = {Karan Samel and Zelin Zhao and Binghong Chen and Shuang Li and Dharmashankar Subramanian and Irfan Essa and Le Song},

url = {https://openreview.net/forum?id=i7h4M45tU8},

year  = {2021},

date = {2021-09-01},

urldate = {2021-09-01},

abstract = {Events across a timeline are a common data representation, seen in different temporal modalities. Individual atomic events can occur in a certain temporal ordering to compose higher-level composite events. Examples of a composite event are a patient's medical symptom or a baseball player hitting a home run, caused distinct temporal orderings of patient vitals and player movements respectively. Such salient composite events are provided as labels in temporal datasets and most works optimize models to predict these composite event labels directly. We focus uncovering the underlying atomic events and their relations that lead to the composite events within a noisy temporal data setting. We propose Neural Temporal Logic Programming (Neural TLP) which first learns implicit temporal relations between atomic events and then lifts logic rules for composite events, given only the composite events labels for supervision. This is done through efficiently searching through the combinatorial space of all temporal logic rules in an end-to-end differentiable manner. We evaluate our method on video and on healthcare data where it outperforms the baseline methods for rule discovery. },

howpublished = {https://openreview.net/forum?id=i7h4M45tU8},

keywords = {activity recognition, arXiv, machine learning, openreview},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2021-Scarafoni-PPLANBSAP,

title = {PLAN-B: Predicting Likely Alternative Next Best Sequences for Action Prediction},

author = {Dan Scarafoni and Irfan Essa and Thomas Ploetz},

url = {https://arxiv.org/abs/2103.15987},

doi = {10.48550/arXiv.2103.15987},

year  = {2021},

date = {2021-03-01},

urldate = {2021-03-01},

journal = {arXiv},

number = {arXiv:2103.15987},

abstract = {Action prediction focuses on anticipating actions before they happen. Recent works leverage probabilistic approaches to describe future uncertainties and sample future actions. However, these methods cannot easily find all alternative predictions, which are essential given the inherent unpredictability of the future, and current evaluation protocols do not measure a system's ability to find such alternatives. We re-examine action prediction in terms of its ability to predict not only the top predictions, but also top alternatives with the accuracy@k metric. In addition, we propose Choice F1: a metric inspired by F1 score which evaluates a prediction system's ability to find all plausible futures while keeping only the most probable ones. To evaluate this problem, we present a novel method, Predicting the Likely Alternative Next Best, or PLAN-B, for action prediction which automatically finds the set of most likely alternative futures. PLAN-B consists of two novel components: (i) a Choice Table which ensures that all possible futures are found, and (ii) a "Collaborative" RNN system which combines both action sequence and feature information. We demonstrate that our system outperforms state-of-the-art results on benchmark datasets.

},

keywords = {activity recognition, arXiv, computer vision},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2020-Wijmans-AVRENT,

title = {Analyzing Visual Representations in Embodied Navigation Tasks},

author = {Erik Wijmans and Julian Straub and Dhruv Batra and Irfan Essa and Judy Hoffman and Ari Morcos},

url = {https://arxiv.org/abs/2003.05993

https://arxiv.org/pdf/2003.05993},

doi = {10.48550/arXiv.2003.05993},

year  = {2020},

date = {2020-03-01},

urldate = {2020-03-01},

journal = {arXiv},

number = {arXiv:2003.05993},

abstract = {Recent advances in deep reinforcement learning require a large amount of training data and generally result in representations that are often over specialized to the target task. In this work, we present a methodology to study the underlying potential causes for this specialization. We use the recently proposed projection weighted Canonical Correlation Analysis (PWCCA) to measure the similarity of visual representations learned in the same environment by performing different tasks. 



We then leverage our proposed methodology to examine the task dependence of visual representations learned on related but distinct embodied navigation tasks. Surprisingly, we find that slight differences in task have no measurable effect on the visual representation for both SqueezeNet and ResNet architectures. We then empirically demonstrate that visual representations learned on one task can be effectively transferred to a different task.},

howpublished = {arXiv:2003.05993},

keywords = {arXiv, embodied agents, navigation},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2018-Balloch-USSRPUSDFT,

title = {Unbiasing Semantic Segmentation For Robot Perception using Synthetic Data Feature Transfer},

author = {Jonathan C Balloch and Varun Agrawal and Irfan Essa and Sonia Chernova},

url = {https://doi.org/10.48550/arXiv.1809.03676},

doi = {10.48550/arXiv.1809.03676},

year  = {2018},

date = {2018-09-01},

urldate = {2018-09-01},

journal = {arXiv},

number = {arXiv:1809.03676},

abstract = {Robot perception systems need to perform reliable image segmentation in real-time on noisy, raw perception data. State-of-the-art segmentation approaches use large CNN models and carefully constructed datasets; however, these models focus on accuracy at the cost of real-time inference. Furthermore, the standard semantic segmentation datasets are not large enough for training CNNs without augmentation and are not representative of noisy, uncurated robot perception data. We propose improving the performance of real-time segmentation frameworks on robot perception data by transferring features learned from synthetic segmentation data. We show that pretraining real-time segmentation architectures with synthetic segmentation data instead of ImageNet improves fine-tuning performance by reducing the bias learned in pretraining and closing the textit{transfer gap} as a result. Our experiments show that our real-time robot perception models pretrained on synthetic data outperform those pretrained on ImageNet for every scale of fine-tuning data examined. Moreover, the degree to which synthetic pretraining outperforms ImageNet pretraining increases as the availability of robot data decreases, making our approach attractive for robotics domains where dataset collection is hard and/or expensive.

},

howpublished = {arXiv:1809.03676},

keywords = {arXiv, robotics, scene understanding},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2018-Hickson-OCLRWWS,

title = {Object category learning and retrieval with weak supervision},

author = {Steven Hickson and Anelia Angelova and Irfan Essa and Rahul Sukthankar},

url = {https://arxiv.org/abs/1801.08985

https://arxiv.org/pdf/1801.08985},

doi = {10.48550/arXiv.1801.08985},

year  = {2018},

date = {2018-07-01},

urldate = {2018-07-01},

journal = {arXiv},

number = {arXiv:1801.08985},

abstract = {We consider the problem of retrieving objects from image data and learning to classify them into meaningful semantic categories with minimal supervision. To that end, we propose a fully differentiable unsupervised deep clustering approach to learn semantic classes in an end-to-end fashion without individual class labeling using only unlabeled object proposals. The key contributions of our work are 1) a kmeans clustering objective where the clusters are learned as parameters of the network and are represented as memory units, and 2) simultaneously building a feature representation, or embedding, while learning to cluster it. This approach shows promising results on two popular computer vision datasets: on CIFAR10 for clustering objects, and on the more complex and challenging Cityscapes dataset for semantically discovering classes which visually correspond to cars, people, and bicycles. Currently, the only supervision provided is segmentation objectness masks, but this method can be extended to use an unsupervised objectness-based object generation mechanism which will make the approach completely unsupervised.

},

howpublished = {arXiv:1801.08985},

keywords = {arXiv, computer vision, machine learning, object detection},

pubstate = {published},

tppubtype = {techreport}

}

@techreport{2018-Alamri-AVSDACD,

title = {Audio Visual Scene-Aware Dialog (AVSD) Challenge at DSTC7},

author = {Huda Alamri and Vincent Cartillier and Raphael Gontijo Lopes and Abhishek Das and Jue Wang and Irfan Essa and Dhruv Batra and Devi Parikh and Anoop Cherian and Tim K Marks and Chiori Hori},

url = {https://video-dialog.com/

https://arxiv.org/abs/1806.00525},

doi = {10.48550/arXiv.1806.00525},

year  = {2018},

date = {2018-06-01},

urldate = {2018-06-01},

journal = {arXiv},

number = {arXiv:1806.00525},

abstract = {Scene-aware dialog systems will be able to have conversations with users about the objects and events around them. Progress on such systems can be made by integrating state-of-the-art technologies from multiple research areas including end-to-end dialog systems visual dialog, and video description. We introduce the Audio Visual Scene Aware Dialog (AVSD) challenge and dataset. In this challenge, which is one track of the 7th Dialog System Technology Challenges (DSTC7) workshop1, the task is to build a system that generates responses in a dialog about an input video

},

howpublished = {arXiv:1806.00525},

keywords = {arXiv, embodied agents, multimedia, vision & language},

pubstate = {published},

tppubtype = {techreport}

}