Publications

@inproceedings{2022-Kong-BLTCLG,

title = {BLT: Bidirectional Layout Transformer for Controllable Layout Generation},

author = {Xiang Kong and Lu Jiang and Huiwen Chang and Han Zhang and Yuan Hao and Haifeng Gong and Irfan Essa},

url = {https://arxiv.org/abs/2112.05112

https://rdcu.be/c61AE},

doi = {10.1007/978-3-031-19790-1_29},

isbn = {978-3-031-19789-5},

year  = {2022},

date = {2022-10-25},

urldate = {2022-10-25},

booktitle = {European Conference on Computer Vision (ECCV)},

volume = {13677},

abstract = {Creating visual layouts is a critical step in graphic design. Automatic generation of such layouts is essential for scalable and diverse visual designs. To advance conditional layout generation, we introduce BLT, a bidirectional layout transformer. BLT differs from previous work on transformers in adopting non-autoregressive transformers. In training, BLT learns to predict the masked attributes by attending to surrounding attributes in two directions. During inference, BLT first generates a draft layout from the input and then iteratively refines it into a high-quality layout by masking out low-confident attributes. The masks generated in both training and inference are controlled by a new hierarchical sampling policy. We verify the proposed model on six benchmarks of diverse design tasks. Experimental results demonstrate two benefits compared to the state-of-the-art layout transformer models. First, our model empowers layout transformers to fulfill controllable layout generation. Second, it achieves up to 10x speedup in generating a layout at inference time than the layout transformer baseline. Code is released at https://shawnkx.github.io/blt.},

keywords = {computer vision, ECCV, generative AI, generative media, google, vision transformer},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2022-Mao-DRSVTR,

title = {Discrete Representations Strengthen Vision Transformer Robustness},

author = {Chengzhi Mao and Lu Jiang and Mostafa Dehghani and Carl Vondrick and Rahul Sukthankar and Irfan Essa},

url = {https://iclr.cc/virtual/2022/poster/6647

https://arxiv.org/abs/2111.10493

https://research.google/pubs/pub51388/

https://openreview.net/forum?id=8hWs60AZcWk},

doi = {10.48550/arXiv.2111.10493},

year  = {2022},

date = {2022-01-28},

urldate = {2022-04-01},

booktitle = {Proceedings of International Conference on Learning Representations (ICLR)},

journal = {arXiv preprint arXiv:2111.10493},

abstract = {Vision Transformer (ViT) is emerging as the state-of-the-art architecture for image recognition. While recent studies suggest that ViTs are more robust than their convolutional counterparts, our experiments find that ViTs trained on ImageNet are overly reliant on local textures and fail to make adequate use of shape information. ViTs thus have difficulties generalizing to out-of-distribution, real-world data. To address this deficiency, we present a simple and effective architecture modification to ViT's input layer by adding discrete tokens produced by a vector-quantized encoder. Different from the standard continuous pixel tokens, discrete tokens are invariant under small perturbations and contain less information individually, which promote ViTs to learn global information that is invariant. Experimental results demonstrate that adding discrete representation on four architecture variants strengthens ViT robustness by up to 12% across seven ImageNet robustness benchmarks while maintaining the performance on ImageNet.},

keywords = {computer vision, google, machine learning, vision transformer},

pubstate = {published},

tppubtype = {inproceedings}

}