@conference {3575,
	title = {Generative modeling of audible shapes for object perception},
	booktitle = {The IEEE International Conference on Computer Vision (ICCV)},
	year = {2017},
	month = {10/2017},
	address = {Venice, Italy},
	abstract = {<p>Humans infer rich knowledge of objects from both auditory and visual cues. Building a machine of such competency, however, is very challenging, due to the great difficulty in capturing large-scale, clean data of objects with both their appearance and the sound they make. In this paper, we present a novel, open-source pipeline that generates audio-visual data, purely from 3D object shapes and their physical properties. Through comparison with audio recordings and human behavioral studies, we validate the accuracy of the sounds it generates. Using this generative model, we are able to construct a synthetic audio-visual dataset, namely Sound-20K, for object perception tasks. We demonstrate that auditory and visual information play complementary roles in object perception, and further, that the representation learned on synthetic audio-visual data can transfer to real-world scenarios.</p>
},
	url = {http://openaccess.thecvf.com/content_iccv_2017/html/Zhang_Generative_Modeling_of_ICCV_2017_paper.html},
	author = {Zhoutong Zhang and Jiajun Wu and Qiujia Li and Zhengjia Huang and James Traer and Josh H. McDermott and Joshua B. Tenenbaum and William T. Freeman}
}
@proceedings {3242,
	title = {Shape and Material from Sound},
	year = {2017},
	month = {12/2017},
	pages = {1278{\textendash}1288},
	address = {Long Beach, CA},
	abstract = {<p>What can we infer from hearing an object falling onto the ground? Based on knowledge of the physical world, humans are able to infer rich information from such limited data: rough shape of the object, its material, the height of falling, etc. In this paper, we aim to approximate such competency. We first mimic the human knowledge about the physical world using a fast physics-based generative model. Then, we present an analysis-by-synthesis approach to infer properties of the falling object. We further approximate human past experience by directly mapping audio to object properties using deep learning with self-supervision. We evaluate our method through behavioral studies, where we compare human predictions with ours on inferring object shape, material, and initial height of falling. Results show that our method achieves near-human performance, without any annotations.</p>
},
	url = {http://papers.nips.cc/paper/6727-shape-and-material-from-sound.pdf},
	author = {zhang, zhoutong and Qiujia Li and Zhengjia Huang and Jiajun Wu and Joshua B. Tenenbaum and William T. Freeman},
	editor = {I. Guyon and U. V. Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett}
}
