@article {1364,
	title = {Deep Captioning with Multimodal Recurrent Neural Networks (m-RNN)},
	number = {033},
	year = {2015},
	month = {05/07/2015},
	abstract = {<p>In this paper, we present a multimodal Recurrent Neural Network (m-RNN) model for generating novel image captions. It directly models the probability distribution of generating a word given previous words and an image. Image captions are generated according to this distribution. The model consists of two sub-networks: a deep recurrent neural network for sentences and a deep convolutional network for images. These two sub-networks interact with each other in a multimodal layer to form the whole m-RNN model. The effectiveness of our model is validated on four benchmark datasets (IAPR TC-12, Flickr 8K, Flickr 30K and MS COCO). Our model outperforms the state-of-the-art methods. In addition, the m-RNN model can be applied to retrieval tasks for retrieving images or sentences, and achieves significant performance improvement over the state-of-the-art methods which directly<br />
optimize the ranking objective function for retrieval.</p>
},
	author = {Junhua Mao and Wei Xu and Yi Yang and Jiang Wang and Zhiheng Huang and Alan Yuille}
}
@conference {1982,
	title = {Learning like a Child: Fast Novel Visual Concept Learning from Sentence Descriptions of Images},
	booktitle = {International Conference of Computer Vision},
	year = {2015},
	month = {12/2015},
	address = {Santiago, Chile},
	abstract = {<pre>
In this paper, we address the task of learning novel visual concepts, and their interactions with other concepts, from a few images with sentence descriptions.
</pre>

<pre>
Using linguistic context and visual features, our method is able to efficiently hypothesize the semantic meaning of new words and add them to its word dictionary so that they can be used to describe images which contain these novel concepts.</pre>

<pre>
Our method has an image captioning module based on m-RNN with several improvements.</pre>

<pre>
In particular, we propose a transposed weight sharing scheme, which not only improves performance on image captioning, but also makes the model more suitable for the novel concept learning task.</pre>

<pre>
We propose methods to prevent <u>overfitting</u> the new concepts. </pre>

<pre>
In addition, three novel concept <u>datasets</u> are constructed for this new task, and are publicly available on the project page.</pre>

<pre>
In the experiments, we show that our method effectively learns novel visual concepts from a few examples without disturbing the previously learned concepts.</pre>

<pre>
The project page is: \url{<u>www</u>.stat.<u>ucla</u>.<u>edu</u>/~<u>junhua</u>.<u>mao</u>/projects/<u>child_learning</u>.<u>html</u>}.</pre>
},
	url = {www.stat.ucla.edu/~junhua.mao/projects/child_learning.html},
	author = {Junhua Mao and Wei Xu and Yi Yang and Jiang Wang and Zhiheng Huang and Alan Yuille}
}
