@article {5011, title = {Three approaches to facilitate DNN generalization to objects in out-of-distribution orientations and illuminations}, number = {119}, year = {2022}, month = {01/2022}, abstract = {

The training data distribution is often biased towards objects in certain orientations and illumination conditions. While humans have a remarkable capability of recognizing objects in out-of-distribution (OoD) orientations and illuminations, Deep Neural Networks (DNNs) severely suffer in this case, even when large amounts of training examples are available. In this paper, we investigate three different approaches to improve DNNs in recognizing objects in OoD orientations and illuminations. Namely, these are (i) training much longer after convergence of the in-distribution (InD) validation accuracy, i.e., late-stopping, (ii) tuning the momentum parameter of the batch normalization layers, and (iii) enforcing invariance of the neural activity in an intermediate layer to orientation and illumination conditions. Each of these approaches substantially improves the DNN{\textquoteright}s OoD accuracy (more than 20\% in some cases). We report results in four datasets: two datasets are modified from the MNIST and iLab datasets, and the other two are novel (one of 3D rendered cars and another of objects taken from various controlled orientations and illumination conditions). These datasets allow to study the effects of different amounts of bias and are challenging as DNNs perform poorly in OoD conditions. Finally, we demonstrate that even though the three approaches focus on different aspects of DNNs, they all tend to lead to the same underlying neural mechanism to enable OoD accuracy gains {\textendash} individual neurons in the intermediate layers become more selective to a category and also invariant to OoD orientations and illuminations. We anticipate this study to be a basis for further improvement of deep neural networks{\textquoteright} OoD generalization performance, which is highly demanded to achieve safe and fair AI applications.

}, author = {Akira Sakai and Taro Sunagawa and Spandan Madan and Kanata Suzuki and Takashi Katoh and Hiromichi Kobashi and Hanspeter Pfister and Pawan Sinha and Xavier Boix and Tomotake Sasaki} } @article {5015, title = {Transformer Module Networks for Systematic Generalization in Visual Question Answering}, number = {121}, year = {2022}, month = {02/2022}, abstract = {Transformer-based models achieve great performance on Visual Question Answering (VQA). How- ever, when we evaluate them on systematic generalization, i.e., handling novel combinations of known concepts, their performance degrades. Neural Module Networks (NMNs) are a promising approach for systematic generalization that consists on composing modules, i.e., neural networks that tackle a sub-task. Inspired by Transformers and NMNs, we propose Transformer Module Network (TMN), a novel Transformer-based model for VQA that dynamically composes modules into a question-specific Transformer network. TMNs achieve state-of-the-art systematic generalization performance in three VQA datasets, namely, CLEVR-CoGenT, CLOSURE and GQA-SGL, in some cases improving more than 30\% over standard Transformers. }, author = {Moyuru Yamada and Vanessa D{\textquoteright}Amario and Kentaro Takemoto and Xavier Boix and Tomotake Sasaki} } @article {4583, title = {On the Capability of Neural Networks to Generalize to Unseen Category-Pose Combinations}, year = {2020}, month = {07/2020}, abstract = {

Recognizing an object{\textquoteright}s category and pose lies at the heart of visual understanding. Recent works suggest that deep neural networks (DNNs) often fail to generalize to category-pose combinations not seen during training. However, it is unclear when and how such generalization may be possible. Does the number of combinations seen during training impact generalization? Is it better to learn category and pose in separate networks, or in a single shared network? Furthermore, what are the neural mechanisms that drive the network{\textquoteright}s generalization? In this paper, we answer these questions by analyzing state-of-the-art DNNs trained to recognize both object category and pose (position, scale, and 3D viewpoint) with quantitative control over the number of category-pose combinations seen during training. We also investigate the emergence of two types of specialized neurons that can explain generalization to unseen combinations{\textemdash}neurons selective to category and invariant to pose, and vice versa. We perform experiments on MNIST extended with position or scale, the iLab dataset with vehicles at different viewpoints, and a challenging new dataset for car model recognition and viewpoint estimation that we introduce in this paper, the Biased-Cars dataset. Our results demonstrate that as the number of combinations seen during training increases, networks generalize better to unseen category-pose combinations, facilitated by an increase in the selectivity and invariance of individual neurons. We find that learning category and pose in separate networks compared to a shared one leads to an increase in such selectivity and invariance, as separate networks are not forced to preserve information about both category and pose. This enables separate networks to significantly outperform shared ones at predicting unseen category-pose combinations.

}, author = {Spandan Madan and Timothy Henry and Jamell Dozier and Helen Ho and Nishchal Bhandari and Tomotake Sasaki and Fredo Durand and Hanspeter Pfister and Xavier Boix} } @article {4478, title = {Do Neural Networks for Segmentation Understand Insideness?}, year = {2020}, month = {04/2020}, abstract = {

The insideness problem is an image segmentation modality that consists of determining which pixels are inside and outside a region. Deep Neural Networks (DNNs) excel in segmentation benchmarks, but it is unclear that they have the ability to solve the insideness problem as it requires evaluating long-range spatial dependencies. In this paper, the insideness problem is analyzed in isolation, without texture or semantic cues, such that other aspects of segmentation do not interfere in the analysis. We demonstrate that DNNs for segmentation with few units have sufficient complexity to solve insideness for any curve. Yet, such DNNs have severe problems to learn general solutions. Only recurrent networks trained with small images learn solutions that generalize well to almost any curve. Recurrent networks can decompose the evaluation of long-range dependencies into a sequence of local operations, and learning with small images alleviates the common difficulties of training recurrent networks with a large number of unrolling steps.

}, author = {Kimberly M. Villalobos and Vilim Štih and Amineh Ahmadinejad and Shobhita Sundaram and Jamell Dozier and Andrew Francl and Frederico Azevedo and Tomotake Sasaki and Xavier Boix} } @article {3962, title = {Can Deep Neural Networks Do Image Segmentation by Understanding Insideness?}, year = {2018}, month = {12/2018}, abstract = {

THIS MEMO IS REPLACED BY CBMM MEMO 105

A key component of visual cognition is the understanding of spatial relationships among objects. Albeit effortless to our visual system, state-of-the-art artificial neural networks struggle to distinguish basic spatial relationships among elements in an image. As shown here, deep neural networks (DNNs) trained with hundreds of thousands of labeled examples cannot accurately distinguish whether pixels lie inside or outside 2D shapes, a problem that seems much simpler than image segmentation. In this paper, we sought to analyze the capability of ANN to solve such inside/outside problems using an analytical approach. We demonstrate that it is a mathematically tractable problem and that two previously proposed algorithms, namely the Ray-Intersection Method and the Coloring Method, achieve perfect accuracy when implemented in the form of DNNs.

}, author = {Kimberly M. Villalobos and Jamel Dozier and Vilim Stih and Andrew Francl and Frederico Azevedo and Tomaso Poggio and Tomotake Sasaki and Xavier Boix} }