@article {5096, title = {Face neurons encode nonsemantic features}, journal = {Proceedings of the National Academy of Sciences}, volume = {119}, year = {2022}, month = {02/2022}, abstract = {

The primate inferior temporal cortex contains neurons that respond more strongly to faces than to other objects. Termed {\textquotedblleft}face neurons,{\textquotedblright} these neurons are thought to be selective for faces as a semantic category. However, face neurons also partly respond to clocks, fruits, and single eyes, raising the question of whether face neurons are better described as selective for visual features related to faces but dissociable from them. We used a recently described algorithm, XDream, to evolve stimuli that strongly activated face neurons. XDream leverages a generative neural network that is not limited to realistic objects. Human participants assessed images evolved for face neurons and for nonface neurons and natural images depicting faces, cars, fruits, etc. Evolved images were consistently judged to be distinct from real faces. Images evolved for face neurons were rated as slightly more similar to faces than images evolved for nonface neurons. There was a correlation among natural images between face neuron activity and subjective {\textquotedblleft}faceness{\textquotedblright} ratings, but this relationship did not hold for face neuron{\textendash}evolved images, which triggered high activity but were rated low in faceness. Our results suggest that so-called face neurons are better described as tuned to visual features rather than semantic categories.

}, issn = {0027-8424}, doi = {10.1073/pnas.2118705119}, url = {https://pnas.org/doi/full/10.1073/pnas.2118705119}, author = {Bardon, Alexandra and Will Xiao and Carlos R Ponce and Margaret S Livingstone and Gabriel Kreiman} } @conference {4821, title = {A Geometric Analysis of Deep Generative Image Models and Its Applications}, booktitle = {Proc. International Conference on Learning Representations, 2021}, year = {2021}, month = {01/2021}, abstract = {

Generative adversarial networks (GANs) have emerged as a powerful unsupervised method to model the statistical patterns of real-world data sets, such as natural images. These networks are trained to map random inputs in their latent space to new samples representative of the learned data. However, the structure of the latent space is hard to intuit due to its high dimensionality and the non-linearity of the generator, which limits the usefulness of the models. Understanding the latent space requires a way to identify input codes for existing real-world images (inversion), and a way to identify directions with known image transformations (interpretability). Here, we use a geometric framework to address both issues simultaneously. We develop an architecture-agnostic method to compute the Riemannian metric of the image manifold created by GANs. The eigen-decomposition of the metric isolates axes that account for different levels of image variability. An empirical analysis of several pretrained GANs shows that image variation around each position is concentrated along surprisingly few major axes (the space is highly anisotropic) and the directions that create this large variation are similar at different positions in the space (the space is homogeneous). We show that many of the top eigenvectors correspond to interpretable transforms in the image space, with a substantial part of eigenspace corresponding to minor transforms which could be compressed out. This geometric understanding unifies key previous results related to GAN interpretability. We show that the use of this metric allows for more efficient optimization in the latent space (e.g. GAN inversion) and facilitates unsupervised discovery of interpretable axes. Our results illustrate that defining the geometry of the GAN image manifold can serve as a general framework for understanding GANs.

}, author = {Binxu Wang and Carlos R Ponce} } @article {4146, title = {Evolving Images for Visual Neurons Using a Deep Generative Network Reveals Coding Principles and Neuronal Preferences}, journal = {Cell }, volume = {177}, year = {2019}, month = {05/2019}, pages = {1009}, chapter = {999}, abstract = {

What specific features should visual neurons encode, given the infinity of real-world images and the limited number of neurons available to represent them? We investigated neuronal selectivity in monkey inferotemporal cortex via the vast hypothesis space of a generative deep neural network, avoiding assumptions about features or semantic categories. A genetic algorithm searched this space for stimuli that maximized neuronal firing. This led to the evolution of rich synthetic images of objects with complex combinations of shapes, colors, and textures, sometimes resembling animals or familiar people, other times revealing novel patterns that did not map to any clear\ semantic category. These results expand our conception of the dictionary of features encoded in the cortex, and the approach can potentially reveal the internal representations of any system whose input can be captured by a generative model.

}, doi = {10.1016/j.cell.2019.04.005}, url = {https://www.cell.com/cell/fulltext/S0092-8674(19)30391-5}, author = {Carlos R Ponce and Will Xiao and Peter F Schade and Till S. Hartmann and Gabriel Kreiman and Margaret S Livingstone} } @article {3039, title = {Seeing faces is necessary for face-domain formation}, journal = {Nature Neuroscience}, volume = {5631628}, year = {2017}, month = {09/2017}, abstract = {

Here we report that monkeys raised without exposure to faces did not develop face domains, but did develop domains for other categories and did show normal retinotopic organization, indicating that early face deprivation leads to a highly selective cortical processing deficit. Therefore, experience must be necessary for the formation (or maintenance) of face domains. Gaze tracking revealed that control monkeys looked preferentially at faces, even at ages prior to the emergence of face domains, but face-deprived monkeys did not, indicating that face looking is not innate. A retinotopic organization is present throughout the visual system at birth, so selective early viewing behavior could bias category-specific visual responses toward particular retinotopic representations, thereby leading to domain formation in stereotyped locations in inferotemporal cortex, without requiring category-specific templates or biases. Thus, we propose that environmental importance influences viewing behavior, viewing behavior drives neuronal activity, and neuronal activity sculpts domain formation.

}, issn = {1097-6256}, doi = {10.1038/nn.4635}, url = {http://www.nature.com/doifinder/10.1038/nn.4635}, author = {Michael J Arcaro and Peter F Schade and Vincent, Justin L and Carlos R Ponce and Margaret S Livingstone} }