@conference {5548, title = {Using Multimodal DNNs to Study Vision-Language Integration in the Brain}, booktitle = {ICLR 2023}, year = {2023}, month = {03/2023}, abstract = {

We leverage a large stereoelectroencephalography (SEEG) dataset consisting of neural recordings during movie viewing and a battery of unimodal and multimodal deep neural network models (SBERT, BEIT, SIMCLR, CLIP, SLIP) to identify candidate sites of multimodal integration in the human brain. Our data-driven method involves three steps: first, we parse the neural data into discrete, distinct event-structures, i.e., image-text pairs defined either by word onset times or visual scene cuts. We then use the activity generated by these event-structures in our candidate models to predict the activity generated in the brain. Finally, using contrasts between models with or without multimodal learning signals, we isolate those neural arrays driven more by multimodal representations than by unimodal representations. Using this method, we identify a sizable set of candidate neural sites that our model predictions suggest are shaped by multimodality (from 3\%-29\%, depending on increasingly conservative statistical inclusion criteria). We note a meaningful cluster of these multimodal electrodes in and around the temporoparietal junction, long theorized to be a hub of multimodal integration.

}, url = {https://openreview.net/pdf?id=OQQ1p0pFP4}, author = {Vighnesh Subramaniam and Colin Conwell and Christopher Wang and Gabriel Kreiman and Boris Katz and Ignacio Cases and Andrei Barbu} } @article {4825, title = {Large-scale benchmarking of deep neural network models in mouse visual cortex reveals patterns similar to those observed in macaque visual cortex}, year = {2021}, abstract = {

What is the representational structure of mouse visual cortex and how is it shaped? Mice obviouslyinteract with the world and recognize objects but unlike in primates, a majority of research to date suggests theactivity of their visual cortex may not be so well described by deep neural networks trained for object recognition.Using the Allen Brain Observatory{\quotesinglbase}{\"A}{\^o}s 2-photon calcium-imaging dataset of activity in over 30,000 rodent visualcortical neurons recorded in response to natural scenes, we work to resolve this discrepancy and demonstrate thatmodern neural networks can indeed be used to explain activity in the mouse visual cortex to a more reasonabledegree than previously suggested. In so doing, we elucidate at large scale the properties of networks whichbest match the biological visual system, with both representational similarity analysis and encoding modelscoming to mostly the same conclusions. Our analysis of 30 object recognition architectures (both pretrainedand randomly initialized) from the PyTorch model zoo demonstrates that deeper, thinner residual networks withbypass connections, fewer parameters shared across many convolutions, and higher scores on the ImageNetimage-recognition challenge tend to be more predictive of the neural activations in our sample. Additionally, wefind a significant degree of overlap between the models that best predict macaque visual cortex (as cataloguedby brain-score.org) and those that best predict mouse visual cortex. In concert, these findings help to bolster themouse brain as a viable source of data for the methods that have been successful thus far in the study of monkeybrains, and provide a preliminary set of design targets for building models that can better take advantage of theunparalleled scale, quality, and resolution of data afforded by calcium-imaging in the mouse brain.

}, author = {Colin Conwell and David Mayo and Michael Buice and Boris Katz and George Alvarez and Andrei Barbu} } @article {5052, title = {Neural Regression, Representational Similarity, Model Zoology Neural Taskonomy at Scale in Rodent Visual Cortex}, year = {2021}, abstract = {

How well do deep neural networks fare as models of mouse visual cortex? A majority of research to date suggests results far more mixed than those produced in the modeling of primate visual cortex. Here, we perform a large-scale bench- marking of dozens of deep neural network models in mouse visual cortex with both representational similarity analysis and neural regression. Using the Allen Brain Observatory{\textquoteright}s 2-photon calcium-imaging dataset of activity in over 6,000 reliable rodent visual cortical neurons recorded in response to natural scenes, we replicate previous findings and resolve previous discrepancies, ultimately demonstrating that modern neural networks can in fact be used to explain activity in the mouse visual cortex to a more reasonable degree than previously suggested. Using our benchmark as an atlas, we offer preliminary answers to overarching questions about levels of analysis, questions about the properties of models that best predict the visual system overall and questions about the mapping between biological and artificial representations. Our results provide a reference point for future ventures in the deep neural network modeling of mouse visual cortex, hinting at novel combinations of mapping method, architecture, and task to more fully characterize the computational motifs of visual representation in a species so central to neuroscience, but with a perceptual physiology and ecology markedly different from the ones we study in primates.

}, author = {Colin Conwell and David Mayo and Michael A. Buice and Boris Katz and George A. Alvarez and Andrei Barbu} } @conference {5107, title = {On the use of Cortical Magnification and Saccades as Biological Proxies for Data Augmentation}, booktitle = {Shared Visual Representations in Human and Machine Intelligence (SVRHM) Workshop at NeurIPS}, year = {2021}, keywords = {Active Perception, Data-Augmentation, Foveation, Self-Supervised Learning}, url = {https://openreview.net/forum?id=Rpazl253IHb}, author = {Binxu Wang and David Mayo and Arturo Deza and Andrei Barbu and Colin Conwell} }