@article {5312, title = {An empirical assay of view-invariant object learning in humans and comparison with baseline image-computable models}, journal = {bioRxiv}, year = {2023}, month = {01/2023}, abstract = {

How humans learn new visual objects is a longstanding scientific problem. Previous work has led to a diverse collection of models for how it is accomplished, but a current limitation in the field is a lack of empirical benchmarks which can be used to evaluate and compare specific models against each other. Here, we use online psychophysics to measure human behavioral learning trajectories over a set of tasks involving novel 3D objects. Consistent with intuition, these results show that humans generally require very few images (≈ 6) to approach their asymptotic accuracy, find some object discriminations more easy to learn than others, and generalize quite well over a range of image transformations after even one view of each object. We then use those data to develop benchmarks that may be used to evaluate a learning model{\textquoteright}s similarity to humans. We make these data and benchmarks publicly available [GitHub], and, to our knowledge, they are currently the largest publicly-available collection of learning-related psychophysics data in humans. Additionally, to serve as baselines for those benchmarks, we implement and test a large number of baseline models (n=1,932), each based on a standard cognitive theory of learning: that humans re-represent images in a fixed, Euclidean space, then learn linear decision boundaries in that space to identify objects in future images. We find some of these baseline models make surprisingly accurate predictions. However, we also find reliable prediction gaps between all baseline models and humans, particularly in the few-shot learning setting.

}, url = {https://www.biorxiv.org/content/10.1101/2022.12.31.522402v1}, author = {Michael J. Lee and James J. DiCarlo} } @article {5527, title = {Robustified ANNs Reveal Wormholes Between Human Category Percepts}, journal = {arXiv}, year = {2023}, month = {10/2023}, abstract = {

The visual object category reports of artificial neural networks (ANNs) are notoriously sensitive to tiny, adversarial image perturbations. Because human category reports (aka human percepts) are thought to be insensitive to those same small-norm perturbations {\textendash} and locally stable in general {\textendash} this argues that ANNs are incomplete scientific models of human visual perception. Consistent with this, we show that when small-norm image perturbations are generated by standard ANN models, human object category percepts are indeed highly stable. However, in this very same {\textquotedblleft}human-presumed-stable{\textquotedblright} regime, we find that robustified ANNs reliably dis- cover low-norm image perturbations that strongly disrupt human percepts. These previously undetectable human perceptual disruptions are massive in amplitude, approaching the same level of sensitivity seen in robustified ANNs. Further, we show that robustified ANNs support precise perceptual state interventions: they guide the construction of low-norm image perturbations that strongly alter human category percepts toward specific prescribed percepts. These observations sug- gest that for arbitrary starting points in image space, there exists a set of nearby {\textquotedblleft}wormholes{\textquotedblright}, each leading the subject from their current category perceptual state into a semantically very different state. Moreover, contemporary ANN models of biological visual processing are now accurate enough to consistently guide us to those portals.

}, url = {https://arxiv.org/abs/2308.06887}, author = {Guy Gaziv and Michael J. Lee and James J. DiCarlo} } @conference {5528, title = {Strong and Precise Modulation of Human Percepts via Robustified ANNs}, booktitle = {NeurIPS 2023}, year = {2023}, month = {12/2023}, abstract = {

The visual object category reports of artificial neural networks (ANNs) are notoriously sensitive to tiny, adversarial image perturbations. Because human category reports (aka human percepts) are thought to be insensitive to those same small-norm perturbations {\textendash} and locally stable in general {\textendash} this argues that ANNs are incomplete scientific models of human visual perception. Consistent with this, we show that when small-norm image perturbations are generated by standard ANN models, human object category percepts are indeed highly stable. However, in this very same {\textquotedblleft}human-presumed-stable{\textquotedblright} regime, we find that robustified ANNs reliably dis- cover low-norm image perturbations that strongly disrupt human percepts. These previously undetectable human perceptual disruptions are massive in amplitude, approaching the same level of sensitivity seen in robustified ANNs. Further, we show that robustified ANNs support precise perceptual state interventions: they guide the construction of low-norm image perturbations that strongly alter human category percepts toward specific prescribed percepts. In sum, these contemporary models of biological visual processing are now accurate enough to guide strong and precise interventions on human perception.

}, url = {https://proceedings.neurips.cc/paper_files/paper/2023/hash/d00904cebc0d5b69fada8ad33d0f1422-Abstract-Conference.html}, author = {Guy Gaziv and Michael J. Lee and James J. DiCarlo} } @article {5283, title = {Adversarially trained neural representations may already be as robust as corresponding biological neural representations}, journal = {arXiv}, year = {2022}, month = {06/2022}, abstract = {

Visual systems of primates are the gold standard of robust perception. There is thus a general belief that mimicking the neural representations that underlie those systems will yield artificial visual systems that are adversarially robust. In this work, we develop a method for performing adversarial visual attacks directly on primate brain activity. We then leverage this method to demonstrate that the above-mentioned belief might not be well founded. Specifically, we report that the biological neurons that make up visual systems of primates exhibit susceptibility to adversarial perturbations that is comparable in magnitude to existing (robustly trained) artificial neural networks.

}, author = {Chong Guo and Michael J. Lee and Guillaume Leclerc and Joel Dapello and Yug Rao and Aleksander Madry and James J. DiCarlo} }