@article {4458, title = {Minimal videos: Trade-off between spatial and temporal information in human and machine vision.}, journal = {Cognition}, year = {2020}, month = {08/2020}, abstract = {

Objects and their parts can be visually recognized from purely spatial or purely temporal information but the mechanisms integrating space and time are poorly understood. Here we show that visual recognition of objects and actions can be achieved by efficiently combining spatial and motion cues in configurations where each source on its own is insufficient for recognition. This analysis is obtained by identifying minimal videos: these are short and tiny video clips in which objects, parts, and actions can be reliably recognized, but any reduction in either space or time makes them unrecognizable. Human recognition in minimal videos is invariably accompanied by full interpretation of the internal components of the video. State-of-the-art deep convolutional networks for dynamic recognition cannot replicate human behavior in these configurations. The gap between human and machine vision demonstrated here is due to critical mechanisms for full spatiotemporal interpretation that are lacking in current computational models.

}, keywords = {Comparing deep neural networks and humans, Integration of spatial and temporal visual information, minimal images, Minimal videos, Visual dynamic recognition}, doi = {10.1016/j.cognition.2020.104263}, url = {https://www.sciencedirect.com/science/article/abs/pii/S0010027720300822}, author = {Guy Ben-Yosef and Gabriel Kreiman and Shimon Ullman} } @article {3627, title = {Image interpretation above and below the object level}, year = {2018}, month = {05/2018}, abstract = {

Computational models of vision have advanced in recent years at a rapid rate, rivaling in some areas human-level performance. Much of the progress to date has focused on analyzing the visual scene at the object level {\textendash} the recognition and localization of objects in the scene. Human understanding of images reaches a richer and deeper image understanding both {\textquoteleft}below{\textquoteright} the object level, such as identifying and localizing object parts and sub-parts, as well as {\textquoteleft}above{\textquoteright} the object levels, such as identifying object relations, and agents with their actions and interactions. In both cases, understanding depends on recovering meaningful structures in the image, their components, properties, and inter-relations, a process referred here as {\textquoteleft}image interpretation{\textquoteright}.

In this paper we describe recent directions, based on human and computer vision studies, towards human-like image interpretation, beyond the reach of current schemes, both below the object level, as well as some aspects of image interpretation at the level of meaningful configurations beyond the recognition of individual objects, in particular, interactions between two people in close contact. In both cases the recognition process depends on the detailed interpretation of so-called {\textquoteright}minimal images{\textquoteright}, and at both levels recognition depends on combining {\textquoteleft}bottom-up{\textquoteright} processing, proceeding from low to higher levels of a processing hierarchy, together with {\textquoteleft}top-down{\textquoteright} processing, proceeding from high to lower levels stages of visual analysis.

}, keywords = {Interaction Recognition, minimal images, Social Interactions, Visual interpretation, visual recognition}, author = {Guy Ben-Yosef and Shimon Ullman} } @article {1760, title = {Atoms of recognition in human and computer vision}, journal = {PNAS }, volume = {113}, year = {2016}, month = {03/2016}, pages = {2744{\textendash}2749}, abstract = {
Discovering the visual features and representations used by thebrain to recognize objects is a central problem in the study of vision. Recently, neural network models of visual object recognition, including biological and deep network models, have shown remarkableprogress and have begun to rival human performance in some challenging tasks. These models are trained on image examples andlearn to extract features and representations and to use them for categorization. It remains unclear, however, whether the representations and learning processes discovered by current models aresimilar to those used by the human visual system. Here we show,by introducing and using minimal recognizable images, that thehuman visual system uses features and processes that are not usedby current models and that are critical for recognition. We found bypsychophysical studies that at the level of minimal recognizableimages a minute change in the image can have a drastic effect onrecognition, thus identifying features that are critical for the task.Simulations then showed that current models cannot explain thissensitivity to precise feature configurations and, more generally,do not learn to recognize minimal images at a human level. The roleof the features shown here is revealed uniquely at the minimal level, where the contribution of each feature is essential. A full understanding of the learning and use of such features will extend ourunderstanding of visual recognition and its cortical mechanisms andwill enhance the capacity of computational models to learn fromvisual experience and to deal with recognition and detailedimage interpretation.
}, keywords = {Computer vision, minimal images, object recognition, visual perception, visual representations}, issn = {1091-6490}, doi = {10.1073/pnas.1513198113}, url = {http://www.pnas.org/content/113/10/2744.abstract}, author = {Shimon Ullman and Liav Assif and Eitan Fetaya and Daniel Harari} }