@article {4458, title = {Minimal videos: Trade-off between spatial and temporal information in human and machine vision.}, journal = {Cognition}, year = {2020}, month = {08/2020}, abstract = {

Objects and their parts can be visually recognized from purely spatial or purely temporal information but the mechanisms integrating space and time are poorly understood. Here we show that visual recognition of objects and actions can be achieved by efficiently combining spatial and motion cues in configurations where each source on its own is insufficient for recognition. This analysis is obtained by identifying minimal videos: these are short and tiny video clips in which objects, parts, and actions can be reliably recognized, but any reduction in either space or time makes them unrecognizable. Human recognition in minimal videos is invariably accompanied by full interpretation of the internal components of the video. State-of-the-art deep convolutional networks for dynamic recognition cannot replicate human behavior in these configurations. The gap between human and machine vision demonstrated here is due to critical mechanisms for full spatiotemporal interpretation that are lacking in current computational models.

}, keywords = {Comparing deep neural networks and humans, Integration of spatial and temporal visual information, minimal images, Minimal videos, Visual dynamic recognition}, doi = {10.1016/j.cognition.2020.104263}, url = {https://www.sciencedirect.com/science/article/abs/pii/S0010027720300822}, author = {Guy Ben-Yosef and Gabriel Kreiman and Shimon Ullman} } @conference {4558, title = {What can human minimal videos tell us about dynamic recognition models?}, booktitle = {International Conference on Learning Representations (ICLR 2020)}, year = {2020}, month = {04/2020}, address = {Virtual Conference}, abstract = {

In human vision objects and their parts can be visually recognized from purely spatial or purely temporal information but the mechanisms integrating space and time are poorly understood. Here we show that human visual recognition of objects and actions can be achieved by efficiently combining spatial and motion cues in configurations where each source on its own is insufficient for recognition. This analysis is obtained by identifying minimal videos: these are short and tiny video clips in which objects, parts, and actions can be reliably recognized, but any reduction in either space or time makes them unrecognizable. State-of-the-art deep networks for dynamic visual recognition cannot replicate human behavior in these configurations. This gap between humans and machines points to critical mechanisms in human dynamic vision that are lacking in current models.

Published as a workshop paper at {\textquotedblleft}Bridging AI and Cognitive Science{\textquotedblright} (ICLR 2020)

}, url = {https://baicsworkshop.github.io/pdf/BAICS_1.pdf}, author = {Guy Ben-Yosef and Gabriel Kreiman and Shimon Ullman} } @conference {4105, title = {Minimal images in deep neural networks: Fragile Object Recognition in Natural Images}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2019}, address = {New Orleans, La}, abstract = {

The human ability to recognize objects is impaired when the object is not shown in full. "Minimal images" are the smallest regions of an image that remain recognizable for humans. Ullman et al. 2016 show that a slight modification of the location and size of the visible region of the minimal image produces a sharp drop in human recognition accuracy. In this paper, we demonstrate that such drops in accuracy due to changes of the visible region are a common phenomenon between humans and existing state-of-the-art deep neural networks (DNNs), and are much more prominent in DNNs. We found many cases where DNNs classified one region correctly and the other incorrectly, though they only differed by one row or column of pixels, and were often bigger than the average human minimal image size. We show that this phenomenon is independent from previous works that have reported lack of invariance to minor modifications in object location in DNNs. Our results thus reveal a new failure mode of DNNs that also affects humans to a much lesser degree. They expose how fragile DNN recognition ability is for natural images even without adversarial patterns being introduced. Bringing the robustness of DNNs in natural images to the human level remains an open challenge for the community.

}, url = {https://arxiv.org/pdf/1902.03227.pdf}, author = {S. Srivastava and Guy Ben-Yosef and X. Boix} } @article {2548, title = {Full interpretation of minimal images.}, journal = {Cognition}, volume = {171}, year = {2018}, month = {02/2018}, pages = {65-84}, chapter = {65}, abstract = {

The goal in this work is to model the process of\  {\textquoteleft}full interpretation{\textquoteright}\  of\  object images,\  which is the ability to identify and localize all semantic features and parts that are recognized by human observers.\  The task is approached\  by dividing the interpretation of\  the complete object to the interpretation of multiple reduced but interpretable local\  regions. In such reduced regions, interpretation is\  simpler,\  since the number of\  se mantic\  components is small, and the variability of possible configurations is low.\ 

We model the interpretation process by identifying primitive components and\  relations that play a useful role in\  local\  interpretation by humans. To identify useful\  compo nents and relations used in the interpretation process, we consider the\  interpretation of\  {\textquoteleft} minimal configurations{\textquoteright} :\  these\  are\  reduced\  local regions , whic h are\  minimal in the sense that further reduction\  renders them unrecognizable and\  uninterpretable.\  We show that\  such\  minimal\  interpretable image s have useful properties,\  which\  we use to identify\  informative\  features and relations used for full interpretation.\  We describe our interpretation model, and show results of\  detailed\  interpretations\  of\  minimal c onfigurations, produced automatically by the model. Finally, we\  discuss\  implications of\  full\  interpretation\  to\  difficult visual tasks, such as recognizing human\  activities or interactions , which are beyond the scope of current models of visual\  recognition .

}, keywords = {Image interpretation, M inimal images, Parts and relations, Top-down processing}, doi = {https://doi.org/10.1016/j.cognition.2017.10.006}, author = {Guy Ben-Yosef and Liav Assif and Shimon Ullman} } @article {4107, title = {Full interpretation of minimal images}, journal = {Cognition}, volume = {171}, year = {2018}, month = {01/2018}, pages = {65 - 84}, abstract = {

The goal in this work is to model the process of {\textquoteleft}full interpretation{\textquoteright} of object images, which is the ability to identify and localize all semantic features and parts that are recognized by human observers. The task is approached by dividing the interpretation of the complete object to the interpretation of multiple reduced but interpretable local regions. In such reduced regions, interpretation is simpler, since the number of semantic components is small, and the variability of possible configurations is low.

We model the interpretation process by identifying primitive components and relations that play a useful role in local interpretation by humans. To identify useful components and relations used in the interpretation process, we consider the interpretation of {\textquoteleft}minimal configurations{\textquoteright}: these are reduced local regions, which are minimal in the sense that further reduction renders them unrecognizable and uninterpretable. We show that such minimal interpretable images have useful properties, which we use to identify informative features and relations used for full interpretation. We describe our interpretation model, and show results of detailed interpretations of minimal configurations, produced automatically by the model. Finally, we discuss possible extensions and implications of full interpretation to difficult visual tasks, such as recognizing social interactions, which are beyond the scope of current models of visual recognition.

}, keywords = {Image interpretation Minimal images Parts and relations Top-down processing}, issn = {00100277}, doi = {10.1016/j.cognition.2017.10.006}, url = {https://linkinghub.elsevier.com/retrieve/pii/S001002771730269Xhttps://api.elsevier.com/content/article/PII:S001002771730269X?httpAccept=text/xmlhttps://api.elsevier.com/content/article/PII:S001002771730269X?httpAccept=text/plain}, author = {Guy Ben-Yosef and Liav Assif and Shimon Ullman} } @article {4106, title = {Image interpretation above and below the object level}, journal = {Interface Focus}, volume = {8}, year = {2018}, month = {06/2018}, pages = {20180020}, abstract = {

Computational models of vision have advanced in recent years at a rapid rate, rivalling in some areas human-level performance. Much of the progress to date has focused on analysing the visual scene at the object level{\textemdash}the recognition and localization of objects in the scene. Human understanding of images reaches a richer and deeper image understanding both {\textquoteleft}below{\textquoteright} the object level, such as identifying and localizing object parts and sub-parts, as well as {\textquoteleft}above{\textquoteright} the object level, such as identifying object relations, and agents with their actions and interactions. In both cases, understanding depends on recovering meaningful structures in the image, and their components, properties and inter-relations, a process referred here as {\textquoteleft}image interpretation{\textquoteright}. In this paper, we describe recent directions, based on human and computer vision studies, towards human-like image interpretation, beyond the reach of current schemes, both below the object level, as well as some aspects of image interpretation at the level of meaningful configurations beyond the recognition of individual objects, and in particular, interactions between two people in close contact. In both cases the recognition process depends on the detailed interpretation of so-called {\textquoteleft}minimal images{\textquoteright}, and at both levels recognition depends on combining {\textquoteleft}bottom-up{\textquoteright} processing, proceeding from low to higher levels of a processing hierarchy, together with {\textquoteleft}top-down{\textquoteright} processing, proceeding from high to lower levels stages of visual analysis.

}, issn = {2042-8898}, doi = {10.1098/rsfs.2018.0020}, url = {https://royalsocietypublishing.org/doi/full/10.1098/rsfs.2018.0020$\#$d3e1503}, author = {Guy Ben-Yosef and Shimon Ullman} } @article {3627, title = {Image interpretation above and below the object level}, year = {2018}, month = {05/2018}, abstract = {

Computational models of vision have advanced in recent years at a rapid rate, rivaling in some areas human-level performance. Much of the progress to date has focused on analyzing the visual scene at the object level {\textendash} the recognition and localization of objects in the scene. Human understanding of images reaches a richer and deeper image understanding both {\textquoteleft}below{\textquoteright} the object level, such as identifying and localizing object parts and sub-parts, as well as {\textquoteleft}above{\textquoteright} the object levels, such as identifying object relations, and agents with their actions and interactions. In both cases, understanding depends on recovering meaningful structures in the image, their components, properties, and inter-relations, a process referred here as {\textquoteleft}image interpretation{\textquoteright}.

In this paper we describe recent directions, based on human and computer vision studies, towards human-like image interpretation, beyond the reach of current schemes, both below the object level, as well as some aspects of image interpretation at the level of meaningful configurations beyond the recognition of individual objects, in particular, interactions between two people in close contact. In both cases the recognition process depends on the detailed interpretation of so-called {\textquoteright}minimal images{\textquoteright}, and at both levels recognition depends on combining {\textquoteleft}bottom-up{\textquoteright} processing, proceeding from low to higher levels of a processing hierarchy, together with {\textquoteleft}top-down{\textquoteright} processing, proceeding from high to lower levels stages of visual analysis.

}, keywords = {Interaction Recognition, minimal images, Social Interactions, Visual interpretation, visual recognition}, author = {Guy Ben-Yosef and Shimon Ullman} } @article {3620, title = {Image interpretation above and below the object level}, journal = {Proceedings of the Royal Society: Interface Focus}, year = {2018}, month = {06/2018}, abstract = {

Computational models of vision have advanced in recent years at a rapid rate, rivaling in some areas human-level performance. Much of the progress to date has focused on analyzing the visual scene at the object level {\textendash} the recognition and localization of objects in the scene. Human understanding of images reaches a richer and deeper image understanding both {\textquoteleft}below{\textquoteright} the object level, such as identifying and localizing object parts and sub-parts, as well as {\textquoteleft}above{\textquoteright} the object levels, such as identifying object relations, and agents with their actions and interactions. In both cases, understanding depends on recovering meaningful structures in the image, their components, properties, and inter-relations, a process referred here as {\textquoteleft}image interpretation{\textquoteright}.

In this paper we describe recent directions, based on human and computer vision studies, towards human-like image interpretation, beyond the reach of current schemes, both below the object level, as well as some aspects of image interpretation at the level of meaningful configurations beyond the recognition of individual objects, in particular, interactions between two people in close contact. In both cases the recognition process depends on the detailed interpretation of so-called {\textquoteright}minimal images{\textquoteright}, and at both levels recognition depends on combining {\textquoteleft}bottom-up{\textquoteright} processing, proceeding from low to higher levels of a processing hierarchy, together with {\textquoteleft}top-down{\textquoteright} processing, proceeding from high to lower levels stages of visual analysis.

}, author = {Guy Ben-Yosef and Shimon Ullman} } @article {3963, title = {Partially Occluded Hands: A challenging new dataset for single-image hand pose estimation}, year = {2018}, month = {12/2018}, abstract = {

Recognizing the pose of hands matters most when hands are interacting with other objects. To understand how well both machines and humans perform on single-image 2D hand-pose reconstruction from RGB images, we collected a challenging dataset of hands interacting with 148 objects. We used a novel methodology that provides the same hand in the same pose both with the object being present and occluding the hand and without the object occluding the hand. Additionally, we collected a wide range of grasps for each object designing the data collection methodology to ensure this diversity. Using this dataset we measured the performance of two state-of-the-art hand-pose recognition methods showing that both are extremely brittle when faced with even light occlusion from an object. This is not evident in previous datasets because they often avoid hand- object occlusions and because they are collected from videos where hands are often between objects and mostly unoccluded. We annotated a subset of the dataset and used that to show that humans are robust with respect to occlusion, and also to characterize human hand perception, the space of grasps that seem to be considered, and the accuracy of reconstructing occluded portions of hands. We expect that such data will be of interest to both the vision community for developing more robust hand-pose algorithms and to the robotic grasp planning community for learning such grasps. The dataset is available at occludedhands.com

}, keywords = {dataset, Partial occlusion, RGB hand-pose reconstruction}, author = {Battushig Myanganbayar and Cristina Mata and Gil Dekel and Katz, Boris and Guy Ben-Yosef and Andrei Barbu} } @conference {3964, title = {Partially Occluded Hands: A challenging new dataset for single-image hand pose estimation}, booktitle = {The 14th Asian Conference on Computer Vision (ACCV 2018)}, year = {2018}, month = {12/2018}, abstract = {

Recognizing the pose of hands matters most when hands are interacting with other objects. To understand how well both machines and humans perform on single-image 2D hand-pose reconstruction from RGB images, we collected a challenging dataset of hands interacting with 148 objects. We used a novel methodology that provides the same hand in the same pose both with the object being present and occluding the hand and without the object occluding the hand. Additionally, we collected a wide range of grasps for each object designing the data collection methodology to ensure this diversity. Using this dataset we measured the performance of two state-of-the-art hand-pose recognition methods showing that both are extremely brittle when faced with even light occlusion from an object. This is not evident in previous datasets because they often avoid hand- object occlusions and because they are collected from videos where hands are often between objects and mostly unoccluded. We annotated a subset of the dataset and used that to show that humans are robust with respect to occlusion, and also to characterize human hand perception, the space of grasps that seem to be considered, and the accuracy of reconstructing occluded portions of hands. We expect that such data will be of interest to both the vision community for developing more robust hand-pose algorithms and to the robotic grasp planning community for learning such grasps. The dataset is available at occludedhands.com

}, keywords = {dataset, Partial occlusion, RGB hand-pose reconstruction}, url = {http://accv2018.net/}, author = {Battushig Myanganbayar and Cristina Mata and Gil Dekel and Boris Katz and Guy Ben-Yosef and Andrei Barbu} } @article {3904, title = {Spatiotemporal interpretation features in the recognition of dynamic images}, year = {2018}, month = {11/2018}, abstract = {

Objects and their parts can be visually recognized and localized from purely spatial information in static images and also from purely temporal information as in the perception of biological motion. Cortical regions have been identified, which appear to specialize in visual recognition based on either static or dynamic cues, but the mechanisms by which spatial and temporal information is integrated is only poorly understood. Here we show that visual recognition of objects and actions can be achieved by efficiently combining spatial and motion cues in configurations where each source on its own is insufficient for recognition. This analysis is obtained by the identification of minimal spatiotemporal configurations: these are short videos in which objects and their parts, along with an action being performed, can be reliably recognized, but any reduction in either space or time makes them unrecognizable. State-of-the-art computational models for recognition from dynamic images based on deep 2D and 3D convolutional networks cannot replicate human recognition in these configurations. Action recognition in minimal spatiotemporal configurations is invariably accompanied by full human interpretation of the internal components of the image and their inter-relations. We hypothesize that this gap is due to mechanisms for full spatiotemporal interpretation process, which in human vision is an integral part of recognizing dynamic event, but is not sufficiently represented in current DNNs.

}, author = {Guy Ben-Yosef and Gabriel Kreiman and Shimon Ullman} } @article {2408, title = {Full interpretation of minimal images}, year = {2017}, month = {02/2017}, abstract = {

The goal in this work is to model the process of {\textquoteleft}full interpretation{\textquoteright} of object images, which is the ability to identify and localize all semantic features and parts that are recognized by human observers. The task is approached by dividing the interpretation of the complete object to the interpretation of multiple reduced but interpretable local regions. In such reduced regions, interpretation is simpler, since the number of semantic components is small, and the variability of possible configurations is low.

We model the interpretation process by identifying primitive components and relations that play a useful role in local interpretation by humans. To identify useful components and relations used in the interpretation process, we consider the interpretation of {\textquoteleft}minimal configurations{\textquoteright}: these are reduced local regions, which are minimal in the sense that further reduction renders them unrecognizable and uninterpretable. We show that such minimal interpretable images have useful properties, which we use to identify informative features and relations used for full interpretation. We describe our interpretation model, and show results of detailed interpretations of minimal configurations, produced automatically by the model. Finally, we discuss implications of full interpretation to difficult visual tasks, such as recognizing human activities or interactions, which are beyond the scope of current models of visual recognition.

This manuscript has beed accepted for publication in Cognition.

}, keywords = {Image interpretation, Parts and relations, Visual object recognition}, author = {Guy Ben-Yosef and Liav Assif and Shimon Ullman} } @conference {2724, title = {A model for interpreting social interactions in local image regions}, booktitle = {AAAI Spring Symposium Series, Science of Intelligence}, year = {2017}, month = {03/2017}, address = {Palo Alto, CA}, abstract = {
Understanding social interactions (such as {\textquoteleft}hug{\textquoteright} or {\textquoteleft}fight{\textquoteright}) is a basic and important capacity of the human visual system, but a challenging and still open problem for modeling. In this work we study visual recognition of social interactions, based on small but recognizable local regions. The approach is based on two novel key components: (i) A given social interaction can be recognized reliably from reduced images (called {\textquoteleft}minimal images{\textquoteright}). (ii) The recognition of a social interaction depends on identifying components and relations within the minimal image (termed {\textquoteleft}interpretation{\textquoteright}). We show psychophysics data for minimal images and modeling results for their interpretation. We discuss the integration of minimal configurations in recognizing social interactions in a detailed, high-resolution image.
}, url = {http://www.aaai.org/ocs/index.php/SSS/SSS17/paper/view/15354}, author = {Guy Ben-Yosef and Alon Yachin and Shimon Ullman} } @article {2723, title = {Recognizing and Interpreting Social Interactions in Local Image Regions}, year = {2016}, note = {

(Accepted for oral presentation)

}, month = {11/2016}, abstract = {

Understanding social interactions (such as {\textquoteright}hug{\textquoteright}\ or {\textquoteright}fight{\textquoteright}) is a basic and important capacity of the human visual system, but a challenging and still open problem for modeling. Here we study visual recognition of social interactions, based on small but recognizable local regions. The approach is based on two novel key components: (i) A given social interaction can be recognized reliably from reduced images (called {\textquoteright}minimal images{\textquoteright}). (ii) The recognition of a social interaction depends on identifying components and relations within the minimal image (termed {\textquoteright}interpretation{\textquoteright}). We show psychophysics data for minimal images and modeling results for their interpretation.\ 

}, author = {Guy Ben-Yosef and Alon Yachin and Shimon Ullman} } @proceedings {1762, title = {A model for full local image interpretation}, year = {2015}, abstract = {

We describe a computational model of humans{\textquoteright} ability to provide a detailed interpretation of a scene{\textquoteright}s components. Humans can identify in an image meaningful components almost everywhere, and identifying these components is an essential part of the visual process, and of understanding the surrounding scene and its potential meaning to the viewer. Detailed interpretation is beyond the scope of current models of visual recognition. Our model suggests that this is a fundamental limitation, related to the fact that existing models rely on feed - forward but limited top - down processing. In our model, a first recognition stage leads to the initial activation of class candidates, which is incomplete and with limited accuracy. This stage then triggers the application of class - specific interpretation and validation processes, which recover richer and more accurate interpretation of the visible scene. We discuss implications of the model for visual interpretation by humans and by computer vision models.

}, author = {Guy Ben-Yosef and Liav Assif and Daniel Harari and Shimon Ullman} }