@conference {4449, title = {Hard choices: Children{\textquoteright}s understanding of the cost of action selection. }, booktitle = {Cognitive Science Society}, year = {2019}, author = {Shari Liu and Fiery A Cushman and Samuel J Gershman and Kool, Wouter and Elizabeth S Spelke} } @article {4199, title = {Incentives Boost Model-Based Control Across a Range of Severity on Several Psychiatric Constructs}, journal = {Biological Psychiatry}, volume = {85}, year = {2019}, month = {03/2019}, pages = {425 - 433}, abstract = {

Background

Human decision making exhibits a mixture of model-based and model-free control. Recent evidence indicates that arbitration between these two modes of control ({\textquotedblleft}metacontrol{\textquotedblright}) is based on their relative costs and benefits. While model-based control may increase accuracy, it requires greater computational resources, so people invoke model-based control only when potential rewards exceed those of model-free control. We used a sequential decision task, while concurrently manipulating performance incentives, to ask if symptoms and traits of psychopathology decrease or increase model-based control in response to incentives.

Methods

We recruited a nonpatient population of 839 online participants using Amazon Mechanical Turk who completed transdiagnostic self-report measures encompassing symptoms, traits, and factors. We fit a dual-controller reinforcement learning model and obtained a computational measure of model-based control separately for small incentives and large incentives.

Results

None of the constructs were related to a failure of large incentives to boost model-based control. In fact, for the sensation seeking trait and anxious-depression factor, higher scores were associated with a larger incentive effect, whereby greater levels of these constructs were associated with larger increases in model-based control. Many constructs showed decreases in model-based control as a function of severity, but a social withdrawal factor was positively correlated; alcohol use and social anxiety were unrelated to model-based control.

Conclusions

Our results demonstrate that model-based control can reliably be improved independent of construct severity for most measures. This suggests that incentives may be a useful intervention for boosting model-based control across a range of symptom and trait severity.

}, keywords = {Computational psychiatry, Habits and goals Incentives, Model-based control, Psychiatric constructs, reinforcement learning}, issn = {00063223}, doi = {10.1016/j.biopsych.2018.06.018}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0006322318316329}, author = {Patzelt, Edward H. and Kool, Wouter and Millner, Alexander J. and Samuel J Gershman} } @article {4201, title = {The transdiagnostic structure of mental effort avoidance}, journal = {Scientific Reports}, volume = {9}, year = {2019}, month = {02/2019}, abstract = {

The law of least mental effort states that, everything else being equal, the brain tries to minimize mental effort expenditure during task performance by avoiding decisions that require greater cognitive demands. Prior studies have shown associations between disruptions in effort expenditure and specific psychiatric illnesses (e.g., schizophrenia and depression) or clinically-related symptoms and traits (e.g., anhedonia and apathy), yet no research has explored this issue transdiagnostically. Specifically, this research has largely focused on a single diagnostic category, symptom, or trait. However, abnormalities in effort expression could be related to several different psychiatrically-relevant constructs that cut across diagnostic boundaries. Therefore, we examined the relationship between avoidance of mental effort and a diverse set of clinically-related symptoms and traits, and transdiagnostic latent factors in a large sample (n = 811). Only lack of perseverance, a dimension of impulsiveness, was associated with increased avoidance of mental effort. In contrast, several constructs were associated with less mental effort avoidance, including positive urgency, distress intolerance, obsessive-compulsive symptoms, disordered eating, and a factor consisting of compulsive behavior and intrusive thoughts. These findings demonstrate that deviations from normative effort expenditure are associated with a number of constructs that are common to several forms of psychiatric illness.

}, doi = {10.1038/s41598-018-37802-1}, url = {http://www.nature.com/articles/s41598-018-37802-1}, author = {Patzelt, Edward H. and Kool, Wouter and Millner, Alexander J. and Samuel J Gershman} } @article {4200, title = {Mental labour}, journal = {Nature Human Behaviour}, volume = {2}, year = {2018}, month = {12/2018}, pages = {899 - 908}, abstract = {

Mental effort is an elementary notion in our folk psychology and a familiar fixture in everyday introspective experience. However, as an object of scientific study, mental effort has remained rather elusive. Cognitive psychology has provided some tools for understanding how effort impacts performance, by linking effort with cognitive control function. What has remained less clear are the principles that govern the allocation of mental effort. Under what circumstances do people choose to invest mental effort, and when do they decline to do so? And what regulates the intensity of mental effort when it is applied? In new and promising work, these questions are being approached with the tools of behavioural economics. Though still in its infancy, this economic approach to mental effort research has already uncovered important aspects of effort-based decision-making, and points clearly to future lines of inquiry, including some intriguing opportunities presented by recent artificial intelligence research.

}, doi = {10.1038/s41562-018-0401-9}, url = {http://www.nature.com/articles/s41562-018-0401-9}, author = {Kool, Wouter and Botvinick, Matthew} } @article {4190, title = {Planning Complexity Registers as a Cost in Metacontrol}, journal = {Journal of Cognitive Neuroscience}, volume = {30}, year = {2018}, month = {10/2018}, pages = {1391 - 1404}, abstract = {

Decision-making algorithms face a basic tradeoff between accuracy and effort (i.e., computational demands). It is widely agreed that humans can choose between multiple decision-making processes that embody different solutions to this tradeoff: Some are computationally cheap but inaccurate, whereas others are computationally expensive but accurate. Recent progress in understanding this tradeoff has been catalyzed by formalizing it in terms of model-free (i.e., habitual) versus model-based (i.e., planning) approaches to reinforcement learning. Intuitively, if two tasks offer the same rewards for accuracy but one of them is much more demanding, we might expect people to rely on habit more in the difficult task: Devoting significant computation to achieve slight marginal accuracy gains would not be "worth it." We test and verify this prediction in a sequential reinforcement learning task. Because our paradigm is amenable to formal analysis, it contributes to the development of a computational model of how people balance the costs and benefits of different decision-making processes in a task-specific manner; in other words, how we decide when hard thinking is worth it.

}, issn = {0898-929X}, doi = {10.1162/jocn_a_01263}, url = {https://www.mitpressjournals.org/doi/abs/10.1162/jocn_a_01263}, author = {Kool, Wouter and Samuel J Gershman and Fiery A Cushman} } @article {3442, title = {Cost-Benefit Arbitration Between Multiple Reinforcement-Learning Systems.}, journal = {Psychol Sci}, volume = {28}, year = {2017}, month = {2017 Sep}, pages = {1321-1333}, abstract = {

Human behavior is sometimes determined by habit and other times by goal-directed planning. Modern reinforcement-learning theories formalize this distinction as a competition between a computationally cheap but inaccurate model-free system that gives rise to habits and a computationally expensive but accurate model-based system that implements planning. It is unclear, however, how people choose to allocate control between these systems. Here, we propose that arbitration occurs by comparing each system{\textquoteright}s task-specific costs and benefits. To investigate this proposal, we conducted two experiments showing that people increase model-based control when it achieves greater accuracy than model-free control, and especially when the rewards of accurate performance are amplified. In contrast, they are insensitive to reward amplification when model-based and model-free control yield equivalent accuracy. This suggests that humans adaptively balance habitual and planned action through on-line cost-benefit analysis.

}, issn = {1467-9280}, doi = {10.1177/0956797617708288}, author = {Kool, Wouter and Samuel J Gershman and Fiery A Cushman} } @article {2645, title = {When Does Model-Based Control Pay Off?}, journal = {PLoS Comput Biol}, volume = {12}, year = {2016}, month = {2016 Aug}, pages = {e1005090}, abstract = {

Many accounts of decision making and reinforcement learning posit the existence of two distinct systems that control choice: a fast, automatic system and a slow, deliberative system. Recent research formalizes this distinction by mapping these systems to "model-free" and "model-based" strategies in reinforcement learning. Model-free strategies are computationally cheap, but sometimes inaccurate, because action values can be accessed by inspecting a look-up table constructed through trial-and-error. In contrast, model-based strategies compute action values through planning in a causal model of the environment, which is more accurate but also more cognitively demanding. It is assumed that this trade-off between accuracy and computational demand plays an important role in the arbitration between the two strategies, but we show that the hallmark task for dissociating model-free and model-based strategies, as well as several related variants, do not embody such a trade-off. We describe five factors that reduce the effectiveness of the model-based strategy on these tasks by reducing its accuracy in estimating reward outcomes and decreasing the importance of its choices. Based on these observations, we describe a version of the task that formally and empirically obtains an accuracy-demand trade-off between model-free and model-based strategies. Moreover, we show that human participants spontaneously increase their reliance on model-based control on this task, compared to the original paradigm. Our novel task and our computational analyses may prove important in subsequent empirical investigations of how humans balance accuracy and demand.

}, issn = {1553-7358}, doi = {10.1371/journal.pcbi.1005090}, author = {Kool, Wouter and Fiery A Cushman and Samuel J Gershman} }