@article {5244, title = {Feature learning in deep classifiers through Intermediate Neural Collapse}, number = {141}, year = {2023}, month = {02/2023}, abstract = {

In this paper, we conduct an empirical study of the feature learning process in deep classifiers. Recent research has identified a training phenomenon called Neural Collapse (NC), in which the top-layer feature embeddings of samples from the same class tend to concentrate around their means, and the top layer{\textquoteright}s weights align with those features. Our study aims to investigate if these properties extend to intermediate layers. We empirically study the evolution of the covariance and mean of representations across different layers and show that as we move deeper into a trained neural network, the within-class covariance decreases relative to the between-class covariance. Additionally, we find that in the top layers, where the between-class covariance is dominant, the subspace spanned by the class means aligns with the subspace spanned by the most significant singular vector components of the weight matrix in the corresponding layer. Finally, we discuss the relationship between NC and Associative Memories (Willshaw et al., 1969).

}, author = {Akshay Rangamani and Marius Lindegaard and Tomer Galanti and Tomaso Poggio} } @article {5466, title = {The Janus effects of SGD vs GD: high noise and low rank}, number = {144}, year = {2023}, month = {12/2024}, abstract = {

It was always obvious that \ SGD with small minibatch size yields for neural networks much higher asymptotic fluctuations in the updates of the weight matrices than GD. It has also been often reported that SGD in deep RELU networks shows empirically a low-rank bias in the weight matrices. A recent \ theoretical analysis derived a bound on the rank and linked it to the size of the SGD fluctuations [25]. In this paper, we provide an empirical and \ theoretical analysis of the convergence of SGD vs GD, first for deep RELU networks and then for the case of linear regression, where sharper estimates can be obtained and which is of independent interest. In the linear case, we prove that the component $W^\perp$ of the matrix $W$ corresponding to the null space of the data matrix $X$ converges to zero for both SGD and GD, provided the regularization term is non-zero. Because of the larger number of updates required to go through all the training data, the convergence rate {\it per epoch} of these components is much faster for SGD than for GD. In practice, SGD has a much stronger bias than GD towards solutions for weight matrices $W$ with high fluctuations -- even when the choice of mini batches is deterministic -- and low rank, provided the initialization is from a random matrix. Thus SGD \ with non-zero regularization, shows the coupled phenomenon of \ asymptotic noise and a low-rank bias-- unlike GD.

}, author = {Mengjia Xu and Tomer Galanti and Akshay Rangamani and Lorenzo Rosasco and Andrea Pinto and Tomaso Poggio} } @article {5360, title = {Skip Connections Increase the Capacity of Associative Memories in Variable Binding Mechanisms}, number = {142}, year = {2023}, month = {06/2023}, abstract = {

The flexibility of intelligent behavior is fundamentally attributed to the ability to separate and assign structural information from content in sensory inputs. Variable binding is the atomic computation that underlies this ability. In this work, we investigate the implementation of variable binding via pointers of assemblies of neurons, which are sets of excitatory neurons that fire together. The Assembly Calculus is a framework that describes a set of operations to create and modify assemblies of neurons. We focus on the project (which creates assemblies) and reciprocal-project (which performs vari- able binding) operations and study the capacity of networks in terms of the number of assemblies that can be reliably created and retrieved. We find that assembly calculus networks implemented through Hebbian plasticity resemble associative memories in their structure and behavior. However, for net- works with N neurons per brain area, the capacity of variable binding networks (0.01N) is an order of magnitude lower than the capacity of assembly creation networks (0.22N). To alleviate this drop in capacity, we propose a skip connection between the input and variable assembly, which boosts the capacity to a similar order of magnitude (0.1N ) as the Project operation, while maintain its biological plausibility.

}, author = {Yi Xie and Yichen Li and Akshay Rangamani} } @conference {5088, title = {Neural Collapse in Deep Homogeneous Classifiers and the role of Weight Decay}, booktitle = {IEEE International Conference on Acoustics, Speech and Signal Processing}, year = {2022}, month = {05/2022}, address = {Singapore}, author = {Andrzej Banburski and Akshay Rangamani} } @article {5137, title = {Understanding the Role of Recurrent Connections in Assembly Calculus}, year = {2022}, month = {07/2022}, abstract = {

In this note, we explore the role of recurrent connections in Assembly Calculus through a number of experiments\ conducted on models with and without recurrent connections. We observe that assemblies can be formed even in the absence of\ recurrent connections, but also find that models with recurrent connections are more robust to noisy inputs. We also investigate\ the spectral structure of the synaptic weights and find intriguing similarities between models of neural assemblies and\ associative memories.

}, author = {Akshay Rangamani and Yi Xie} } @article {4880, title = {Dynamics and Neural Collapse in Deep Classifiers trained with the Square Loss}, year = {2021}, abstract = {

We overview several properties -- old and new -- \ of training overparametrized deep networks under the square loss. We first consider a model of the dynamics of gradient flow under the square loss in deep homogeneous ReLU networks. We study the convergence to a solution with the absolute minimum $\rho$, which is the product of the Frobenius norms of each layer weight matrix, when normalization by \ Lagrange multipliers (LM) is used together with Weight Decay (WD) under different forms of gradient descent. A main property of the minimizers that bounds their expected error {\it for a specific network architecture} is $\rho$. In particular, we derive novel norm-based bounds for convolutional layers that are orders of magnitude better than classical bounds for dense networks. Next we prove that quasi-interpolating solutions obtained by Stochastic Gradient Descent (SGD) in the presence of WD have a bias towards low rank weight matrices -- that, as we also explain, should improve generalization. The same analysis predicts the existence of an inherent SGD noise for deep networks. In both cases, we verify our predictions experimentally. We then predict Neural Collapse and its properties without any specific assumption -- unlike other published proofs. Our analysis supports the idea that the advantage of deep networks relative to other classifiers is greater for the problems that are appropriate for sparse deep architectures such as CNNs. The deep reason compositionally sparse \ target functions \ can be approximated well by {\textquoteleft}{\textquoteleft}sparse{\textquoteright}{\textquoteright} deep networks without incurring in the curse of dimensionality.

}, author = {M. Xu and Akshay Rangamani and Andrzej Banburski and Q. Liao and Tomer Galanti and Tomaso Poggio} } @article {4568, title = {For interpolating kernel machines, the minimum norm ERM solution is the most stable}, number = {108}, year = {2020}, month = {06/2020}, abstract = {

We study the average CVloo stability of kernel ridge-less regression and derive corresponding risk bounds. We show that the interpolating solution with minimum norm has the best CVloo stability, which in turn is controlled by the condition number of the empirical kernel matrix. The latter can be characterized in the asymptotic regime where both the dimension and cardinality of the data go to infinity. Under the assumption of random kernel matrices, the corresponding test error follows a double descent curve.

}, author = {Akshay Rangamani and Lorenzo Rosasco and Tomaso Poggio} }