@article {5242, title = {Norm-Based Generalization Bounds for Compositionally Sparse Neural Networks}, year = {2023}, abstract = {

In this paper, we investigate the Rademacher complexity of deep sparse neural networks, where each neuron receives a small number of inputs. We prove generalization bounds for multilayered sparse ReLU neural networks, including convolutional neural networks. These bounds differ from previous ones, as they consider the norms of the convolutional filters instead of the norms of the associated Toeplitz matrices, independently of weight sharing between neurons.

As we show theoretically, these bounds may be orders of magnitude better than standard norm-based generalization bounds and empirically, they are almost non-vacuous in estimating generalization in various simple classification problems. Taken together, these results suggest that compositional sparsity of the underlying target function is critical to the success of deep neural networks.

}, author = {Tomer Galanti and Mengjia Xu and Liane Galanti and Tomaso Poggio} } @conference {5496, title = {Norm-based Generalization Bounds for Sparse Neural Networks}, booktitle = {NeurIPS 2023}, year = {2023}, month = {12/2023}, address = {New Orleans}, abstract = {

In this paper, we derive norm-based generalization bounds for sparse ReLU neural networks, including convolutional neural networks. These bounds differ from previous ones because they consider the sparse structure of the neural network architecture and the norms of the convolutional filters, rather than the norms of the (Toeplitz) matrices associated with the convolutional layers. Theoretically, we demonstrate that these bounds are significantly tighter than standard norm-based generalization bounds. Empirically, they offer relatively tight estimations of generalization for various simple classification problems. Collectively, these findings suggest that the sparsity of the underlying target function and the model{\textquoteright}s architecture plays a crucial role in the success of deep learning.

}, url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/8493e190ff1bbe3837eca821190b61ff-Paper-Conference.pdf}, author = {Tomer Galanti and Mengjia Xu and Liane Galanti and Tomaso Poggio} } @article {5089, title = {On the Implicit Bias Towards Minimal Depth of Deep Neural Networks}, journal = {arXiv}, year = {2022}, month = {03/2022}, abstract = {

We study the implicit bias of gradient based training methods to favor low-depth solutions when training deep neural networks. Recent results in the literature suggest that penultimate layer representations learned by a classifier over multiple classes exhibit a clustering property, called neural collapse. We demonstrate empirically that neural collapse extends beyond the penultimate layer and emerges in intermediate layers as well. In this regards, we hypothesize and empirically show that gradient based methods are implicitly biased towards selecting neural networks of minimal depth for achieving this clustering property.

}, url = {https://arxiv.org/abs/2202.09028}, author = {Tomer Galanti and Liane Galanti} }