@article {5243,
	title = {SGD and Weight Decay Provably Induce a Low-Rank Bias in Deep Neural Networks},
	year = {2023},
	abstract = {<p>In this paper, we study the bias of Stochastic Gradient Descent (SGD) to learn low-rank weight matrices when training deep ReLU neural networks. Our results show that training neural networks with mini-batch SGD and weight decay causes a bias towards rank minimization over the weight matrices. Specifically, we show, both theoretically and empirically, that this bias is more pronounced when using smaller batch sizes, higher learning rates, or increased weight decay. Additionally, we predict and observe empirically that weight decay is necessary to achieve this bias. Finally, we empirically investigate the connection between this bias and generalization, finding that it has a marginal effect on generalization. Our analysis is based on a minimal set of assumptions and applies to neural networks of any width or depth, including those with residual connections and convolutional layers.</p>
},
	author = {Tomer Galanti and Zachary Siegel and Aparna Gupte and Tomaso Poggio}
}
@article {5092,
	title = {PCA as a defense against some adversaries},
	year = {2022},
	abstract = {<p>Neural network classifiers are known to be highly vulnerable to adversarial perturbations in their inputs. Under the hypothesis that adversarial examples lie outside of the sub-manifold of natural images, previous work has investigated the impact of principal components in data on adversarial robustness. In this paper we show that there exists a very simple defense mechanism in the case where adversarial images are separable in a previously defined $(k,p)$ metric. This defense is very successful against the\&nbsp; popular Carlini-Wagner attack, but less so against some other common attacks like FGSM. It is interesting to note that the defense is still successful for relatively large perturbations.</p>
},
	author = {Aparna Gupte and Andrzej Banburski and Tomaso Poggio}
}
