Our paper on convergence of gradient flow on multi-layer linear networks [1] has been accepted to International Conference on Machine Learning! Congrats Hancheng!
[Bibtex] [Abstract] [Download PDF]
In this paper, we analyze the convergence of gradient flow on a multi-layer linear model with a loss function of the form $f(W_1 W_2 łdots W_L)$. We show that when $f$ satisfies the gradient dominance property, proper weight initialization leads to exponential convergence of the gradient flow to a global minimum of the loss. Moreover, the convergence rate depends on two trajectory-specific quantities that are controlled by the weight initialization: the imbalance matrices, which measure the difference between the weights of adjacent layers, and the least singular value of the weight product $W = W_1 W_2 łdots W_L$. Our analysis exploits the fact that the gradient of the overparameterized loss can be written as the composition of the non-overparametrized gradient with a time-varying (weight-dependent) linear operator whose smallest eigenvalue controls the convergence rate. The key challenge we address is to derive a uniform lower bound for this time-varying eigenvalue that lead to improved rates for several multi-layer network models studied in the literature.
@inproceedings{mvm2023icml,
abstract = {In this paper, we analyze the convergence of gradient flow on a multi-layer linear model with a loss function of the form $f(W_1 W_2 łdots W_L)$. We show that when $f$ satisfies the gradient dominance property, proper weight initialization leads to exponential convergence of the gradient flow to a global minimum of the loss. Moreover, the convergence rate depends on two trajectory-specific quantities that are controlled by the weight initialization: the imbalance matrices, which measure the difference between the weights of adjacent layers, and the least singular value of the weight product $W = W_1 W_2 łdots W_L$. Our analysis exploits the fact that the gradient of the overparameterized loss can be written as the composition of the non-overparametrized gradient with a time-varying (weight-dependent) linear operator whose smallest eigenvalue controls the convergence rate. The key challenge we address is to derive a uniform lower bound for this time-varying eigenvalue that lead to improved rates for several multi-layer network models studied in the literature.},
author = {Min, Hancheng and Vidal, Rene and Mallada, Enrique},
bdsk-url-3 = {https://mallada.ece.jhu.edu/pubs/2023-ICML-MVM.pdf},
booktitle = {International Conference on Machine Learning (ICML)},
grants = {TRIPODS-1934979, CAREER-1752362},
month = {4},
pages = {1-8},
record = {presented Jul. 2023, accepted Apr. 2023, submitted Jan. 2023},
title = {On the Convergence of Gradient Flow on Multi-layer Linear Models},
url = {https://mallada.ece.jhu.edu/pubs/2023-ICML-MVM.pdf},
year = {2023}
}