@article{(Open Science Index):https://publications.waset.org/pdf/10013347,
	  title     = {An Empirical Study on Switching Activation Functions in Shallow and Deep Neural Networks},
	  author    = {Apoorva Vinod and  Archana Mathur and  Snehanshu Saha},
	  country	= {},
	  institution	= {},
	  abstract     = {Though there exists a plethora of Activation Functions (AFs) used in single and multiple hidden layer Neural Networks (NN), their behavior always raised curiosity, whether used in combination or singly. The popular AFs – Sigmoid, ReLU, and Tanh – have performed prominently well for shallow and deep architectures. Most of the time, AFs are used singly in multi-layered NN, and, to the best of our knowledge, their performance is never studied and analyzed deeply when used in combination. In this manuscript, we experiment on multi-layered NN architecture (both on shallow and deep architectures; Convolutional NN and VGG16) and investigate how well the network responds to using two different AFs (Sigmoid-Tanh, Tanh-ReLU, ReLU-Sigmoid) used alternately against a traditional, single (Sigmoid-Sigmoid, Tanh-Tanh, ReLU-ReLU) combination. Our results show that on using two different AFs, the network achieves better accuracy, substantially lower loss, and faster convergence on 4 computer vision (CV) and 15 Non-CV (NCV) datasets. When using different AFs, not only was the accuracy greater by 6-7%, but we also accomplished convergence twice as fast. We present a case study to investigate the probability of networks suffering vanishing and exploding gradients when using two different AFs. Additionally, we theoretically showed that a composition of two or more AFs satisfies Universal Approximation Theorem (UAT). },
	    journal   = {International Journal of Computer and Systems Engineering},
	  volume    = {17},
	  number    = {11},
	  year      = {2023},
	  pages     = {630 - 637},
	  ee        = {https://publications.waset.org/pdf/10013347},
	  url   	= {https://publications.waset.org/vol/203},
	  bibsource = {https://publications.waset.org/},
	  issn  	= {eISSN: 1307-6892},
	  publisher = {World Academy of Science, Engineering and Technology},
	  index 	= {Open Science Index 203, 2023},
	}