[{"_id":"15039","article_number":"2311.06103","type":"preprint","status":"public","citation":{"apa":"Prach, B., & Lampert, C. (n.d.). 1-Lipschitz neural networks are more expressive with N-activations. arXiv. https://doi.org/10.48550/ARXIV.2311.06103","ama":"Prach B, Lampert C. 1-Lipschitz neural networks are more expressive with N-activations. arXiv. doi:10.48550/ARXIV.2311.06103","ieee":"B. Prach and C. Lampert, “1-Lipschitz neural networks are more expressive with N-activations,” arXiv. .","short":"B. Prach, C. Lampert, ArXiv (n.d.).","mla":"Prach, Bernd, and Christoph Lampert. “1-Lipschitz Neural Networks Are More Expressive with N-Activations.” ArXiv, 2311.06103, doi:10.48550/ARXIV.2311.06103.","ista":"Prach B, Lampert C. 1-Lipschitz neural networks are more expressive with N-activations. arXiv, 2311.06103.","chicago":"Prach, Bernd, and Christoph Lampert. “1-Lipschitz Neural Networks Are More Expressive with N-Activations.” ArXiv, n.d. https://doi.org/10.48550/ARXIV.2311.06103."},"date_updated":"2024-03-04T07:02:39Z","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","article_processing_charge":"No","external_id":{"arxiv":["2311.06103"]},"author":[{"last_name":"Prach","full_name":"Prach, Bernd","id":"2D561D42-C427-11E9-89B4-9C1AE6697425","first_name":"Bernd"},{"last_name":"Lampert","orcid":"0000-0001-8622-7887","full_name":"Lampert, Christoph","first_name":"Christoph","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87"}],"title":"1-Lipschitz neural networks are more expressive with N-activations","department":[{"_id":"GradSch"},{"_id":"ChLa"}],"abstract":[{"text":"A crucial property for achieving secure, trustworthy and interpretable deep learning systems is their robustness: small changes to a system's inputs should not result in large changes to its outputs. Mathematically, this means one strives for networks with a small Lipschitz constant. Several recent works have focused on how to construct such Lipschitz networks, typically by imposing constraints on the weight matrices. In this work, we study an orthogonal aspect, namely the role of the activation function. We show that commonly used activation functions, such as MaxMin, as well as all piece-wise linear ones with two segments unnecessarily restrict the class of representable functions, even in the simplest one-dimensional setting. We furthermore introduce the new N-activation function that is provably more expressive than currently popular activation functions. We provide code at this https URL.","lang":"eng"}],"oa_version":"Preprint","oa":1,"main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2311.06103","open_access":"1"}],"month":"11","year":"2023","publication_status":"submitted","publication":"arXiv","language":[{"iso":"eng"}],"day":"10","date_created":"2024-02-28T17:59:32Z","date_published":"2023-11-10T00:00:00Z","doi":"10.48550/ARXIV.2311.06103"},{"_id":"11839","conference":{"start_date":"2022-10-23","end_date":"2022-10-27","location":"Tel Aviv, Israel","name":"ECCV: European Conference on Computer Vision"},"type":"conference","status":"public","date_updated":"2023-05-03T08:00:46Z","department":[{"_id":"GradSch"},{"_id":"ChLa"}],"abstract":[{"text":"It is a highly desirable property for deep networks to be robust against\r\nsmall input changes. One popular way to achieve this property is by designing\r\nnetworks with a small Lipschitz constant. In this work, we propose a new\r\ntechnique for constructing such Lipschitz networks that has a number of\r\ndesirable properties: it can be applied to any linear network layer\r\n(fully-connected or convolutional), it provides formal guarantees on the\r\nLipschitz constant, it is easy to implement and efficient to run, and it can be\r\ncombined with any training objective and optimization method. In fact, our\r\ntechnique is the first one in the literature that achieves all of these\r\nproperties simultaneously. Our main contribution is a rescaling-based weight\r\nmatrix parametrization that guarantees each network layer to have a Lipschitz\r\nconstant of at most 1 and results in the learned weight matrices to be close to\r\northogonal. Hence we call such layers almost-orthogonal Lipschitz (AOL).\r\nExperiments and ablation studies in the context of image classification with\r\ncertified robust accuracy confirm that AOL layers achieve results that are on\r\npar with most existing methods. Yet, they are simpler to implement and more\r\nbroadly applicable, because they do not require computationally expensive\r\nmatrix orthogonalization or inversion steps as part of the network\r\narchitecture. We provide code at https://github.com/berndprach/AOL.","lang":"eng"}],"oa_version":"Preprint","main_file_link":[{"url":" https://doi.org/10.48550/arXiv.2208.03160","open_access":"1"}],"alternative_title":["LNCS"],"scopus_import":"1","intvolume":" 13681","month":"10","publication_status":"published","publication_identifier":{"isbn":["9783031198021"],"eisbn":["9783031198038"]},"language":[{"iso":"eng"}],"volume":13681,"citation":{"ista":"Prach B, Lampert C. 2022. Almost-orthogonal layers for efficient general-purpose Lipschitz networks. Computer Vision – ECCV 2022. ECCV: European Conference on Computer Vision, LNCS, vol. 13681, 350–365.","chicago":"Prach, Bernd, and Christoph Lampert. “Almost-Orthogonal Layers for Efficient General-Purpose Lipschitz Networks.” In Computer Vision – ECCV 2022, 13681:350–65. Springer Nature, 2022. https://doi.org/10.1007/978-3-031-19803-8_21.","ieee":"B. Prach and C. Lampert, “Almost-orthogonal layers for efficient general-purpose Lipschitz networks,” in Computer Vision – ECCV 2022, Tel Aviv, Israel, 2022, vol. 13681, pp. 350–365.","short":"B. Prach, C. Lampert, in:, Computer Vision – ECCV 2022, Springer Nature, 2022, pp. 350–365.","apa":"Prach, B., & Lampert, C. (2022). Almost-orthogonal layers for efficient general-purpose Lipschitz networks. In Computer Vision – ECCV 2022 (Vol. 13681, pp. 350–365). Tel Aviv, Israel: Springer Nature. https://doi.org/10.1007/978-3-031-19803-8_21","ama":"Prach B, Lampert C. Almost-orthogonal layers for efficient general-purpose Lipschitz networks. In: Computer Vision – ECCV 2022. Vol 13681. Springer Nature; 2022:350-365. doi:10.1007/978-3-031-19803-8_21","mla":"Prach, Bernd, and Christoph Lampert. “Almost-Orthogonal Layers for Efficient General-Purpose Lipschitz Networks.” Computer Vision – ECCV 2022, vol. 13681, Springer Nature, 2022, pp. 350–65, doi:10.1007/978-3-031-19803-8_21."},"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","external_id":{"arxiv":["2208.03160"]},"article_processing_charge":"No","author":[{"full_name":"Prach, Bernd","last_name":"Prach","first_name":"Bernd","id":"2D561D42-C427-11E9-89B4-9C1AE6697425"},{"first_name":"Christoph","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","last_name":"Lampert","orcid":"0000-0001-8622-7887","full_name":"Lampert, Christoph"}],"title":"Almost-orthogonal layers for efficient general-purpose Lipschitz networks","oa":1,"publisher":"Springer Nature","quality_controlled":"1","year":"2022","publication":"Computer Vision – ECCV 2022","day":"23","page":"350-365","date_created":"2022-08-12T15:09:47Z","date_published":"2022-10-23T00:00:00Z","doi":"10.1007/978-3-031-19803-8_21"}]