@inproceedings{6942,
abstract = {Graph games and Markov decision processes (MDPs) are standard models in reactive synthesis and verification of probabilistic systems with nondeterminism. The class of đťś” -regular winning conditions; e.g., safety, reachability, liveness, parity conditions; provides a robust and expressive specification formalism for properties that arise in analysis of reactive systems. The resolutions of nondeterminism in games and MDPs are represented as strategies, and we consider succinct representation of such strategies. The decision-tree data structure from machine learning retains the flavor of decisions of strategies and allows entropy-based minimization to obtain succinct trees. However, in contrast to traditional machine-learning problems where small errors are allowed, for winning strategies in graph games and MDPs no error is allowed, and the decision tree must represent the entire strategy. In this work we propose decision trees with linear classifiers for representation of strategies in graph games and MDPs. We have implemented strategy representation using this data structure and we present experimental results for problems on graph games and MDPs, which show that this new data structure presents a much more efficient strategy representation as compared to standard decision trees.},
author = {Ashok, Pranav and BrĂˇzdil, TomĂˇĹˇ and Chatterjee, Krishnendu and KĹ™etĂnskĂ˝, Jan and Lampert, Christoph and Toman, Viktor},
booktitle = {16th International Conference on Quantitative Evaluation of Systems},
isbn = {9783030302801},
issn = {0302-9743},
location = {Glasgow, United Kingdom},
pages = {109--128},
publisher = {Springer Nature},
title = {{Strategy representation by decision trees with linear classifiers}},
doi = {10.1007/978-3-030-30281-8_7},
volume = {11785},
year = {2019},
}
@inproceedings{6569,
abstract = {Knowledge distillation, i.e. one classifier being trained on the outputs of another classifier, is an empirically very successful technique for knowledge transfer between classifiers. It has even been observed that classifiers learn much faster and more reliably if trained with the outputs of another classifier as soft labels, instead of from ground truth data. So far, however, there is no satisfactory theoretical explanation of this phenomenon. In this work, we provide the first insights into the working mechanisms of distillation by studying the special case of linear and deep linear classifiers. Specifically, we prove a generalization bound that establishes fast convergence of the expected risk of a distillation-trained linear classifier. From the bound and its proof we extract three keyfactors that determine the success of distillation: data geometry â€“ geometric properties of the datadistribution, in particular class separation, has an immediate influence on the convergence speed of the risk; optimization biasâ€“ gradient descentoptimization finds a very favorable minimum of the distillation objective; and strong monotonicityâ€“ the expected risk of the student classifier always decreases when the size of the training set grows.},
author = {Bui Thi Mai, Phuong and Lampert, Christoph},
booktitle = {Proceedings of the 36th International Conference on Machine Learning},
location = {Long Beach, CA, United States},
pages = {5142--5151},
publisher = {PMLR},
title = {{Towards understanding knowledge distillation}},
volume = {97},
year = {2019},
}
@inproceedings{6482,
abstract = {Computer vision systems for automatic image categorization have become accurate and reliable enough that they can run continuously for days or even years as components of real-world commercial applications. A major open problem in this context, however, is quality control. Good classification performance can only be expected if systems run under the specific conditions, in particular data distributions, that they were trained for. Surprisingly, none of the currently used deep network architectures have a built-in functionality that could detect if a network operates on data from a distribution it was not trained for, such that potentially a warning to the human users could be triggered. In this work, we describe KS(conf), a procedure for detecting such outside of specifications (out-of-specs) operation, based on statistical testing of the network outputs. We show by extensive experiments using the ImageNet, AwA2 and DAVIS datasets on a variety of ConvNets architectures that KS(conf) reliably detects out-of-specs situations. It furthermore has a number of properties that make it a promising candidate for practical deployment: it is easy to implement, adds almost no overhead to the system, works with all networks, including pretrained ones, and requires no a priori knowledge of how the data distribution could change. },
author = {Sun, RĂ©my and Lampert, Christoph},
isbn = {9783030129385},
issn = {0302-9743},
location = {Stuttgart, Germany},
pages = {244--259},
publisher = {Springer Nature},
title = {{KS(conf): A light-weight test if a ConvNet operates outside of Its specifications}},
doi = {10.1007/978-3-030-12939-2_18},
volume = {11269},
year = {2019},
}
@article{6944,
abstract = {We study the problem of automatically detecting if a given multi-class classifier operates outside of its specifications (out-of-specs), i.e. on input data from a different distribution than what it was trained for. This is an important problem to solve on the road towards creating reliable computer vision systems for real-world applications, because the quality of a classifierâ€™s predictions cannot be guaranteed if it operates out-of-specs. Previously proposed methods for out-of-specs detection make decisions on the level of single inputs. This, however, is insufficient to achieve low false positive rate and high false negative rates at the same time. In this work, we describe a new procedure named KS(conf), based on statistical reasoning. Its main component is a classical Kolmogorovâ€“Smirnov test that is applied to the set of predicted confidence values for batches of samples. Working with batches instead of single samples allows increasing the true positive rate without negatively affecting the false positive rate, thereby overcoming a crucial limitation of single sample tests. We show by extensive experiments using a variety of convolutional network architectures and datasets that KS(conf) reliably detects out-of-specs situations even under conditions where other tests fail. It furthermore has a number of properties that make it an excellent candidate for practical deployment: it is easy to implement, adds almost no overhead to the system, works with any classifier that outputs confidence scores, and requires no a priori knowledge about how the data distribution could change.},
author = {Sun, RĂ©my and Lampert, Christoph},
issn = {1573-1405},
journal = {International Journal of Computer Vision},
publisher = {Springer Nature},
title = {{KS(conf): A light-weight test if a multiclass classifier operates outside of its specifications}},
doi = {10.1007/s11263-019-01232-x},
year = {2019},
}
@inproceedings{6590,
abstract = {Modern machine learning methods often require more data for training than a single expert can provide. Therefore, it has become a standard procedure to collect data from external sources, e.g. via crowdsourcing. Unfortunately, the quality of these sources is not always guaranteed. As additional complications, the data might be stored in a distributed way, or might even have to remain private. In this work, we address the question of how to learn robustly in such scenarios. Studying the problem through the lens of statistical learning theory, we derive a procedure that allows for learning from all available sources, yet automatically suppresses irrelevant or corrupted data. We show by extensive experiments that our method provides significant improvements over alternative approaches from robust statistics and distributed optimization. },
author = {Konstantinov, Nikola H and Lampert, Christoph},
booktitle = {Proceedings of the 36th International Conference on Machine Learning},
location = {Long Beach, CA, USA},
pages = {3488--3498},
publisher = {PMLR},
title = {{Robust learning from untrusted sources}},
volume = {97},
year = {2019},
}