LIMBO

2019

Chameleon: A Hybrid, Proactive Auto-Scaling Mechanism on a Level-Playing Field. A. Bauer; N. Herbst; S. Spinner; A. Ali-Eldin; S. Kounev; in IEEE Transactions on Parallel and Distributed Systems (2019). 30(4) 800–813.
[ BibTeX ] [ EndNote ]
 
@article{BaHeSpAEKo2019-TPDS-ChameleonAutoscaler, author = {Bauer, Andr{é} and Herbst, Nikolas and Spinner, Simon and Ali-Eldin, Ahmed and Kounev, Samuel}, journal = {IEEE Transactions on Parallel and Distributed Systems}, keywords = {Metrics_and_benchmarking_methodologies Resource_management Performance descartes Prediction Elasticity Chameleon LIMBO Tool Online_monitoring_and_forecasting PRISMA Self-aware-computing Cloud BUNGEE Application_quality_of_service_management Application-aware t_journalmagazine SPEC Self-adaptive-systems}, month = {April}, number = 4, pages = {800–813}, publisher = {IEEE}, title = {{Chameleon: A Hybrid, Proactive Auto-Scaling Mechanism on a Level-Playing Field}}, volume = 30, year = 2019 }
%0 Journal Article %1 BaHeSpAEKo2019-TPDS-ChameleonAutoscaler %A Bauer, André %A Herbst, Nikolas %A Spinner, Simon %A Ali-Eldin, Ahmed %A Kounev, Samuel %D 2019 %I IEEE %J IEEE Transactions on Parallel and Distributed Systems %N 4 %P 800--813 %T Chameleon: A Hybrid, Proactive Auto-Scaling Mechanism on a Level-Playing Field %V 30
Methoden und Messverfahren für Automatisches Skalieren in Elastischen Cloud Umgebungen. N. Herbst; (2019, Juni).
[ BibTeX ] [ EndNote ]
 
@misc{talk-Herbst-GIAwardNomination-Dagstuhl, author = {Herbst, Nikolas}, howpublished = {GI Dissertationspreis: Seminar f{ü}r Nominierte Doktoranden, Dagstuhl, Germany, June 28, 2019}, keywords = {Telescope Metrics_and_benchmarking_methodologies Resource_management descartes Prediction Chameleon Instrumentation_profiling_and_workload_characterization LIMBO Online_monitoring_and_forecasting se2_talk Cloud Award BUNGEE Virtualization t_bookchapter}, month = {June}, title = {{Methoden und Messverfahren f{ü}r Automatisches Skalieren in Elastischen Cloud Umgebungen}}, year = 2019 }
%0 Generic %1 talk-Herbst-GIAwardNomination-Dagstuhl %A Herbst, Nikolas %D 2019 %T Methoden und Messverfahren für Automatisches Skalieren in Elastischen Cloud Umgebungen
Measuring, Rating, and Predicting the Energy Efficiency of Servers. J. von Kistowski; (2019, März).
[ BibTeX ] [ EndNote ] [ URL ]
 
@phdthesis{Kistowski2019-phd, author = {von Kistowski, J{ó}akim}, keywords = {Supervised_PhD_Thesis t_dissertation descartes Prediction LIMBO Power}, month = {March}, school = {University of W{ü}rzburg, Germany}, title = {Measuring, Rating, and Predicting the Energy Efficiency of Servers}, year = 2019 }
%0 Thesis %1 Kistowski2019-phd %A von Kistowski, Jóakim %D 2019 %T Measuring, Rating, and Predicting the Energy Efficiency of Servers %U https://opus.bibliothek.uni-wuerzburg.de/frontdoor/index/index/docId/17847

2018

Methods and Benchmarks for Auto-Scaling Mechanisms in Elastic Cloud Environments. N. Herbst; (2018, Juli).
[ BibTeX ] [ EndNote ] [ URL ]
 
@phdthesis{Herbst2018-phd, author = {Herbst, Nikolas}, keywords = {Telescope Metrics_and_benchmarking_methodologies Resource_management Supervised_PhD_Thesis descartes Elasticity Chameleon LIMBO Tool Online_monitoring_and_forecasting t_dissertation Cloud WCF BUNGEE Application_quality_of_service_management SPEC}, month = {July}, note = {<b>{Distinction, SPEC Kaivalya Dixit Distinguished Dissertation Award 2018}</b>}, school = {University of W{ü}rzburg, Germany}, title = {Methods and Benchmarks for Auto-Scaling Mechanisms in Elastic Cloud Environments}, year = 2018 }
%0 Thesis %1 Herbst2018-phd %A Herbst, Nikolas %D 2018 %T Methods and Benchmarks for Auto-Scaling Mechanisms in Elastic Cloud Environments %U https://opus.bibliothek.uni-wuerzburg.de/frontdoor/index/index/docId/16431
How we built a scalable micro-service application - The Tea Store. N. Herbst; (2018, Juli).
[ BibTeX ] [ EndNote ] [ URL ]
 
@misc{talk-Herbst-ScrumScale, author = {Herbst, Nikolas}, howpublished = {Keynote Talk at the ScrumScale workshop, Oslo, Norway, July 5, 2018}, keywords = {Statistical_estimation_and_machine_learning Telescope Resource_management Performance descartes Elasticity Talks LIMBO Online_monitoring_and_forecasting se2_talk t_talk Cloud SPEC}, month = {July}, title = {{How we built a scalable micro-service application - The Tea Store}}, year = 2018 }
%0 Generic %1 talk-Herbst-ScrumScale %A Herbst, Nikolas %D 2018 %T How we built a scalable micro-service application - The Tea Store %U https://www.dataforeningen.no/smidig-skalerbarhet.6113996-134138.html

2017

Runtime Prediction of Power Consumption for Server Reconfigurations. M. Deffner; Thesis; University of Würzburg; Am Hubland, Informatikgebäude, 97074 Würzburg, Germany. (2017, Mai).
[ BibTeX ] [ EndNote ]
 
@mastersthesis{Deffner2017, address = {Am Hubland, Informatikgeb{ä}ude, 97074 W{ü}rzburg, Germany}, author = {Deffner, Maximilian}, keywords = {Thesis_supervised_by_SE_member Statistical_estimation_and_machine_learning Performance descartes Prediction LIMBO se2_mastersthesis Online_monitoring_and_forecasting Self-aware-computing Thesis_supervised_by_Joakim_vonKistowski Application-aware Virtualization Power-energy_efficient_computing Power}, month = {May}, school = {University of W{ü}rzburg}, title = {{Runtime Prediction of Power Consumption for Server Reconfigurations}}, type = {{Master Thesis}}, year = 2017 }
%0 Thesis %1 Deffner2017 %A Deffner, Maximilian %C Am Hubland, Informatikgebäude, 97074 Würzburg, Germany %D 2017 %T Runtime Prediction of Power Consumption for Server Reconfigurations
Design and Evaluation of a Proactive, Application-Aware Auto-Scaler. A. Bauer; N. Herbst; S. Kounev; in Proceedings of the 8th ACM/SPEC International Conference on Performance Engineering (ICPE 2017) (2017).
[ BibTeX ] [ EndNote ]
 
@inproceedings{BaHeKo2017-ICPE-Auto-Scaler-Tutorial, author = {Bauer, Andr{é} and Herbst, Nikolas and Kounev, Samuel}, booktitle = {{Proceedings of the 8th ACM/SPEC International Conference on Performance Engineering (ICPE 2017)}}, keywords = {DML Online_monitoring_and_forecasting descartes WCF Automated_model_learning Elasticity t_tutorial BUNGEE LibReDE Chameleon Application-aware LIMBO}, month = {April}, title = {{Design and Evaluation of a Proactive, Application-Aware Auto-Scaler}}, year = 2017 }
%0 Conference Paper %1 BaHeKo2017-ICPE-Auto-Scaler-Tutorial %A Bauer, André %A Herbst, Nikolas %A Kounev, Samuel %B Proceedings of the 8th ACM/SPEC International Conference on Performance Engineering (ICPE 2017) %D 2017 %T Design and Evaluation of a Proactive, Application-Aware Auto-Scaler
Modeling and Extracting Load Intensity Profiles. J. von Kistowski; N. Herbst; S. Kounev; H. Groenda; C. Stier; S. Lehrig; in ACM Transactions on Autonomous and Adaptive Systems (TAAS) (2017). 11(4) 23:1–23:28.
[ Abstract ] [ BibTeX ] [ EndNote ] [ URL ]
 
Today's system developers and operators face the challenge of creating software systems that make efficient use of dynamically allocated resources under highly variable and dynamic load profiles, while at the same time delivering reliable performance. Autonomic controllers, e.g., an advanced auto-scaling mechanism in a cloud computing context, can benefit from an abstracted load model as knowledge to reconfigure on time and precisely. Existing workload characterization approaches have limited support to capture variations the inter-arrival times of incoming work units over time (i.e., a variable load profile). For example, industrial and scientific benchmarks support constant or stepwise increasing load, or inter-arrival times defined by statistical distributions or recorded traces. These options show shortcomings either in representative character of load variation patterns or in abstraction and flexibility of their format. In this article, we present the Descartes Load Intensity Model (DLIM) approach addressing these issues. DLIM provides a modeling formalism for describing load intensity variations over time. A DLIM instance is a compact formal description of a load intensity trace. DLIM-based tools provide features for benchmarking, performance and recorded load intensity trace analysis. As manually obtaining and maintaining DLIM instances becomes time consuming, we contribute three automated extraction methods and devised metrics for comparison and method selection. We discuss how these features are used to enhance system management approaches for adaptations during run-time, and how they are integrated into simulation contexts and enable benchmarking of elastic or adaptive behavior. We show that automatically extracted DLIM instances exhibit an average modeling error of 15.2% over ten different real-world traces that cover between two weeks and seven months. These results underline DLIM model expressiveness. In terms of accuracy and processing speed, our proposed extraction methods for the descriptive models are comparable to existing time series decomposition methods. Additionally, we illustrate DLIM applicability by outlining approaches of workload modeling in systems engineering that employ or rely on our proposed load intensity modeling formalism.
@article{KiHeKo-TAAS17-ModelExtractLoadProfiles, abstract = {Today's system developers and operators face the challenge of creating software systems that make efficient use of dynamically allocated resources under highly variable and dynamic load profiles, while at the same time delivering reliable performance. Autonomic controllers, e.g., an advanced auto-scaling mechanism in a cloud computing context, can benefit from an abstracted load model as knowledge to reconfigure on time and precisely. Existing workload characterization approaches have limited support to capture variations the inter-arrival times of incoming work units over time (i.e., a variable load profile). For example, industrial and scientific benchmarks support constant or stepwise increasing load, or inter-arrival times defined by statistical distributions or recorded traces. These options show shortcomings either in representative character of load variation patterns or in abstraction and flexibility of their format. In this article, we present the Descartes Load Intensity Model (DLIM) approach addressing these issues. DLIM provides a modeling formalism for describing load intensity variations over time. A DLIM instance is a compact formal description of a load intensity trace. DLIM-based tools provide features for benchmarking, performance and recorded load intensity trace analysis. As manually obtaining and maintaining DLIM instances becomes time consuming, we contribute three automated extraction methods and devised metrics for comparison and method selection. We discuss how these features are used to enhance system management approaches for adaptations during run-time, and how they are integrated into simulation contexts and enable benchmarking of elastic or adaptive behavior. We show that automatically extracted DLIM instances exhibit an average modeling error of 15.2\% over ten different real-world traces that cover between two weeks and seven months. These results underline DLIM model expressiveness. In terms of accuracy and processing speed, our proposed extraction methods for the descriptive models are comparable to existing time series decomposition methods. Additionally, we illustrate DLIM applicability by outlining approaches of workload modeling in systems engineering that employ or rely on our proposed load intensity modeling formalism.}, address = {New York, NY, USA}, author = {von Kistowski, J{ó}akim and Herbst, Nikolas and Kounev, Samuel and Groenda, Henning and Stier, Christian and Lehrig, Sebastian}, journal = {ACM Transactions on Autonomous and Adaptive Systems (TAAS)}, keywords = {Tool Resource_management descartes Self-aware-computing Automated_model_learning Elasticity BUNGEE t_journalmagazine SPEC LIMBO Self-adaptive-systems}, month = {January}, number = 4, pages = {23:1&ndash;23:28}, publisher = {ACM}, title = {{Modeling and Extracting Load Intensity Profiles}}, volume = 11, year = 2017 }
%0 Journal Article %1 KiHeKo-TAAS17-ModelExtractLoadProfiles %A von Kistowski, Jóakim %A Herbst, Nikolas %A Kounev, Samuel %A Groenda, Henning %A Stier, Christian %A Lehrig, Sebastian %C New York, NY, USA %D 2017 %I ACM %J ACM Transactions on Autonomous and Adaptive Systems (TAAS) %N 4 %P 23:1--23:28 %T Modeling and Extracting Load Intensity Profiles %U http://doi.acm.org/10.1145/3019596 %V 11 %X Today's system developers and operators face the challenge of creating software systems that make efficient use of dynamically allocated resources under highly variable and dynamic load profiles, while at the same time delivering reliable performance. Autonomic controllers, e.g., an advanced auto-scaling mechanism in a cloud computing context, can benefit from an abstracted load model as knowledge to reconfigure on time and precisely. Existing workload characterization approaches have limited support to capture variations the inter-arrival times of incoming work units over time (i.e., a variable load profile). For example, industrial and scientific benchmarks support constant or stepwise increasing load, or inter-arrival times defined by statistical distributions or recorded traces. These options show shortcomings either in representative character of load variation patterns or in abstraction and flexibility of their format. In this article, we present the Descartes Load Intensity Model (DLIM) approach addressing these issues. DLIM provides a modeling formalism for describing load intensity variations over time. A DLIM instance is a compact formal description of a load intensity trace. DLIM-based tools provide features for benchmarking, performance and recorded load intensity trace analysis. As manually obtaining and maintaining DLIM instances becomes time consuming, we contribute three automated extraction methods and devised metrics for comparison and method selection. We discuss how these features are used to enhance system management approaches for adaptations during run-time, and how they are integrated into simulation contexts and enable benchmarking of elastic or adaptive behavior. We show that automatically extracted DLIM instances exhibit an average modeling error of 15.2% over ten different real-world traces that cover between two weeks and seven months. These results underline DLIM model expressiveness. In terms of accuracy and processing speed, our proposed extraction methods for the descriptive models are comparable to existing time series decomposition methods. Additionally, we illustrate DLIM applicability by outlining approaches of workload modeling in systems engineering that employ or rely on our proposed load intensity modeling formalism.

2015

BUNGEE: An Elasticity Benchmark for Self-Adaptive IaaS Cloud Environments. N. R. Herbst; S. Kounev; A. Weber; H. Groenda; in Proceedings of the 10th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2015) (2015).
[ Abstract ] [ BibTeX ] [ EndNote ]
 
Today's infrastructure clouds provide resource elasticity (i.e. auto-scaling) mechanisms enabling self-adaptive resource provisioning to reflect variations in the load intensity over time. These mechanisms impact on the application performance, however, their effect in specific situations is hard to quantify and compare. To evaluate the quality of elasticity mechanisms provided by different platforms and configurations, respective metrics and benchmarks are required. Existing metrics for elasticity only consider the time required to provision and deprovision resources or the costs impact of adaptations. Existing benchmarks lack the capability to handle open workloads with realistic load intensity profiles and do not explicitly distinguish between the performance exhibited by the provisioned underlying resources, on the one hand, and the quality of the elasticity mechanisms themselves, on the other hand. In this paper, we propose reliable metrics for quantifying the timing aspects and accuracy of elasticity. Based on these metrics, we propose a novel approach for benchmarking the elasticity of Infrastructure-as-a-Service (IaaS) cloud platforms independent of the performance exhibited by the provisioned underlying resources. We show that the proposed metrics provide consistent ranking of elastic platforms on an ordinal scale. Finally, we present an extensive case study of real-world complexity demonstrating that the proposed approach is applicable in realistic scenarios and can cope with different levels of resource efficiency.
@inproceedings{HeKoWeGr_2015_SEAMS_BUNGEE, abstract = {Today's infrastructure clouds provide resource elasticity (i.e. auto-scaling) mechanisms enabling self-adaptive resource provisioning to reflect variations in the load intensity over time. These mechanisms impact on the application performance, however, their effect in specific situations is hard to quantify and compare. To evaluate the quality of elasticity mechanisms provided by different platforms and configurations, respective metrics and benchmarks are required. Existing metrics for elasticity only consider the time required to provision and deprovision resources or the costs impact of adaptations. Existing benchmarks lack the capability to handle open workloads with realistic load intensity profiles and do not explicitly distinguish between the performance exhibited by the provisioned underlying resources, on the one hand, and the quality of the elasticity mechanisms themselves, on the other hand. In this paper, we propose reliable metrics for quantifying the timing aspects and accuracy of elasticity. Based on these metrics, we propose a novel approach for benchmarking the elasticity of Infrastructure-as-a-Service (IaaS) cloud platforms independent of the performance exhibited by the provisioned underlying resources. We show that the proposed metrics provide consistent ranking of elastic platforms on an ordinal scale. Finally, we present an extensive case study of real-world complexity demonstrating that the proposed approach is applicable in realistic scenarios and can cope with different levels of resource efficiency.}, author = {Herbst, Nikolas Roman and Kounev, Samuel and Weber, Andreas and Groenda, Henning}, booktitle = {Proceedings of the 10th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2015)}, keywords = {Tool Metrics_and_benchmarking_methodologies Resource_management t_full descartes Self-aware-computing Elasticity BUNGEE Virtualization LIMBO}, month = {May}, note = {Acceptance rate: 29\%}, title = {{BUNGEE: An Elasticity Benchmark for Self-Adaptive IaaS Cloud Environments}}, year = 2015 }
%0 Conference Paper %1 HeKoWeGr_2015_SEAMS_BUNGEE %A Herbst, Nikolas Roman %A Kounev, Samuel %A Weber, Andreas %A Groenda, Henning %B Proceedings of the 10th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2015) %D 2015 %T BUNGEE: An Elasticity Benchmark for Self-Adaptive IaaS Cloud Environments %X Today's infrastructure clouds provide resource elasticity (i.e. auto-scaling) mechanisms enabling self-adaptive resource provisioning to reflect variations in the load intensity over time. These mechanisms impact on the application performance, however, their effect in specific situations is hard to quantify and compare. To evaluate the quality of elasticity mechanisms provided by different platforms and configurations, respective metrics and benchmarks are required. Existing metrics for elasticity only consider the time required to provision and deprovision resources or the costs impact of adaptations. Existing benchmarks lack the capability to handle open workloads with realistic load intensity profiles and do not explicitly distinguish between the performance exhibited by the provisioned underlying resources, on the one hand, and the quality of the elasticity mechanisms themselves, on the other hand. In this paper, we propose reliable metrics for quantifying the timing aspects and accuracy of elasticity. Based on these metrics, we propose a novel approach for benchmarking the elasticity of Infrastructure-as-a-Service (IaaS) cloud platforms independent of the performance exhibited by the provisioned underlying resources. We show that the proposed metrics provide consistent ranking of elastic platforms on an ordinal scale. Finally, we present an extensive case study of real-world complexity demonstrating that the proposed approach is applicable in realistic scenarios and can cope with different levels of resource efficiency.
Modeling and Extracting Load Intensity Profiles. J. von Kistowski; N. R. Herbst; D. Zoller; S. Kounev; A. Hotho; in Proceedings of the 10th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2015) (2015).
[ Abstract ] [ BibTeX ] [ EndNote ]
 
Today's system developers and operators face the challenge of creating software systems that make efficient use of dynamically allocated resources under highly variable and dynamic load profiles, while at the same time delivering reliable performance. Benchmarking of systems under these constraints is difficult, as state-of-the-art benchmarking frameworks provide only limited support for emulating such dynamic and highly variable load profiles for the creation of realistic workload scenarios. Industrial benchmarks typically confine themselves to workloads with constant or stepwise increasing loads. Alternatively, they support replaying of recorded load traces. Statistical load intensity descriptions also do not sufficiently capture concrete pattern load profile variations over time. To address these issues, we present the Descartes Load Intensity Model (DLIM). DLIM provides a modeling formalism for describing load intensity variations over time. A DLIM instance can be used as a compact representation of a recorded load intensity trace, providing a powerful tool for benchmarking and performance analysis. As manually obtaining DLIM instances can be time consuming, we present three different automated extraction methods, which also help to enable autonomous system analysis for self-adaptive systems. Model expressiveness is validated using the presented extraction methods. Extracted DLIM instances exhibit a median modeling error of 12.4% on average over nine different real-world traces covering between two weeks and seven months. Additionally, extraction methods perform orders of magnitude faster than existing time series decomposition approaches.
@inproceedings{KiHeZoKoHo2015_SEAMS_DLIMExtraction, abstract = {Today's system developers and operators face the challenge of creating software systems that make efficient use of dynamically allocated resources under highly variable and dynamic load profiles, while at the same time delivering reliable performance. Benchmarking of systems under these constraints is difficult, as state-of-the-art benchmarking frameworks provide only limited support for emulating such dynamic and highly variable load profiles for the creation of realistic workload scenarios. Industrial benchmarks typically confine themselves to workloads with constant or stepwise increasing loads. Alternatively, they support replaying of recorded load traces. Statistical load intensity descriptions also do not sufficiently capture concrete pattern load profile variations over time. To address these issues, we present the Descartes Load Intensity Model (DLIM). DLIM provides a modeling formalism for describing load intensity variations over time. A DLIM instance can be used as a compact representation of a recorded load intensity trace, providing a powerful tool for benchmarking and performance analysis. As manually obtaining DLIM instances can be time consuming, we present three different automated extraction methods, which also help to enable autonomous system analysis for self-adaptive systems. Model expressiveness is validated using the presented extraction methods. Extracted DLIM instances exhibit a median modeling error of 12.4% on average over nine different real-world traces covering between two weeks and seven months. Additionally, extraction methods perform orders of magnitude faster than existing time series decomposition approaches.}, author = {von Kistowski, J{ó}akim and Herbst, Nikolas Roman and Zoller, Daniel and Kounev, Samuel and Hotho, Andreas}, booktitle = {Proceedings of the 10th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2015)}, keywords = {Metrics_and_benchmarking_methodologies Resource_management Performance t_full descartes Automated_model_learning Elasticity Meta-models Instrumentation_profiling_and_workload_characterization SPEC LIMBO}, month = {may}, note = {Acceptance rate: 29\%}, title = {{Modeling and Extracting Load Intensity Profiles}}, year = 2015 }
%0 Conference Paper %1 KiHeZoKoHo2015_SEAMS_DLIMExtraction %A von Kistowski, Jóakim %A Herbst, Nikolas Roman %A Zoller, Daniel %A Kounev, Samuel %A Hotho, Andreas %B Proceedings of the 10th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2015) %D 2015 %T Modeling and Extracting Load Intensity Profiles %X Today's system developers and operators face the challenge of creating software systems that make efficient use of dynamically allocated resources under highly variable and dynamic load profiles, while at the same time delivering reliable performance. Benchmarking of systems under these constraints is difficult, as state-of-the-art benchmarking frameworks provide only limited support for emulating such dynamic and highly variable load profiles for the creation of realistic workload scenarios. Industrial benchmarks typically confine themselves to workloads with constant or stepwise increasing loads. Alternatively, they support replaying of recorded load traces. Statistical load intensity descriptions also do not sufficiently capture concrete pattern load profile variations over time. To address these issues, we present the Descartes Load Intensity Model (DLIM). DLIM provides a modeling formalism for describing load intensity variations over time. A DLIM instance can be used as a compact representation of a recorded load intensity trace, providing a powerful tool for benchmarking and performance analysis. As manually obtaining DLIM instances can be time consuming, we present three different automated extraction methods, which also help to enable autonomous system analysis for self-adaptive systems. Model expressiveness is validated using the presented extraction methods. Extracted DLIM instances exhibit a median modeling error of 12.4% on average over nine different real-world traces covering between two weeks and seven months. Additionally, extraction methods perform orders of magnitude faster than existing time series decomposition approaches.

2014

Resource Elasticity Benchmarking in Cloud Environments. A. Weber; Thesis; Karlsruhe Institute of Technology (KIT); Am Fasanengarten 5, 76131 Karlsruhe, Germany. (2014, August).
[ Abstract ] [ BibTeX ] [ EndNote ]
 
Auto-scaling features offered by today's cloud infrastructures provide increased flexibility, especially for customers that experience high variations in the load intensity over time. However, auto-scaling features introduce new system quality attributes when considering their accuracy and timing. Therefore, distinguishing between different offerings has become a complex task, as it is not yet supported by reliable metrics and measurement approaches. This thesis discusses the shortcomings of existing approaches for measuring and evaluating elastic behavior and proposes a novel benchmark methodology specifically designed for evaluating the elasticity aspects of modern cloud platforms. The benchmarking concept uses open workloads with realistic load intensity profiles in order to induce resource demand variations on the benchmarked system and compares them with the actual variation of the allocated resources. To ensure a fair elasticity comparison between systems with di erent underlying hardware performance, the load intensity profiles are adjusted to induce identical resource demand variations on all compared platforms. Furthermore, this thesis proposes new metrics that capture the accuracy of resource allocations and deallocations, as well as the timing aspects of an auto-scaling mechanism, explicitly. The benchmark concept comprises four activities: The System Analysis evaluates the load processing capabilities of the benchmarked platform for different scaling stages. The Benchmark Calibration then uses the analysis results and adjusts a given load intensity profile in a system specific manner. Within the Measurement activity, the evaluated platformis exposed to a load varying ccording to the adjusted intensity profile. The final Elasticity Evaluation measures the quality of the observed elastic behavior using the proposed elasticity metrics. A java based framework for benchmarking the elasticity of IaaS cloud platforms called BUNGEE implements this concept and automates benchmarking activities. At the moment, BUNGEE allows to analyze the elasticity of CloudStack and Amazon Web Service (AWS) based clouds that scale CPU-bound virtual machines horizontally. Within an extensive evaluation, this thesis demonstrates the ability of the proposed elasticity metrics to consistently rank elastic systems on an ordinal scale. A case study that uses a realistic load profile, consisting of several millions of request submissions, exhibits the applicability of the benchmarking methodology for realistic scenarios. The case study is conducted on a private as well as on a public cloud and uses eleven different elasticity rule configurations and four instance types assigned to resources with different levels of effciency.
@mastersthesis{Weber2014, abstract = {{Auto-scaling features offered by today's cloud infrastructures provide increased flexibility, especially for customers that experience high variations in the load intensity over time. However, auto-scaling features introduce new system quality attributes when considering their accuracy and timing. Therefore, distinguishing between different offerings has become a complex task, as it is not yet supported by reliable metrics and measurement approaches. This thesis discusses the shortcomings of existing approaches for measuring and evaluating elastic behavior and proposes a novel benchmark methodology specifically designed for evaluating the elasticity aspects of modern cloud platforms. The benchmarking concept uses open workloads with realistic load intensity profiles in order to induce resource demand variations on the benchmarked system and compares them with the actual variation of the allocated resources. To ensure a fair elasticity comparison between systems with di erent underlying hardware performance, the load intensity profiles are adjusted to induce identical resource demand variations on all compared platforms. Furthermore, this thesis proposes new metrics that capture the accuracy of resource allocations and deallocations, as well as the timing aspects of an auto-scaling mechanism, explicitly. The benchmark concept comprises four activities: The System Analysis evaluates the load processing capabilities of the benchmarked platform for different scaling stages. The Benchmark Calibration then uses the analysis results and adjusts a given load intensity profile in a system specific manner. Within the Measurement activity, the evaluated platformis exposed to a load varying ccording to the adjusted intensity profile. The final Elasticity Evaluation measures the quality of the observed elastic behavior using the proposed elasticity metrics. A java based framework for benchmarking the elasticity of IaaS cloud platforms called BUNGEE implements this concept and automates benchmarking activities. At the moment, BUNGEE allows to analyze the elasticity of CloudStack and Amazon Web Service (AWS) based clouds that scale CPU-bound virtual machines horizontally. Within an extensive evaluation, this thesis demonstrates the ability of the proposed elasticity metrics to consistently rank elastic systems on an ordinal scale. A case study that uses a realistic load profile, consisting of several millions of request submissions, exhibits the applicability of the benchmarking methodology for realistic scenarios. The case study is conducted on a private as well as on a public cloud and uses eleven different elasticity rule configurations and four instance types assigned to resources with different levels of effciency.}}, address = {Am Fasanengarten 5, 76131 Karlsruhe, Germany}, author = {Weber, Andreas}, keywords = {Thesis_supervised_by_SE_member Metrics_and_benchmarking_methodologies Performance descartes Elasticity LIMBO Tool se2_mastersthesis Cloud BUNGEE Supervised_by_Nikolas_Herbst Virtualization Self-adaptive-systems}, month = {August}, school = {{Karlsruhe Institute of Technology (KIT)}}, title = {{Resource Elasticity Benchmarking in Cloud Environments}}, type = {{Master Thesis}}, year = 2014 }
%0 Thesis %1 Weber2014 %A Weber, Andreas %C Am Fasanengarten 5, 76131 Karlsruhe, Germany %D 2014 %T Resource Elasticity Benchmarking in Cloud Environments %X Auto-scaling features offered by today's cloud infrastructures provide increased flexibility, especially for customers that experience high variations in the load intensity over time. However, auto-scaling features introduce new system quality attributes when considering their accuracy and timing. Therefore, distinguishing between different offerings has become a complex task, as it is not yet supported by reliable metrics and measurement approaches. This thesis discusses the shortcomings of existing approaches for measuring and evaluating elastic behavior and proposes a novel benchmark methodology specifically designed for evaluating the elasticity aspects of modern cloud platforms. The benchmarking concept uses open workloads with realistic load intensity profiles in order to induce resource demand variations on the benchmarked system and compares them with the actual variation of the allocated resources. To ensure a fair elasticity comparison between systems with di erent underlying hardware performance, the load intensity profiles are adjusted to induce identical resource demand variations on all compared platforms. Furthermore, this thesis proposes new metrics that capture the accuracy of resource allocations and deallocations, as well as the timing aspects of an auto-scaling mechanism, explicitly. The benchmark concept comprises four activities: The System Analysis evaluates the load processing capabilities of the benchmarked platform for different scaling stages. The Benchmark Calibration then uses the analysis results and adjusts a given load intensity profile in a system specific manner. Within the Measurement activity, the evaluated platformis exposed to a load varying ccording to the adjusted intensity profile. The final Elasticity Evaluation measures the quality of the observed elastic behavior using the proposed elasticity metrics. A java based framework for benchmarking the elasticity of IaaS cloud platforms called BUNGEE implements this concept and automates benchmarking activities. At the moment, BUNGEE allows to analyze the elasticity of CloudStack and Amazon Web Service (AWS) based clouds that scale CPU-bound virtual machines horizontally. Within an extensive evaluation, this thesis demonstrates the ability of the proposed elasticity metrics to consistently rank elastic systems on an ordinal scale. A case study that uses a realistic load profile, consisting of several millions of request submissions, exhibits the applicability of the benchmarking methodology for realistic scenarios. The case study is conducted on a private as well as on a public cloud and uses eleven different elasticity rule configurations and four instance types assigned to resources with different levels of effciency.
Towards a Resource Elasticity Benchmark for Cloud Environments. A. Weber; N. R. Herbst; H. Groenda; S. Kounev; in Proceedings of the 2nd International Workshop on Hot Topics in Cloud Service Scalability (HotTopiCS 2014), co-located with the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014) (2014). 5:1–5:8.
[ Abstract ] [ BibTeX ] [ EndNote ] [ URL ]
 
Auto-scaling features offered by today's cloud infrastructures provide increased flexibility especially for customers that experience high variations in the load intensity over time. However, auto-scaling features introduce new system quality attributes when considering their accuracy, timing, and boundaries. Therefore, distinguishing between different offerings has become a complex task, as it is not yet supported by reliable metrics and measurement approaches. In this paper, we discuss shortcomings of existing approaches for measuring and evaluating elastic behavior and propose a novel benchmark methodology specifically designed for evaluating the elasticity aspects of modern cloud platforms. The benchmark is based on open workloads with realistic load variation profiles that are calibrated to induce identical resource demand variations independent of the underlying hardware performance. Furthermore, we propose new metrics that capture the accuracy of resource allocations and de-allocations, as well as the timing aspects of an auto-scaling mechanism explicitly.
@inproceedings{WeHeGrKo2014-HotTopicsWS-ElaBench, abstract = {{Auto-scaling features offered by today's cloud infrastructures provide increased flexibility especially for customers that experience high variations in the load intensity over time. However, auto-scaling features introduce new system quality attributes when considering their accuracy, timing, and boundaries. Therefore, distinguishing between different offerings has become a complex task, as it is not yet supported by reliable metrics and measurement approaches. In this paper, we discuss shortcomings of existing approaches for measuring and evaluating elastic behavior and propose a novel benchmark methodology specifically designed for evaluating the elasticity aspects of modern cloud platforms. The benchmark is based on open workloads with realistic load variation profiles that are calibrated to induce identical resource demand variations independent of the underlying hardware performance. Furthermore, we propose new metrics that capture the accuracy of resource allocations and de-allocations, as well as the timing aspects of an auto-scaling mechanism explicitly.}}, address = {New York, NY, USA}, author = {Weber, Andreas and Herbst, Nikolas Roman and Groenda, Henning and Kounev, Samuel}, booktitle = {Proceedings of the 2nd International Workshop on Hot Topics in Cloud Service Scalability (HotTopiCS 2014), co-located with the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014)}, keywords = {Tool Metrics_and_benchmarking_methodologies Resource_management Performance descartes Cloud Elasticity BUNGEE LIMBO t_workshop}, month = {March}, pages = {5:1&ndash;5:8}, publisher = {ACM}, series = {HotTopiCS '14}, title = {{Towards a Resource Elasticity Benchmark for Cloud Environments}}, year = 2014 }
%0 Conference Paper %1 WeHeGrKo2014-HotTopicsWS-ElaBench %A Weber, Andreas %A Herbst, Nikolas Roman %A Groenda, Henning %A Kounev, Samuel %B Proceedings of the 2nd International Workshop on Hot Topics in Cloud Service Scalability (HotTopiCS 2014), co-located with the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014) %C New York, NY, USA %D 2014 %I ACM %P 5:1--5:8 %T Towards a Resource Elasticity Benchmark for Cloud Environments %U http://doi.acm.org/10.1145/2649563.2649571 %X Auto-scaling features offered by today's cloud infrastructures provide increased flexibility especially for customers that experience high variations in the load intensity over time. However, auto-scaling features introduce new system quality attributes when considering their accuracy, timing, and boundaries. Therefore, distinguishing between different offerings has become a complex task, as it is not yet supported by reliable metrics and measurement approaches. In this paper, we discuss shortcomings of existing approaches for measuring and evaluating elastic behavior and propose a novel benchmark methodology specifically designed for evaluating the elasticity aspects of modern cloud platforms. The benchmark is based on open workloads with realistic load variation profiles that are calibrated to induce identical resource demand variations independent of the underlying hardware performance. Furthermore, we propose new metrics that capture the accuracy of resource allocations and de-allocations, as well as the timing aspects of an auto-scaling mechanism explicitly.
LIMBO - A Load Intensity Modeling Platform. J. von Kistowski; (2014, März).
[ BibTeX ] [ EndNote ]
 
@misc{talk-Kistowski-SPECRG2014, author = {von Kistowski, J{ó}akim}, howpublished = {Talk at SPEC RG Annual Meeting 2014}, keywords = {Metrics_and_benchmarking_methodologies Resource_management Performance t_talk descartes Automated_model_learning Elasticity Meta-models Talks Instrumentation_profiling_and_workload_characterization SPEC LIMBO}, month = {March}, title = {{LIMBO - A Load Intensity Modeling Platform}}, year = 2014 }
%0 Generic %1 talk-Kistowski-SPECRG2014 %A von Kistowski, Jóakim %D 2014 %T LIMBO - A Load Intensity Modeling Platform
Using and Extending LIMBO for the Descriptive Modeling of Arrival Behaviors. J. von Kistowski; N. R. Herbst; S. Kounev; in Proceedings of the Symposium on Software Performance 2014 (2014). 131–140.
[ Abstract ] [ BibTeX ] [ EndNote ]
 
LIMBO is a tool for the creation of load profiles with variable intensity over time both from scratch and from existing data. Primarily, LIMBO's intended use is the description of load arrival behaviours in open workloads. Specifically, LIMBO can be employed for the creation of custom request or user arrival time-stamps or for the re-parameterization of existing traces. LIMBO bases on the Descartes Load Intensity Model (DLIM) for the formalized description of its load intensity profiles. The DLIM formalism can be understood as a structure for piece-wise defined and combined mathematical functions. We outline DLIM and its elements and demonstrate its integration within LIMBO. LIMBO is capable of generating request or user arrival time stamps from DLIM instances. In a next step, these generated time-stamps can be used for both open workload based benchmarking and simulations. The TimestampTimer plug-in for JMeter already allows the former. LIMBO also offers a range of tools for easy load intensity modeling and analysis, including, but not limited to, a visual decomposition of load intensity time-series into seasonal and trend parts, a simplified load intensity model as part of a model creation wizard, and an automated model instance extractor. As part of our LIMBO tutorial, we explain these features in detail. We demonstrate common use cases for LIMBO and show how they are supported. We also focus on LIMBOs extensible architecture and discuss how to use LIMBO as part of another tool-chain.
@inproceedings{KiHeKo2014-SOSP-LIMBO, abstract = {{LIMBO is a tool for the creation of load profiles with variable intensity over time both from scratch and from existing data. Primarily, LIMBO's intended use is the description of load arrival behaviours in open workloads. Specifically, LIMBO can be employed for the creation of custom request or user arrival time-stamps or for the re-parameterization of existing traces. LIMBO bases on the Descartes Load Intensity Model (DLIM) for the formalized description of its load intensity profiles. The DLIM formalism can be understood as a structure for piece-wise defined and combined mathematical functions. We outline DLIM and its elements and demonstrate its integration within LIMBO. LIMBO is capable of generating request or user arrival time stamps from DLIM instances. In a next step, these generated time-stamps can be used for both open workload based benchmarking and simulations. The TimestampTimer plug-in for JMeter already allows the former. LIMBO also offers a range of tools for easy load intensity modeling and analysis, including, but not limited to, a visual decomposition of load intensity time-series into seasonal and trend parts, a simplified load intensity model as part of a model creation wizard, and an automated model instance extractor. As part of our LIMBO tutorial, we explain these features in detail. We demonstrate common use cases for LIMBO and show how they are supported. We also focus on LIMBOs extensible architecture and discuss how to use LIMBO as part of another tool-chain.}}, author = {von Kistowski, J{ó}akim and Herbst, Nikolas Roman and Kounev, Samuel}, booktitle = {Proceedings of the Symposium on Software Performance 2014}, keywords = {Tool Metrics_and_benchmarking_methodologies Resource_management Performance descartes Award Automated_model_learning Meta-models Instrumentation_profiling_and_workload_characterization LIMBO t_poster}, month = {November}, note = {<b>Best Poster Award</b>}, pages = {131&ndash;140}, publisher = {University of Stuttgart, Faculty of Computer Science, Electrical Engineering, and Information Technology}, title = {{Using and Extending LIMBO for the Descriptive Modeling of Arrival Behaviors}}, year = 2014 }
%0 Conference Paper %1 KiHeKo2014-SOSP-LIMBO %A von Kistowski, Jóakim %A Herbst, Nikolas Roman %A Kounev, Samuel %B Proceedings of the Symposium on Software Performance 2014 %D 2014 %I University of Stuttgart, Faculty of Computer Science, Electrical Engineering, and Information Technology %P 131--140 %T Using and Extending LIMBO for the Descriptive Modeling of Arrival Behaviors %X LIMBO is a tool for the creation of load profiles with variable intensity over time both from scratch and from existing data. Primarily, LIMBO's intended use is the description of load arrival behaviours in open workloads. Specifically, LIMBO can be employed for the creation of custom request or user arrival time-stamps or for the re-parameterization of existing traces. LIMBO bases on the Descartes Load Intensity Model (DLIM) for the formalized description of its load intensity profiles. The DLIM formalism can be understood as a structure for piece-wise defined and combined mathematical functions. We outline DLIM and its elements and demonstrate its integration within LIMBO. LIMBO is capable of generating request or user arrival time stamps from DLIM instances. In a next step, these generated time-stamps can be used for both open workload based benchmarking and simulations. The TimestampTimer plug-in for JMeter already allows the former. LIMBO also offers a range of tools for easy load intensity modeling and analysis, including, but not limited to, a visual decomposition of load intensity time-series into seasonal and trend parts, a simplified load intensity model as part of a model creation wizard, and an automated model instance extractor. As part of our LIMBO tutorial, we explain these features in detail. We demonstrate common use cases for LIMBO and show how they are supported. We also focus on LIMBOs extensible architecture and discuss how to use LIMBO as part of another tool-chain.
LIMBO: A Tool For Modeling Variable Load Intensities. J. von Kistowski; N. R. Herbst; S. Kounev; in Proceedings of the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014) (2014). 225–226.
[ Abstract ] [ BibTeX ] [ EndNote ] [ URL ]
 
Modern software systems are expected to deliver reliable performance under highly variable load intensities while at the same time making efficient use of dynamically allocated resources. Conventional benchmarking frameworks provide limited support for emulating such highly variable and dynamic load profiles and workload scenarios. Industrial benchmarks typically use workloads with constant or stepwise increasing load intensity, or they simply replay recorded workload traces. In this paper, we present LIMBO - an Eclipse-based tool for modeling variable load intensity profiles based on the Descartes Load Intensity Model as an underlying modeling formalism.
@inproceedings{KiHeKo2014-ICPEDemo-LIMBO, abstract = {{Modern software systems are expected to deliver reliable performance under highly variable load intensities while at the same time making efficient use of dynamically allocated resources. Conventional benchmarking frameworks provide limited support for emulating such highly variable and dynamic load profiles and workload scenarios. Industrial benchmarks typically use workloads with constant or stepwise increasing load intensity, or they simply replay recorded workload traces. In this paper, we present LIMBO - an Eclipse-based tool for modeling variable load intensity profiles based on the Descartes Load Intensity Model as an underlying modeling formalism.}}, address = {New York, NY, USA}, author = {von Kistowski, J{ó}akim and Herbst, Nikolas Roman and Kounev, Samuel}, booktitle = {Proceedings of the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014)}, keywords = {Tool Metrics_and_benchmarking_methodologies Resource_management Performance descartes t_demo Automated_model_learning Elasticity Meta-models Instrumentation_profiling_and_workload_characterization LIMBO}, month = {March}, pages = {225&ndash;226}, publisher = {ACM}, series = {ICPE '14}, title = {{LIMBO: A Tool For Modeling Variable Load Intensities}}, year = 2014 }
%0 Conference Paper %1 KiHeKo2014-ICPEDemo-LIMBO %A von Kistowski, Jóakim %A Herbst, Nikolas Roman %A Kounev, Samuel %B Proceedings of the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014) %C New York, NY, USA %D 2014 %I ACM %P 225--226 %T LIMBO: A Tool For Modeling Variable Load Intensities %U http://doi.acm.org/10.1145/2568088.2576092 %X Modern software systems are expected to deliver reliable performance under highly variable load intensities while at the same time making efficient use of dynamically allocated resources. Conventional benchmarking frameworks provide limited support for emulating such highly variable and dynamic load profiles and workload scenarios. Industrial benchmarks typically use workloads with constant or stepwise increasing load intensity, or they simply replay recorded workload traces. In this paper, we present LIMBO - an Eclipse-based tool for modeling variable load intensity profiles based on the Descartes Load Intensity Model as an underlying modeling formalism.
Modeling Variations in Load Intensity over Time. J. von Kistowski; N. R. Herbst; S. Kounev; in Proceedings of the 3rd International Workshop on Large-Scale Testing (LT 2014), co-located with the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014) (2014). 1–4.
[ Abstract ] [ BibTeX ] [ EndNote ] [ URL ]
 
Today's software systems are expected to deliver reliable performance under highly variable load intensities while at the same time making efficient use of dynamically allocated resources. Conventional benchmarking frameworks provide limited support for emulating such highly variable and dynamic load profiles and workload scenarios. Industrial benchmarks typically use workloads with constant or stepwise increasing load intensity, or they simply replay recorded workload traces. Based on this observation, we identify the need for means allowing flexible definition of load profiles and address this by introducing two meta-models at different abstraction levels. At the lower abstraction level, the Descartes Load Intensity Meta-Model (DLIM) offers a structured and accessible way of describing the load intensity over time by editing and combining mathematical functions. The High-Level Descartes Load Intensity Meta-Model (HLDLIM) allows the description of load variations using few defined parameters that characterize the seasonal patterns, trends, bursts and noise parts. We demonstrate that both meta-models are capable of capturing real-world load profiles with acceptable accuracy through comparison with a real life trace.
@inproceedings{KiHeKo2014-LT-DLIM, abstract = {{Today's software systems are expected to deliver reliable performance under highly variable load intensities while at the same time making efficient use of dynamically allocated resources. Conventional benchmarking frameworks provide limited support for emulating such highly variable and dynamic load profiles and workload scenarios. Industrial benchmarks typically use workloads with constant or stepwise increasing load intensity, or they simply replay recorded workload traces. Based on this observation, we identify the need for means allowing flexible definition of load profiles and address this by introducing two meta-models at different abstraction levels. At the lower abstraction level, the Descartes Load Intensity Meta-Model (DLIM) offers a structured and accessible way of describing the load intensity over time by editing and combining mathematical functions. The High-Level Descartes Load Intensity Meta-Model (HLDLIM) allows the description of load variations using few defined parameters that characterize the seasonal patterns, trends, bursts and noise parts. We demonstrate that both meta-models are capable of capturing real-world load profiles with acceptable accuracy through comparison with a real life trace.}}, address = {New York, NY, USA}, author = {von Kistowski, J{ó}akim and Herbst, Nikolas Roman and Kounev, Samuel}, booktitle = {Proceedings of the 3rd International Workshop on Large-Scale Testing (LT 2014), co-located with the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014)}, keywords = {Metrics_and_benchmarking_methodologies Resource_management Performance descartes Automated_model_learning Elasticity Meta-models Instrumentation_profiling_and_workload_characterization LIMBO t_workshop}, month = {March}, pages = {1&ndash;4}, publisher = {ACM}, title = {{Modeling Variations in Load Intensity over Time}}, year = 2014 }
%0 Conference Paper %1 KiHeKo2014-LT-DLIM %A von Kistowski, Jóakim %A Herbst, Nikolas Roman %A Kounev, Samuel %B Proceedings of the 3rd International Workshop on Large-Scale Testing (LT 2014), co-located with the 5th ACM/SPEC International Conference on Performance Engineering (ICPE 2014) %C New York, NY, USA %D 2014 %I ACM %P 1--4 %T Modeling Variations in Load Intensity over Time %U http://doi.acm.org/10.1145/2577036.2577037 %X Today's software systems are expected to deliver reliable performance under highly variable load intensities while at the same time making efficient use of dynamically allocated resources. Conventional benchmarking frameworks provide limited support for emulating such highly variable and dynamic load profiles and workload scenarios. Industrial benchmarks typically use workloads with constant or stepwise increasing load intensity, or they simply replay recorded workload traces. Based on this observation, we identify the need for means allowing flexible definition of load profiles and address this by introducing two meta-models at different abstraction levels. At the lower abstraction level, the Descartes Load Intensity Meta-Model (DLIM) offers a structured and accessible way of describing the load intensity over time by editing and combining mathematical functions. The High-Level Descartes Load Intensity Meta-Model (HLDLIM) allows the description of load variations using few defined parameters that characterize the seasonal patterns, trends, bursts and noise parts. We demonstrate that both meta-models are capable of capturing real-world load profiles with acceptable accuracy through comparison with a real life trace.