Unity Scopes API
Benchmark.h
1 /*
2  * Copyright (C) 2013 Canonical Ltd
3  *
4  * This program is free software: you can redistribute it and/or modify
5  * it under the terms of the GNU Lesser General Public License version 3 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
19 #pragma once
20 
21 #include <unity/scopes/ActionMetadata.h>
22 #include <unity/scopes/CannedQuery.h>
23 #include <unity/scopes/Result.h>
24 #include <unity/scopes/SearchMetadata.h>
25 
26 #include <unity/scopes/testing/Statistics.h>
27 
28 #include <chrono>
29 #include <functional>
30 #include <iosfwd>
31 #include <memory>
32 
33 namespace unity
34 {
35 
36 namespace scopes
37 {
38 class ScopeBase;
39 
40 namespace testing
41 {
42 
47 class Benchmark
48 {
49 public:
54  struct Result
55  {
57  std::size_t sample_size{0};
59  struct Timing : public Sample
60  {
62  typedef std::chrono::duration<double> Seconds;
63 
65  Timing() = default;
66  Timing(const Timing&) = default;
67  Timing(Timing&&) = default;
71  Sample::SizeType get_size() const;
77  void enumerate(const Sample::Enumerator& enumerator) const;
78 
87  bool is_significantly_faster_than_reference(const Timing& reference, double alpha = 0.05) const;
88 
99  double mean,
100  double std_dev,
101  double alpha = 0.05) const;
102 
111  bool is_significantly_slower_than_reference(const Timing& reference, double alpha = 0.05) const;
112 
123  double mean,
124  double std_dev,
125  double alpha = 0.05) const;
126 
128  Seconds min{Seconds::min()};
130  Seconds max{Seconds::min()};
132  Seconds mean{Seconds::min()};
134  Seconds std_dev{Seconds::min()};
136  Seconds kurtosis{Seconds::min()};
138  Seconds skewness{Seconds::min()};
140  std::vector<std::pair<Seconds, double>> histogram{};
142  std::vector<Seconds> sample{};
143  } timing{};
144 
150  void load_from(std::istream& in);
151 
157  void save_to(std::ostream& out);
158 
164  void load_from_xml(std::istream& in);
165 
171  void save_to_xml(std::ostream& out);
172  };
173 
179  {
181  std::size_t histogram_bin_count{10};
182  };
183 
189  {
191  std::size_t trial_count{25};
193  std::chrono::microseconds per_trial_timeout{std::chrono::seconds{10}};
196  };
197 
203  {
205  typedef std::function<
206  std::pair<
210 
218  };
219 
225  {
227  typedef std::function<
228  std::pair<
231  >()
233 
241  };
242 
248  {
250  typedef std::function<
251  std::pair<
255 
263  };
264 
270  {
272  typedef std::function<
273  std::tuple<
276  std::string,
277  std::string
278  >()
280 
288  };
289 
291  virtual ~Benchmark() = default;
292  Benchmark(const Benchmark&) = delete;
293  Benchmark(Benchmark&&) = delete;
294 
295  Benchmark& operator=(const Benchmark&) = delete;
296  Benchmark& operator=(Benchmark&&) = delete;
307  virtual Result for_query(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
308  QueryConfiguration configuration) = 0;
309 
318  virtual Result for_preview(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
319  PreviewConfiguration configuration) = 0;
320 
329  virtual Result for_activation(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
330  ActivationConfiguration configuration) = 0;
331 
340  virtual Result for_action(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
341  ActionConfiguration configuration) = 0;
342 
343 protected:
344  Benchmark() = default;
345 };
346 
347 bool operator==(const Benchmark::Result& lhs, const Benchmark::Result& rhs);
348 
349 std::ostream& operator<<(std::ostream&, const Benchmark::Result&);
350 
351 } // namespace testing
352 
353 } // namespace scopes
354 
355 } // namespace unity
The ActionConfiguration struct constains all options controlling the benchmark of scope action activa...
Definition: Benchmark.h:269
std::function< std::pair< unity::scopes::Result, unity::scopes::ActionMetadata >) > Sampler
Definition: Benchmark.h:232
Seconds min
Definition: Benchmark.h:128
bool is_significantly_slower_than_reference(const Timing &reference, double alpha=0.05) const
Checks if a timing sample is statistically significantly slower than a reference timing sample...
Definition: Benchmark.cpp:216
bool is_significantly_faster_than_reference(const Timing &reference, double alpha=0.05) const
Checks if a timing sample is statistically significantly faster than a reference timing sample...
Definition: Benchmark.cpp:170
StatisticsConfiguration statistics_configuration
Definition: Benchmark.h:195
std::function< std::tuple< unity::scopes::Result, unity::scopes::ActionMetadata, std::string, std::string >) > Sampler
Definition: Benchmark.h:279
virtual Result for_action(const std::shared_ptr< unity::scopes::ScopeBase > &scope, ActionConfiguration configuration)=0
for_preview executes a benchmark to measure the scope's action activation performance.
The StatisticsConfiguration struct contains options controlling the calculation of benchmark result s...
Definition: Benchmark.h:178
void save_to_xml(std::ostream &out)
save_to_xml stores a result as xml to the given output stream.
Definition: Benchmark.cpp:113
Metadata passed with search requests.
Definition: SearchMetadata.h:45
TrialConfiguration trial_configuration
Definition: Benchmark.h:262
unity::scopes::testing::Benchmark::Result::Timing timing
Runtime-specific sample data.
virtual Result for_activation(const std::shared_ptr< unity::scopes::ScopeBase > &scope, ActivationConfiguration configuration)=0
for_preview executes a benchmark to measure the scope's activation performance.
std::function< void(ValueType)> Enumerator
Definition: Statistics.h:44
Metadata passed to scopes for preview and activation.
Definition: ActionMetadata.h:41
The attributes of a result returned by a Scope.
Definition: Result.h:50
The Sample class models the interface to a sample of raw observations and their statistical propertie...
Definition: Statistics.h:36
Seconds skewness
Definition: Benchmark.h:138
Definition: OnlineAccountClient.h:39
The PreviewConfiguration struct constains all options controlling the benchmark of scope preview oper...
Definition: Benchmark.h:224
TrialConfiguration trial_configuration
Definition: Benchmark.h:287
Seconds kurtosis
Definition: Benchmark.h:136
The ActivationConfiguration struct constains all options controlling the benchmark of scope activatio...
Definition: Benchmark.h:247
TrialConfiguration trial_configuration
Definition: Benchmark.h:217
void enumerate(const Sample::Enumerator &enumerator) const
Definition: Benchmark.cpp:164
std::size_t sample_size
Definition: Benchmark.h:57
std::function< std::pair< unity::scopes::Result, unity::scopes::ActionMetadata >) > Sampler
Definition: Benchmark.h:254
TrialConfiguration trial_configuration
Definition: Benchmark.h:240
Top-level namespace for all things Unity-related.
Definition: Version.h:49
void save_to(std::ostream &out)
save_to stores a result to the given output stream.
Definition: Benchmark.cpp:137
std::chrono::duration< double > Seconds
Definition: Benchmark.h:62
The Result struct encapsulates all of the result gathered from one individual benchmark run consistin...
Definition: Benchmark.h:54
std::vector< Seconds > sample
Definition: Benchmark.h:142
std::size_t SizeType
Definition: Statistics.h:40
std::function< std::pair< unity::scopes::CannedQuery, unity::scopes::SearchMetadata >) > Sampler
Definition: Benchmark.h:209
Sampler sampler
Definition: Benchmark.h:215
std::size_t histogram_bin_count
Definition: Benchmark.h:181
Seconds std_dev
Definition: Benchmark.h:134
std::size_t trial_count
Definition: Benchmark.h:191
virtual Result for_preview(const std::shared_ptr< unity::scopes::ScopeBase > &scope, PreviewConfiguration configuration)=0
for_preview executes a benchmark to measure the scope's preview performance.
Sampler sampler
Definition: Benchmark.h:285
Parameters of a search query.
Definition: CannedQuery.h:49
Sample::ValueType get_variance() const
Definition: Benchmark.cpp:159
std::vector< std::pair< Seconds, double > > histogram
Definition: Benchmark.h:140
Sample::ValueType get_mean() const
Definition: Benchmark.cpp:154
Seconds max
Definition: Benchmark.h:130
virtual Result for_query(const std::shared_ptr< unity::scopes::ScopeBase > &scope, QueryConfiguration configuration)=0
for_query executes a benchmark to measure the scope's query performance.
The QueryConfiguration struct constains all options controlling the benchmark of scope query operatio...
Definition: Benchmark.h:202
The TrialConfiguration struct contains options controlling the execution of individual trials...
Definition: Benchmark.h:188
Sample::SizeType get_size() const
Definition: Benchmark.cpp:149
The Benchmark class defines an interface to provide scope authors with runtime benchmarking capabilit...
Definition: Benchmark.h:47
Seconds mean
Definition: Benchmark.h:132
void load_from(std::istream &in)
load_from restores a result from the given input stream.
Definition: Benchmark.cpp:125
std::chrono::microseconds per_trial_timeout
Definition: Benchmark.h:193
void load_from_xml(std::istream &in)
load_from_xml restores a result stored as xml from the given input stream.
Definition: Benchmark.cpp:101
double ValueType
Definition: Statistics.h:42