Fawkes API  Fawkes Development Version
lookup_estimator.cpp
1 /***************************************************************************
2  * lookup_estimator.cpp - Estimate skill exec times by random db lookups
3  *
4  * Created: Tue 24 Jan 2020 11:25:31 CET 16:35
5  * Copyright 2020 Tarik Viehmann <viehmann@kbsg.rwth-aachen.de>
6  ****************************************************************************/
7 
8 /* This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU Library General Public License for more details.
17  *
18  * Read the full text in the LICENSE.GPL file in the doc directory.
19  */
20 
21 #include "lookup_estimator.h"
22 
23 #include <aspect/logging.h>
24 #include <config/yaml.h>
25 #include <core/threading/mutex.h>
26 #include <core/threading/mutex_locker.h>
27 
28 #include <bsoncxx/builder/basic/document.hpp>
29 #include <bsoncxx/builder/basic/kvp.hpp>
30 #include <bsoncxx/exception/exception.hpp>
31 #include <chrono>
32 #include <mongocxx/client.hpp>
33 #include <mongocxx/exception/operation_exception.hpp>
34 #include <thread>
35 
36 namespace fawkes {
37 
38 /** @class LookupEstimator
39  * Estimate the execution time of skills by drawing a random sample from a
40  * set of possible values stored in a mongodb database.
41  */
42 
43 /** Constructor.
44  * @param mongo_connection_manager The mongodb manager to connect to a lookup collection
45  * @param config The config to retrieve database related info and the skills to estimates
46  * @param cfg_prefix The config prefix under which the estimator-specific configurations are found
47  * @param logger The logger to inform about client connection status
48  */
50  Configuration * config,
51  const std::string & cfg_prefix,
52  Logger * logger)
53 : ExecutionTimeEstimator(config, cfg_prefix),
54  mongo_connection_manager_(mongo_connection_manager),
55  logger_(logger),
56  fully_match_args_(config_, cfg_prefix_, "fully-match-args", true),
57  include_failures_(config, cfg_prefix_, "include-failures", false),
58  instance_(
59  config_->get_string_or_default((std::string(cfg_prefix_) + "/instance").c_str(), "default")),
60  database_(
61  config_->get_string_or_default((std::string(cfg_prefix_) + "/database").c_str(), "skills")),
62  collection_(config_->get_string_or_default((std::string(cfg_prefix_) + "/collection").c_str(),
63  "exec_times"))
64 {
65  mongodb_client_lookup_ = mongo_connection_manager_->create_client(instance_);
66  logger_->log_info(logger_name_,
67  "Using instance %s, database %s, collection %s",
68  instance_.c_str(),
69  database_.c_str(),
70  collection_.c_str());
71 }
72 
73 bool
75 {
76  // lock as mongocxx::client is not thread-safe
77  MutexLocker lock(mutex_);
78  // if all skills should be looked up by default, then the skills_ contain
79  // those skills that should not be estimated via lookup
80  try {
81  using bsoncxx::builder::basic::document;
82  using bsoncxx::builder::basic::kvp;
83 
84  document query = get_skill_query(skill);
85  query.append(kvp("outcome", static_cast<int>(SkillerInterface::SkillStatusEnum::S_FINAL)));
86  bsoncxx::stdx::optional<bsoncxx::document::value> found_entry =
87  mongodb_client_lookup_->database(database_)[collection_].find_one(query.view());
88  return found_entry.has_value();
89  } catch (mongocxx::operation_exception &e) {
90  std::string error =
91  std::string("Error trying to lookup " + skill.skill_name + "\n Exception: " + e.what());
92  logger_->log_error(logger_name_, "%s", error.c_str());
93  return false;
94  }
95 }
96 
97 float
99 {
100  using bsoncxx::builder::basic::document;
101  using bsoncxx::builder::basic::kvp;
102  // pipeline to pick a random sample out of all documents with matching name
103  // field
104  document query = get_skill_query(skill);
105  if (!get_property(include_failures_)) {
106  query.append(kvp("outcome", static_cast<int>(SkillerInterface::SkillStatusEnum::S_FINAL)));
107  }
108  mongocxx::pipeline p{};
109  p.match(query.view());
110  p.sample(1);
111 
112  // default values in case lookup fails
113  error_ = "";
114  outcome_ = SkillerInterface::SkillStatusEnum::S_FINAL;
115 
116  // lock as mongocxx::client is not thread-safe
117  MutexLocker lock(mutex_);
118  try {
119  if (get_property(include_failures_)) {
120  query.append(kvp("outcome", (int)SkillerInterface::SkillStatusEnum::S_FINAL));
121  }
122  mongocxx::cursor sample_cursor =
123  mongodb_client_lookup_->database(database_)[collection_].aggregate(p);
124  auto doc = *(sample_cursor.begin());
125  float res = 0.f;
126  switch (doc[duration_field_].get_value().type()) {
127  case bsoncxx::type::k_double:
128  res = static_cast<float>(doc[duration_field_].get_double().value);
129  break;
130  case bsoncxx::type::k_int32:
131  res = static_cast<float>(doc[duration_field_].get_int32().value);
132  break;
133  default:
134  throw fawkes::Exception(("Unexpected type "
135  + bsoncxx::to_string(doc[duration_field_].get_value().type())
136  + " when looking up skill exec duration.")
137  .c_str());
138  }
139  error_ = doc["error"].get_utf8().value.to_string();
140  outcome_ = SkillerInterface::SkillStatusEnum(doc["outcome"].get_int32().value);
141  return res / speed_;
142  } catch (mongocxx::operation_exception &e) {
143  std::string error =
144  std::string("Error for lookup of " + skill.skill_name + "\n Exception: " + e.what());
145  logger_->log_error(logger_name_, "%s", error.c_str());
146  throw;
147  }
148 }
149 
150 std::pair<SkillerInterface::SkillStatusEnum, std::string>
152 {
153  return make_pair(outcome_, error_);
154 }
155 
156 bsoncxx::builder::basic::document
157 LookupEstimator::get_skill_query(const Skill &skill) const
158 {
159  using bsoncxx::builder::basic::document;
160  using bsoncxx::builder::basic::kvp;
161  // pipeline to pick a random sample out of all documents with matching name
162  // field
163  document query = document();
164  query.append(kvp(skill_name_field_, skill.skill_name));
165  if (get_property(fully_match_args_)) {
166  for (const auto &skill_arg : skill.skill_args) {
167  query.append((kvp("$or", kvp("args." + skill_arg.first, skill_arg.second)),
168  kvp("args." + skill_arg.first, ".*")));
169  }
170  } else if (active_whitelist_entry_ != whitelist_.end()) {
171  for (const auto &skill_arg : active_whitelist_entry_->second.skill_args) {
172  query.append((kvp("$or", kvp("args." + skill_arg.first, skill_arg.second)),
173  kvp("args." + skill_arg.first, ".*")));
174  }
175  }
176  return query;
177 }
178 
179 } // namespace fawkes
Interface for configuration handling.
Definition: config.h:68
Base class for exceptions in Fawkes.
Definition: exception.h:36
A structured representation of a skill.
std::string skill_name
The name of the skill.
std::unordered_map< std::string, std::string > skill_args
A map of the skill's argument keys to argument values.
An abstract estimator for the execution time of a skill.
std::map< std::string, Skill >::const_iterator active_whitelist_entry_
Points to the whitelist entry that matches the skill to execute.
const std::map< std::string, Skill > whitelist_
Whitelist of skills that the estimator is allowed to process.
T get_property(const Property< T > &property) const
Get the current property value for active_whitelist_entry_.
const float speed_
Config estimator-specific speedup factor.
Interface for logging.
Definition: logger.h:42
virtual void log_error(const char *component, const char *format,...)=0
Log error message.
virtual void log_info(const char *component, const char *format,...)=0
Log informational message.
LookupEstimator(MongoDBConnCreator *mongo_connection_manager, Configuration *config, const std::string &cfg_prefix, Logger *logger)
Constructor.
float get_execution_time(const Skill &skill) override
Get the estimated execution time for the given skill string.
bool can_provide_exec_time(const Skill &skill) const override
Check if this estimator can give an estimate for a given skill.
std::pair< SkillerInterface::SkillStatusEnum, std::string > execute(const Skill &skill) override
Let the estimator know that we are executing this skill, so it can apply possible side effects.
Interface for a MongoDB connection creator.
virtual mongocxx::client * create_client(const std::string &config_name="")=0
Create a new MongoDB client.
Mutex locking helper.
Definition: mutex_locker.h:34
SkillStatusEnum
This determines the current status of skill execution.
Fawkes library namespace.