8 #include "AggMemberDatasetDimensionCache.h" 9 #include "AggMemberDataset.h" 16 #include "BESInternalError.h" 19 #include "TheBESKeys.h" 22 static const string BES_DATA_ROOT(
"BES.Data.RootDirectory");
23 static const string BES_CATALOG_ROOT(
"BES.Catalog.catalog.RootDirectory");
29 AggMemberDatasetDimensionCache *AggMemberDatasetDimensionCache::d_instance = 0;
30 bool AggMemberDatasetDimensionCache::d_enabled =
true;
32 const string AggMemberDatasetDimensionCache::CACHE_DIR_KEY =
"NCML.DimensionCache.directory";
33 const string AggMemberDatasetDimensionCache::PREFIX_KEY =
"NCML.DimensionCache.prefix";
34 const string AggMemberDatasetDimensionCache::SIZE_KEY =
"NCML.DimensionCache.size";
41 unsigned long AggMemberDatasetDimensionCache::getCacheSizeFromConfig(){
45 unsigned long size_in_megabytes = 0;
48 std::istringstream iss(size);
49 iss >> size_in_megabytes;
52 string msg =
"[ERROR] AggMemberDatasetDimensionCache::getCacheSize() - The BES Key " + SIZE_KEY +
" is not set! It MUST be set to utilize the NcML Dimension Cache. ";
53 BESDEBUG(
"cache", msg << endl);
56 return size_in_megabytes;
63 string AggMemberDatasetDimensionCache::getCacheDirFromConfig(){
69 string msg =
"[ERROR] AggMemberDatasetDimensionCache::getSubDirFromConfig() - The BES Key " + CACHE_DIR_KEY +
" is not set! It MUST be set to utilize the NcML Dimension Cache. ";
70 BESDEBUG(
"cache", msg << endl);
82 string AggMemberDatasetDimensionCache::getDimCachePrefixFromConfig(){
90 string msg =
"[ERROR] AggMemberDatasetDimensionCache::getResultPrefix() - The BES Key " + PREFIX_KEY +
" is not set! It MUST be set to utilize the NcML Dimension Cache. ";
91 BESDEBUG(
"cache", msg << endl);
104 string AggMemberDatasetDimensionCache::getBesDataRootDirFromConfig(){
106 string cacheDir =
"";
111 string msg = ((string)
"[ERROR] AggMemberDatasetDimensionCache::getStoredResultsDir() - Neither the BES Key ") + BES_CATALOG_ROOT +
112 "or the BES key " + BES_DATA_ROOT +
" have been set! One MUST be set to utilize the NcML Dimension Cache. ";
113 BESDEBUG(
"cache", msg << endl);
124 AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache()
126 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - BEGIN" << endl);
128 d_dimCacheDir = getCacheDirFromConfig();
129 d_dataRootDir = getBesDataRootDirFromConfig();
131 d_dimCacheFilePrefix = getDimCachePrefixFromConfig();
132 d_maxCacheSize = getCacheSizeFromConfig();
134 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache() - Stored results cache configuration params: " << d_dimCacheDir <<
", " << d_dimCacheFilePrefix <<
", " << d_maxCacheSize << endl);
137 initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
139 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - END" << endl);
146 AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache(
const string &data_root_dir,
const string &cache_dir,
const string &prefix,
unsigned long long size){
148 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - BEGIN" << endl);
150 d_dataRootDir = data_root_dir;
151 d_dimCacheDir = cache_dir;
152 d_dimCacheFilePrefix = prefix;
153 d_maxCacheSize = size;
156 initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
158 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::AggMemberDatasetDimensionCache() - END" << endl);
169 if (d_enabled && d_instance == 0){
170 if (libdap::dir_exists(cache_dir)) {
176 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
177 "Cache is DISABLED"<< endl);
181 atexit(delete_instance);
183 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
184 "Cache is ENABLED"<< endl);
199 if (d_enabled && d_instance == 0) {
205 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
206 "Cache is DISABLED"<< endl);
210 atexit(delete_instance);
212 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::"<<__func__ <<
"() - " <<
213 "Cache is ENABLED"<< endl);
224 void AggMemberDatasetDimensionCache::delete_instance() {
225 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::delete_instance() - Deleting singleton BESStoredDapResultCache instance." << endl);
232 AggMemberDatasetDimensionCache::~AggMemberDatasetDimensionCache()
247 bool AggMemberDatasetDimensionCache::is_valid(
const string &cache_file_name,
const string &local_id)
253 off_t entry_size = 0;
254 time_t entry_time = 0;
256 if (stat(cache_file_name.c_str(), &buf) == 0) {
257 entry_size = buf.st_size;
258 entry_time = buf.st_mtime;
267 time_t dataset_time = entry_time;
268 if (stat(datasetFileName.c_str(), &buf) == 0) {
269 dataset_time = buf.st_mtime;
278 if (dataset_time > entry_time)
293 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - BEGIN" << endl );
297 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - local resource id: "<< local_id << endl );
299 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - cache_file_name: "<< cache_file_name << endl );
307 if (!is_valid(cache_file_name, local_id)){
308 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
313 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - Dimension cache file exists. Loading dimension cache from file: " << cache_file_name << endl);
315 ifstream istrm(cache_file_name.c_str());
317 throw libdap::InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to read cached dimensions.");
337 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - Created and locked cache file: " << cache_file_name << endl);
340 ofstream ostrm(cache_file_name.c_str());
342 throw libdap::InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to write cached response.");
369 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - Couldn't create and lock cache file, But I got a read lock. " 370 "Cache file may have been rebuilt by another process. " 371 "Cache file: " << cache_file_name << endl);
374 throw libdap::InternalErr(__FILE__, __LINE__,
"AggMemberDatasetDimensionCache::loadDimensionCache() - Cache error during function invocation.");
378 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - unlocking and closing cache file "<< cache_file_name << endl );
382 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - caught exception, unlocking cache and re-throw." << endl );
387 BESDEBUG(
"cache",
"AggMemberDatasetDimensionCache::loadDimensionCache() - END (local_id=`"<< local_id <<
"')" << endl );
virtual void saveDimensionCache(std::ostream &ostr)=0
virtual void unlock_cache()
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big.
exception thrown if inernal error encountered
static string lowercase(const string &s)
virtual void update_and_purge(const std::string &new_file)
Purge files from the cache.
virtual unsigned long long update_cache_info(const std::string &target)
Update the cache info file to include 'target'.
virtual void fillDimensionCacheByUsingDDS()=0
virtual void loadDimensionCache(std::istream &istr)=0
bool cache_enabled() const
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
virtual void unlock_and_close(const std::string &target)
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
static TheBESKeys * TheKeys()
static AggMemberDatasetDimensionCache * get_instance()
virtual bool create_and_lock(const std::string &target, int &fd)
Create a file in the cache and lock it for write access.
void initialize(const std::string &cache_dir, const std::string &prefix, unsigned long long size)
Initialize an instance of FileLockingCache.
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
void loadDimensionCache(AggMemberDataset *amd)
const std::string & getLocation() const
virtual bool get_read_lock(const std::string &target, int &fd)
Get a read-only lock on the file if it exists.
static string assemblePath(const string &firstPart, const string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
virtual void purge_file(const std::string &file)
Purge a single file from the cache.