xrootd
XrdFileCacheInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDFILECACHE_INFO_HH__
2 #define __XRDFILECACHE_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <stdio.h>
22 #include <time.h>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 class XrdOssDF;
31 class XrdCksCalc;
32 class XrdSysTrace;
33 
34 
35 namespace XrdCl
36 {
37 class Log;
38 }
39 
40 namespace XrdFileCache
41 {
42 class Stats;
43 
44 //----------------------------------------------------------------------------
46 //----------------------------------------------------------------------------
47 
48 class Info
49 {
50 public:
51  // !Access statistics
52  struct AStat
53  {
54  time_t AttachTime;
55  time_t DetachTime;
56  long long BytesDisk;
57  long long BytesRam;
58  long long BytesMissed;
59 
61  };
62 
63  struct Store {
64  int m_version;
65  long long m_bufferSize;
66  long long m_fileSize;
67  unsigned char *m_buff_synced;
68  char m_cksum[16];
69  time_t m_creationTime;
70  size_t m_accessCnt;
71  std::vector<AStat> m_astats;
72 
74  };
75 
76 
77  //------------------------------------------------------------------------
79  //------------------------------------------------------------------------
80  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
81 
82  //------------------------------------------------------------------------
84  //------------------------------------------------------------------------
85  ~Info();
86 
87  //---------------------------------------------------------------------
89  //---------------------------------------------------------------------
90  void SetBitWritten(int i);
91 
92  //---------------------------------------------------------------------
94  //---------------------------------------------------------------------
95  bool TestBitWritten(int i) const;
96 
97  //---------------------------------------------------------------------
99  //---------------------------------------------------------------------
100  bool TestBitPrefetch(int i) const;
101 
102  //---------------------------------------------------------------------
104  //---------------------------------------------------------------------
105  void SetBitPrefetch(int i);
106 
107  //---------------------------------------------------------------------
109  //---------------------------------------------------------------------
110  void SetBitSynced(int i);
111 
112  //---------------------------------------------------------------------
114  //---------------------------------------------------------------------
116 
117  void SetBufferSize(long long);
118 
119  void SetFileSize(long long);
120 
121  //---------------------------------------------------------------------
125  //---------------------------------------------------------------------
126  void ResizeBits(int n);
127 
128  //---------------------------------------------------------------------
135  //---------------------------------------------------------------------
136  bool Read(XrdOssDF* fp, const std::string &fname = "<unknown>");
137 
138  //---------------------------------------------------------------------
141  //---------------------------------------------------------------------
142  bool Write(XrdOssDF* fp, const std::string &fname = "<unknown>");
143 
144  //---------------------------------------------------------------------
146  //---------------------------------------------------------------------
148 
149  //---------------------------------------------------------------------
151  //---------------------------------------------------------------------
153 
154  //---------------------------------------------------------------------
156  //---------------------------------------------------------------------
158 
160  //---------------------------------------------------------------------
161  void WriteIOStat(Stats& s);
162 
163  //---------------------------------------------------------------------
165  //---------------------------------------------------------------------
167 
168  //---------------------------------------------------------------------
170  //---------------------------------------------------------------------
171  void WriteIOStatSingle(long long bytes_disk);
172 
173  //---------------------------------------------------------------------
175  //---------------------------------------------------------------------
176  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
177 
178  //---------------------------------------------------------------------
180  //---------------------------------------------------------------------
181  bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const;
182 
183  //---------------------------------------------------------------------
185  //---------------------------------------------------------------------
186  int GetSizeInBytes() const;
187 
188  //---------------------------------------------------------------------
190  //---------------------------------------------------------------------
191  int GetSizeInBits() const;
192 
193  //---------------------------------------------------------------------
195  //---------------------------------------------------------------------
196  long long GetFileSize() const;
197 
198  //---------------------------------------------------------------------
200  //---------------------------------------------------------------------
201  bool GetLatestDetachTime(time_t& t) const;
202 
203  //---------------------------------------------------------------------
205  //---------------------------------------------------------------------
206  long long GetBufferSize() const;
207 
208  //---------------------------------------------------------------------
210  //---------------------------------------------------------------------
211  bool IsComplete() const;
212 
213  //---------------------------------------------------------------------
215  //---------------------------------------------------------------------
216  int GetNDownloadedBlocks() const;
217 
218  //---------------------------------------------------------------------
220  //---------------------------------------------------------------------
221  long long GetNDownloadedBytes() const;
222 
223  //---------------------------------------------------------------------
225  //---------------------------------------------------------------------
226  int GetLastDownloadedBlock() const;
227 
228  //---------------------------------------------------------------------
230  //---------------------------------------------------------------------
231  long long GetExpectedDataFileSize() const;
232 
233  //---------------------------------------------------------------------
235  //---------------------------------------------------------------------
237 
238  //---------------------------------------------------------------------
240  //---------------------------------------------------------------------
241  size_t GetAccessCnt() { return m_store.m_accessCnt; }
242 
243  //---------------------------------------------------------------------
245  //---------------------------------------------------------------------
246  int GetVersion() { return m_store.m_version; }
247 
248  //---------------------------------------------------------------------
250  //---------------------------------------------------------------------
251  const Store& RefStoredData() const { return m_store; }
252 
253  //---------------------------------------------------------------------
255  //---------------------------------------------------------------------
256  void GetCksum( unsigned char* buff, char* digest);
257 
258  const static char* m_infoExtension;
259  const static char* m_traceID;
260  const static int m_defaultVersion;
261  const static size_t m_maxNumAccess;
262 
263  XrdSysTrace* GetTrace() const {return m_trace; }
264 
265  static size_t GetMaxNumAccess() { return m_maxNumAccess; }
266 
267 protected:
269 
272  unsigned char *m_buff_written;
273  unsigned char *m_buff_prefetch;
274 
276  bool m_complete;
277 
278 private:
279  inline unsigned char cfiBIT(int n) const { return 1 << n; }
280 
281  // split reading for V1
282  bool ReadV1(XrdOssDF* fp, const std::string &fname);
284 };
285 
286 //------------------------------------------------------------------------------
287 
288 inline bool Info::TestBitWritten(int i) const
289 {
290  const int cn = i/8;
291  assert(cn < GetSizeInBytes());
292 
293  const int off = i - cn*8;
294  return (m_buff_written[cn] & cfiBIT(off)) != 0;
295 }
296 
297 inline void Info::SetBitWritten(int i)
298 {
299  const int cn = i/8;
300  assert(cn < GetSizeInBytes());
301 
302  const int off = i - cn*8;
303  m_buff_written[cn] |= cfiBIT(off);
304 }
305 
306 inline void Info::SetBitPrefetch(int i)
307 {
308  if (!m_buff_prefetch) return;
309 
310  const int cn = i/8;
311  assert(cn < GetSizeInBytes());
312 
313  const int off = i - cn*8;
314  m_buff_prefetch[cn] |= cfiBIT(off);
315 }
316 
317 inline bool Info::TestBitPrefetch(int i) const
318 {
319  if (!m_buff_prefetch) return false;
320 
321  const int cn = i/8;
322  assert(cn < GetSizeInBytes());
323 
324  const int off = i - cn*8;
325  return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
326 }
327 
328 inline void Info::SetBitSynced(int i)
329 {
330  const int cn = i/8;
331  assert(cn < GetSizeInBytes());
332 
333  const int off = i - cn*8;
334  m_store.m_buff_synced[cn] |= cfiBIT(off);
335 }
336 
337 //------------------------------------------------------------------------------
338 
339 inline int Info::GetNDownloadedBlocks() const
340 {
341  int cntd = 0;
342  for (int i = 0; i < m_sizeInBits; ++i)
343  if (TestBitWritten(i)) cntd++;
344 
345  return cntd;
346 }
347 
348 inline long long Info::GetNDownloadedBytes() const
349 {
351 }
352 
354 {
355  for (int i = m_sizeInBits - 1; i >= 0; --i)
356  if (TestBitWritten(i)) return i;
357 
358  return -1;
359 }
360 
361 inline long long Info::GetExpectedDataFileSize() const
362 {
363  int last_block = GetLastDownloadedBlock();
364  if (last_block == m_sizeInBits - 1)
365  return m_store.m_fileSize;
366  else
367  return (last_block + 1) * m_store.m_bufferSize;
368 }
369 
370 inline int Info::GetSizeInBytes() const
371 {
372  if (m_sizeInBits)
373  return ((m_sizeInBits - 1)/8 + 1);
374  else
375  return 0;
376 }
377 
378 inline int Info::GetSizeInBits() const
379 {
380  return m_sizeInBits;
381 }
382 
383 inline long long Info::GetFileSize() const
384 {
385  return m_store.m_fileSize;
386 }
387 
388 inline bool Info::IsComplete() const
389 {
390  return m_complete;
391 }
392 
393 inline bool Info::IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
394 {
395  // TODO rewrite to use full byte comparisons outside of edges ?
396  // Also, it is always called with fisrtsdx = 0, lastIdx = m_sizeInBits.
397  for (int i = firstIdx; i < lastIdx; ++i)
398  if (! TestBitWritten(i)) return true;
399 
400  return false;
401 }
402 
404 {
406 }
407 
408 inline long long Info::GetBufferSize() const
409 {
410  return m_store.m_bufferSize;
411 }
412 
413 }
414 #endif
XrdFileCache::Info::m_sizeInBits
int m_sizeInBits
cached
Definition: XrdFileCacheInfo.hh:275
XrdClConstants.hh
XrdFileCache::Info::TestBitPrefetch
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
Definition: XrdFileCacheInfo.hh:317
XrdFileCache::Info::IsAnythingEmptyInRng
bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdFileCacheInfo.hh:393
XrdFileCache::Info::WriteIOStatDetach
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
XrdFileCache::Info::GetLatestDetachTime
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
XrdFileCache::Info::m_maxNumAccess
static const size_t m_maxNumAccess
Definition: XrdFileCacheInfo.hh:261
XrdFileCache::Info
Status of cached file. Can be read from and written into a binary file.
Definition: XrdFileCacheInfo.hh:49
XrdFileCache::Info::m_complete
bool m_complete
cached
Definition: XrdFileCacheInfo.hh:276
XrdFileCache::Info::m_store
Store m_store
Definition: XrdFileCacheInfo.hh:270
XrdFileCache::Info::Read
bool Read(XrdOssDF *fp, const std::string &fname="<unknown>")
Rea load content from cinfo file into this object.
XrdFileCache::Info::m_buff_prefetch
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdFileCacheInfo.hh:273
XrdSysPthread.hh
XrdFileCache::Info::AStat::BytesMissed
long long BytesMissed
read from ram
Definition: XrdFileCacheInfo.hh:58
XrdFileCache::Info::m_cksCalc
XrdCksCalc * m_cksCalc
Definition: XrdFileCacheInfo.hh:283
XrdFileCache::Info::DisableDownloadStatus
void DisableDownloadStatus()
Disable allocating, writing, and reading of downlaod status.
XrdFileCache::Info::GetVersion
int GetVersion()
Get version.
Definition: XrdFileCacheInfo.hh:246
XrdOssDF
Definition: XrdOss.hh:60
XrdFileCache::Stats
Statistics of disk cache utilisation.
Definition: XrdFileCacheStats.hh:31
XrdFileCache::Info::m_trace
XrdSysTrace * m_trace
Definition: XrdFileCacheInfo.hh:268
XrdFileCache::Info::m_defaultVersion
static const int m_defaultVersion
Definition: XrdFileCacheInfo.hh:260
XrdFileCache::Info::SetBufferSize
void SetBufferSize(long long)
XrdFileCache::Info::AStat
Definition: XrdFileCacheInfo.hh:53
XrdFileCache::Info::GetNDownloadedBlocks
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdFileCacheInfo.hh:339
XrdFileCache::Info::Store::m_version
int m_version
info version
Definition: XrdFileCacheInfo.hh:64
XrdFileCache::Info::cfiBIT
unsigned char cfiBIT(int n) const
Definition: XrdFileCacheInfo.hh:279
XrdFileCache::Info::WriteIOStat
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
XrdFileCache::Info::TestBitWritten
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
Definition: XrdFileCacheInfo.hh:288
XrdFileCache::Info::GetCksum
void GetCksum(unsigned char *buff, char *digest)
Get md5 cksum.
XrdFileCache::Info::m_infoExtension
static const char * m_infoExtension
Definition: XrdFileCacheInfo.hh:258
XrdFileCache
Definition: XrdFileCache.hh:40
XrdFileCache::Info::AStat::DetachTime
time_t DetachTime
open time
Definition: XrdFileCacheInfo.hh:55
XrdFileCache::Info::IsComplete
bool IsComplete() const
Get complete status.
Definition: XrdFileCacheInfo.hh:388
XrdFileCache::Info::GetFileSize
long long GetFileSize() const
Get file size.
Definition: XrdFileCacheInfo.hh:383
XrdFileCache::Info::SetBitPrefetch
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
Definition: XrdFileCacheInfo.hh:306
XrdFileCache::Info::UpdateDownloadCompleteStatus
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdFileCacheInfo.hh:403
XrdFileCache::Info::Store::m_creationTime
time_t m_creationTime
time the info file was created
Definition: XrdFileCacheInfo.hh:69
XrdFileCache::Info::SetBitWritten
void SetBitWritten(int i)
Mark block as written to disk.
Definition: XrdFileCacheInfo.hh:297
XrdFileCache::Info::Store::m_cksum
char m_cksum[16]
cksum of downloaded information
Definition: XrdFileCacheInfo.hh:68
XrdFileCache::Info::WriteIOStatSingle
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
XrdFileCache::Info::RefStoredData
const Store & RefStoredData() const
Get stored data.
Definition: XrdFileCacheInfo.hh:251
XrdFileCache::Info::AStat::AStat
AStat()
read remote client
Definition: XrdFileCacheInfo.hh:60
XrdFileCache::Info::Store::Store
Store()
Definition: XrdFileCacheInfo.hh:73
XrdFileCache::Info::GetTrace
XrdSysTrace * GetTrace() const
Definition: XrdFileCacheInfo.hh:263
XrdFileCache::Info::~Info
~Info()
Destructor.
XrdFileCache::Info::GetLastDownloadedBlock
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdFileCacheInfo.hh:353
XrdFileCache::Info::Store::m_accessCnt
size_t m_accessCnt
number of written AStat structs
Definition: XrdFileCacheInfo.hh:70
XrdFileCache::Info::GetNDownloadedBytes
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdFileCacheInfo.hh:348
XrdFileCache::Info::ResetAllAccessStats
void ResetAllAccessStats()
Reset IO Stats.
XrdCl
Definition: XrdClAnyObject.hh:26
XrdFileCache::Info::GetBufferSize
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdFileCacheInfo.hh:408
XrdClDefaultEnv.hh
XrdFileCache::Info::Store::m_fileSize
long long m_fileSize
number of file blocks
Definition: XrdFileCacheInfo.hh:66
XrdFileCache::Info::SetAllBitsSynced
void SetAllBitsSynced()
Mark all blocks as synced to disk.
XrdFileCache::Info::m_hasPrefetchBuffer
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdFileCacheInfo.hh:271
XrdFileCache::Info::GetExpectedDataFileSize
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdFileCacheInfo.hh:361
XrdFileCache::Info::ResizeBits
void ResizeBits(int n)
Reserve buffer for fileSize/bufferSize bytes.
XrdFileCache::Info::Info
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
XrdFileCache::Info::m_traceID
static const char * m_traceID
Definition: XrdFileCacheInfo.hh:259
XrdFileCache::Info::GetMaxNumAccess
static size_t GetMaxNumAccess()
Definition: XrdFileCacheInfo.hh:265
XrdFileCache::Info::Write
bool Write(XrdOssDF *fp, const std::string &fname="<unknown>")
XrdFileCache::Info::GetSizeInBits
int GetSizeInBits() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdFileCacheInfo.hh:378
XrdFileCache::Info::m_buff_written
unsigned char * m_buff_written
download state vector
Definition: XrdFileCacheInfo.hh:272
XrdFileCache::Info::Store::m_astats
std::vector< AStat > m_astats
number of last m_maxAcessCnts
Definition: XrdFileCacheInfo.hh:71
XrdSysTrace
Definition: XrdSysTrace.hh:49
XrdFileCache::Info::ReadV1
bool ReadV1(XrdOssDF *fp, const std::string &fname)
XrdFileCache::Info::Store::m_bufferSize
long long m_bufferSize
prefetch buffer size
Definition: XrdFileCacheInfo.hh:65
XrdFileCache::Info::SetBitSynced
void SetBitSynced(int i)
Mark block as synced to disk.
Definition: XrdFileCacheInfo.hh:328
XrdFileCache::Info::AStat::BytesDisk
long long BytesDisk
close time
Definition: XrdFileCacheInfo.hh:56
XrdFileCache::Info::GetAccessCnt
size_t GetAccessCnt()
Get number of accesses.
Definition: XrdFileCacheInfo.hh:241
XrdFileCache::Info::AStat::BytesRam
long long BytesRam
read from disk
Definition: XrdFileCacheInfo.hh:57
XrdCksCalc
Definition: XrdCksCalc.hh:40
XrdFileCache::Info::WriteIOStatAttach
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
XrdFileCache::Info::AStat::AttachTime
time_t AttachTime
Definition: XrdFileCacheInfo.hh:54
XrdFileCache::Info::SetFileSize
void SetFileSize(long long)
XrdFileCache::Info::GetSizeInBytes
int GetSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdFileCacheInfo.hh:370
XrdFileCache::Info::Store
Definition: XrdFileCacheInfo.hh:63
XrdFileCache::Info::Store::m_buff_synced
unsigned char * m_buff_synced
disk written state vector
Definition: XrdFileCacheInfo.hh:67
XrdFileCache::Info::WriteIOStatSingle
void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc)
Write open/close with given time and bytes read from disk.