MIRA
StripedStorage.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2012 by
3  * MetraLabs GmbH (MLAB), GERMANY
4  * and
5  * Neuroinformatics and Cognitive Robotics Labs (NICR) at TU Ilmenau, GERMANY
6  * All rights reserved.
7  *
8  * Contact: info@mira-project.org
9  *
10  * Commercial Usage:
11  * Licensees holding valid commercial licenses may use this file in
12  * accordance with the commercial license agreement provided with the
13  * software or, alternatively, in accordance with the terms contained in
14  * a written agreement between you and MLAB or NICR.
15  *
16  * GNU General Public License Usage:
17  * Alternatively, this file may be used under the terms of the GNU
18  * General Public License version 3.0 as published by the Free Software
19  * Foundation and appearing in the file LICENSE.GPL3 included in the
20  * packaging of this file. Please review the following information to
21  * ensure the GNU General Public License version 3.0 requirements will be
22  * met: http://www.gnu.org/copyleft/gpl.html.
23  * Alternatively you may (at your option) use any later version of the GNU
24  * General Public License if such license has been publicly approved by
25  * MLAB and NICR (or its successors, if any).
26  *
27  * IN NO EVENT SHALL "MLAB" OR "NICR" BE LIABLE TO ANY PARTY FOR DIRECT,
28  * INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF
29  * THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF "MLAB" OR
30  * "NICR" HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  *
32  * "MLAB" AND "NICR" SPECIFICALLY DISCLAIM ANY WARRANTIES, INCLUDING,
33  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
34  * FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
35  * ON AN "AS IS" BASIS, AND "MLAB" AND "NICR" HAVE NO OBLIGATION TO
36  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS OR MODIFICATIONS.
37  */
38 
48 #ifndef _MIRA_STRIPEDSTORAGE_H_
49 #define _MIRA_STRIPEDSTORAGE_H_
50 
51 #include <error/Exceptions.h>
54 #include <utils/Path.h>
55 #include <utils/Time.h>
56 
57 namespace mira {
58 
60 
62 {
63 public:
65  setFilePath("");
66  }
67 
69  StripedStorageBase(std::string baseName, std::string extension, uint32 maxFiles) :
71  setFilePath("");
72  }
73 
75  StripedStorageBase(const Path& directory, std::string baseName, std::string extension, uint32 maxFiles) :
77  setFilePath(directory);
78  }
79 
80 public:
82  void setFilePath(const Path& path) {
83  filePath = resolvePath(path);
84  }
85 
86  const Path& getFilePath() {
87  return filePath;
88  }
89 
91  Path generateFilename(uint32 fileNr) const
92  {
93  std::string filename = fileNameBase + toString(fileNr) + extension;
94  return filePath / filename;
95  }
96 
98  std::list<Path> getFiles() const
99  {
100  std::list<Path> list;
101  for (uint32 i = 0; i < maxFiles; ++i)
102  list.push_back(generateFilename(i));
103  return list;
104  }
105 
108  {
109  auto files = getFiles();
110  foreach(auto f, files)
111  boost::filesystem::remove(f);
112  mCurrentFileIdx = 0;
113  }
114 
115  std::string fileNameBase;
116  std::string extension;
117  uint32 maxFiles;
118 
119 protected:
122 };
123 
125 
132 {
133 public:
135  StripedStorageXML(std::string baseName, uint32 maxFiles) : StripedStorageBase(baseName, ".xml", maxFiles) {}
136  StripedStorageXML(const Path& directory, std::string baseName, uint32 maxFiles) :
137  StripedStorageBase(directory, baseName, ".xml", maxFiles) {}
138 
139 public:
144  template <typename T>
145  Path store(const T& data, const Time& at = Time::now())
146  {
147  // Store the content in an XML file.
148  const Path filename = generateFilename(mCurrentFileIdx);
149 
150  XMLDom xml;
151  XMLSerializer s(xml);
152  s.serialize(fileNameBase, data);
153  s.serialize("Date", at);
154  xml.saveToFile(filename, false);
155 
156  MIRA_LOG(TRACE) << "StripedStorageXML: Wrote to file " << filename << " with timestamp " << at.toUnixTimestamp();
157 
158  // Switch to the next file at the end, after everything was okay.
160 
161  return filename;
162  }
163 
171  template <typename T>
172  std::pair<T, Time> retrieve(bool (*validCheckFunc)(const T&) = NULL) const
173  {
174  std::pair<T, Time> newestData;
175  std::pair<T, Time> pendingData;
176 
177  newestData.second = Time::unixEpoch();
178  bool found = false;
179 
180  for(uint32 i = 0; i < maxFiles; ++i)
181  {
182  const Path filename = generateFilename(i);
183 
184  try
185  {
186  MIRA_LOG(TRACE) << "StripedStorageXML: Trying to restore from file " << filename;
187 
188  XMLDom xml;
189  xml.loadFromFile(filename);
190 
191  XMLDeserializer ds(xml);
192  ds.deserialize("Date", pendingData.second);
193 
194  if(pendingData.second <= newestData.second)
195  MIRA_THROW(XLogical, "Data timestamp below required freshness data, aborting");
196 
197  // We have found data that is newer than the one we have restored so far
198  ds.deserialize(fileNameBase.c_str(), pendingData.first);
199  if(!validCheckFunc || validCheckFunc(pendingData.first))
200  {
201  newestData = pendingData;
202  found = true;
203  }
204  } catch(const Exception& ex) {
205  MIRA_LOG(TRACE) << "StripedStorageXML: Could not restore from file " << filename << ", exception: " << ex.what();
206  // ignore and continue with next file
207  }
208  }
209 
210  // We have no file that contains valid data
211  if(!found)
212  MIRA_THROW(XIO, "Unable to retrieve persistent information");
213 
214  return newestData;
215  }
216 };
217 
219 
221 {
222 public:
224  StripedStorageBinary(std::string baseName, uint32 maxFiles) : StripedStorageBase(baseName, ".bin", maxFiles) {}
225  StripedStorageBinary(const Path& directory, std::string baseName, uint32 maxFiles) :
226  StripedStorageBase(directory, baseName, ".bin", maxFiles) {}
227 
228 public:
233  template <typename T>
234  Path store(const T& data, const Time& at = Time::now())
235  {
236  // Store the content in an XML file.
237  const Path filename = generateFilename(mCurrentFileIdx);
238 
239  std::ofstream ofs(filename.c_str());
240  BinaryStlOstream bos(ofs);
241  BinaryStreamSerializer s(bos);
242  try {
243  s.serialize(at, false);
244  s.serialize(data, false);
245  s.serialize(at, false); // again for sanity check in retrieve
246  }
247  catch(Exception& ex) {
248  MIRA_RETHROW(ex, "While writing to " << filename);
249  }
250 
251  if (ofs.fail())
252  MIRA_THROW(XIO, "Failed writing to " << filename);
253 
254  MIRA_LOG(TRACE) << "StripedStorageBinary: Wrote to file " << filename << " with timestamp " << at.toUnixTimestamp();
255 
256  // Switch to the next file at the end, after everything was okay.
258 
259  return filename;
260  }
261 
269  template <typename T>
270  std::pair<T, Time> retrieve(bool (*validCheckFunc)(const T&) = NULL) const
271  {
272  std::pair<T, Time> newestData;
273  std::pair<T, Time> pendingData;
274 
275  newestData.second = Time::unixEpoch();
276  bool found = false;
277 
278  for(uint32 i = 0; i < maxFiles; ++i)
279  {
280  const Path filename = generateFilename(i);
281 
282  try
283  {
284  MIRA_LOG(TRACE) << "StripedStorageBinary: Trying to restore from file " << filename;
285 
286  std::ifstream ifs(filename.c_str());
287  BinaryStlIstream bis(ifs);
288  BinaryStreamDeserializer ds(bis);
289  ds.deserialize(pendingData.second, false);
290 
291  if(pendingData.second <= newestData.second)
292  MIRA_THROW(XLogical, "Data timestamp below required freshness data, aborting");
293 
294  // We have found data that is newer than the one we have restored so far
295  ds.deserialize(pendingData.first, false);
296  Time sanityTime;
297  ds.deserialize(sanityTime, false);
298  if(sanityTime != pendingData.second)
299  MIRA_THROW(XLogical, "Deserialized time sanity check failed");
300 
301  if(!validCheckFunc || validCheckFunc(pendingData.first))
302  {
303  newestData = pendingData;
304  found = true;
305  }
306  } catch(const Exception& ex) {
307  MIRA_LOG(TRACE) << "StripedStorageBinary: Could not restore from file " << filename << ", exception: " << ex.what();
308  // ignore and continue with next file
309  }
310  }
311 
312  // We have no file that contains valid data
313  if(!found)
314  MIRA_THROW(XIO, "Unable to retrieve persistent information");
315 
316  return newestData;
317  }
318 };
319 
321 
322 }
323 
324 #endif
Definition: BinarySerializer.h:324
const Path & getFilePath()
Definition: StripedStorage.h:86
A STL conform wrapper for libxml2 to read XML files as DOM.
Definition: XMLDom.h:73
Stores data sequentially into different files (striped) providing a reliable way to retrieve data lat...
Definition: StripedStorage.h:131
void saveToFile(const Path &filename, const std::string &encoding="UTF-8", bool resolve=true) const
Save the XML document to a file.
void setFilePath(const Path &path)
Resolves path and assigns to filePath.
Definition: StripedStorage.h:82
Path store(const T &data, const Time &at=Time::now())
Stores the given data in one of the files adding a timestamp and returns the filename of the written ...
Definition: StripedStorage.h:234
void deserialize(const std::string &name, T &value)
overwritten from Deserializer, to handle exceptions and to add additional information about file name...
Definition: XMLSerializer.h:340
Definition: XMLSerializer.h:121
specialize cv::DataType for our ImgPixel and inherit from cv::DataType<Vec>
Definition: IOService.h:67
boost::filesystem::path Path
Typedef of a Path (shorter version for boost::filesystem::path)
Definition: Path.h:69
#define MIRA_LOG(level)
Use this macro to log data.
Definition: LoggingCore.h:529
Time and Duration wrapper class.
#define MIRA_RETHROW(ex, msg)
Macro for rethrowing an exception with file and line information and for adding additional informatio...
Definition: Exception.h:148
Definition: BinarySerializer.h:991
void loadFromFile(const Path &filename, bool resolve=true)
Load and parse an XML document from a file.
StripedStorageBase(std::string baseName, std::string extension, uint32 maxFiles)
Create a striped storage providing a base file name and the maximum number of files.
Definition: StripedStorage.h:69
Definition: LoggingCore.h:79
static Time unixEpoch()
Returns the unix epoch 1.1.1970 0:0:0.000.
Definition: Time.h:511
void serialize(const std::string &name, const T &value, const std::string &comment="")
Serializes the specified object value under the given name.
Definition: Serializer.h:204
#define MIRA_THROW(ex, msg)
Macro for throwing an exception.
Definition: Exception.h:82
void serialize(const T &value, bool enableTypeCheck=true)
Provides a special serialize interface for the BinarySerializer.
Definition: BinarySerializer.h:598
std::string toString(const T &value, int precision=-1)
Converts any data type to string (the data type must support the stream << operator).
Definition: ToString.h:256
Wrapper class for boost::posix_time::ptime for adding more functionality to it.
Definition: Time.h:418
StripedStorageBinary()
Definition: StripedStorage.h:223
Output stream adapter that can be assigned to any output stream and allows binary output using the <<...
Definition: BinaryStream.h:293
Commonly used exception classes.
Binary serializer and deserializer.
StripedStorageBase()
Definition: StripedStorage.h:64
std::pair< T, Time > retrieve(bool(*validCheckFunc)(const T &)=NULL) const
Retrieves the data from one of the files.
Definition: StripedStorage.h:270
Definition: StripedStorage.h:61
uint32 mCurrentFileIdx
Definition: StripedStorage.h:121
StripedStorageXML(std::string baseName, uint32 maxFiles)
Definition: StripedStorage.h:135
std::list< Path > getFiles() const
Returns a list of filenames (including paths) for all files that are used for persistent storage...
Definition: StripedStorage.h:98
void deleteAllFiles()
Delete all files that were stored by the persistent storage.
Definition: StripedStorage.h:107
Path store(const T &data, const Time &at=Time::now())
Stores the given data in one of the files adding a timestamp and returns the filename of the written ...
Definition: StripedStorage.h:145
Path filePath
The path where the files are stored.
Definition: StripedStorage.h:120
StripedStorageXML(const Path &directory, std::string baseName, uint32 maxFiles)
Definition: StripedStorage.h:136
Base class for exceptions.
Definition: Exception.h:199
StripedStorageBinary(const Path &directory, std::string baseName, uint32 maxFiles)
Definition: StripedStorage.h:225
StripedStorageBase(const Path &directory, std::string baseName, std::string extension, uint32 maxFiles)
Create a striped storage providing a path, a base file name and the maximum number of files...
Definition: StripedStorage.h:75
Input stream adapter that can be assigned to any input stream and allows binary input using the >> st...
Definition: BinaryStream.h:523
void deserialize(T &value, bool enableTypeCheck=true, bool recursive=false)
Provides a special deserialize interface for the BinaryDeserializer.
Definition: BinarySerializer.h:1340
MIRA_BASE_EXPORT Path resolvePath(const Path &path, bool makeAbsolute=true)
Resolves a path.
Functions for modifying file system paths.
static Time now() static Time eternity()
Returns the current utc based time.
Definition: Time.h:481
Definition: StripedStorage.h:220
Path generateFilename(uint32 fileNr) const
Returns the filename (including path) of the file with the specified nr.
Definition: StripedStorage.h:91
XMLSerializer and XMLDeserializer.
std::pair< T, Time > retrieve(bool(*validCheckFunc)(const T &)=NULL) const
Retrieves the data from one of the files.
Definition: StripedStorage.h:172
virtual const char * what() const MIRA_NOEXCEPT_OR_NOTHROW
Returns the text of exception containing the information given in MIRA_THROW and MIRA_RETHROW as well...
uint32 maxFiles
Maximum number of used files.
Definition: StripedStorage.h:117
StripedStorageBinary(std::string baseName, uint32 maxFiles)
Definition: StripedStorage.h:224
StripedStorageXML()
Definition: StripedStorage.h:134
std::string fileNameBase
The base filename.
Definition: StripedStorage.h:115
std::string extension
The file name extension.
Definition: StripedStorage.h:116
Deserializer for serializing objects from XML format.
Definition: XMLSerializer.h:313