Package CedarBackup3 :: Package actions :: Module store
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.actions.store

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2004-2007,2010,2015 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python 3 (>= 3.4) 
 29  # Project  : Cedar Backup, release 3 
 30  # Purpose  : Implements the standard 'store' action. 
 31  # 
 32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 33   
 34  ######################################################################## 
 35  # Module documentation 
 36  ######################################################################## 
 37   
 38  """ 
 39  Implements the standard 'store' action. 
 40  @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck 
 41  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 42  @author: Dmitry Rutsky <rutsky@inbox.ru> 
 43  """ 
 44   
 45   
 46  ######################################################################## 
 47  # Imported modules 
 48  ######################################################################## 
 49   
 50  # System modules 
 51  import sys 
 52  import os 
 53  import logging 
 54  import datetime 
 55  import tempfile 
 56   
 57  # Cedar Backup modules 
 58  from CedarBackup3.filesystem import compareContents 
 59  from CedarBackup3.util import isStartOfWeek 
 60  from CedarBackup3.util import mount, unmount, displayBytes 
 61  from CedarBackup3.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile 
 62  from CedarBackup3.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR 
 63   
 64   
 65  ######################################################################## 
 66  # Module-wide constants and variables 
 67  ######################################################################## 
 68   
 69  logger = logging.getLogger("CedarBackup3.log.actions.store") 
 70   
 71   
 72  ######################################################################## 
 73  # Public functions 
 74  ######################################################################## 
 75   
 76  ########################## 
 77  # executeStore() function 
 78  ########################## 
 79   
 80  # pylint: disable=W0613 
81 -def executeStore(configPath, options, config):
82 """ 83 Executes the store backup action. 84 85 @note: The rebuild action and the store action are very similar. The 86 main difference is that while store only stores a single day's staging 87 directory, the rebuild action operates on multiple staging directories. 88 89 @note: When the store action is complete, we will write a store indicator to 90 the daily staging directory we used, so it's obvious that the store action 91 has completed. 92 93 @param configPath: Path to configuration file on disk. 94 @type configPath: String representing a path on disk. 95 96 @param options: Program command-line options. 97 @type options: Options object. 98 99 @param config: Program configuration. 100 @type config: Config object. 101 102 @raise ValueError: Under many generic error conditions 103 @raise IOError: If there are problems reading or writing files. 104 """ 105 logger.debug("Executing the 'store' action.") 106 if sys.platform == "darwin": 107 logger.warning("Warning: the store action is not fully supported on Mac OS X.") 108 logger.warning("See the Cedar Backup software manual for further information.") 109 if config.options is None or config.store is None: 110 raise ValueError("Store configuration is not properly filled in.") 111 if config.store.checkMedia: 112 checkMediaState(config.store) # raises exception if media is not initialized 113 rebuildMedia = options.full 114 logger.debug("Rebuild media flag [%s]", rebuildMedia) 115 todayIsStart = isStartOfWeek(config.options.startingDay) 116 stagingDirs = _findCorrectDailyDir(options, config) 117 writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs) 118 if config.store.checkData: 119 if sys.platform == "darwin": 120 logger.warning("Warning: consistency check cannot be run successfully on Mac OS X.") 121 logger.warning("See the Cedar Backup software manual for further information.") 122 else: 123 logger.debug("Running consistency check of media.") 124 consistencyCheck(config, stagingDirs) 125 writeStoreIndicator(config, stagingDirs) 126 logger.info("Executed the 'store' action successfully.")
127 128 129 ######################## 130 # writeImage() function 131 ######################## 132
133 -def writeImage(config, newDisc, stagingDirs):
134 """ 135 Builds and writes an ISO image containing the indicated stage directories. 136 137 The generated image will contain each of the staging directories listed in 138 C{stagingDirs}. The directories will be placed into the image at the root by 139 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the 140 disc at C{/2005/02/10}. 141 142 @note: This function is implemented in terms of L{writeImageBlankSafe}. The 143 C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}. 144 145 @param config: Config object. 146 @param newDisc: Indicates whether the disc should be re-initialized 147 @param stagingDirs: Dictionary mapping directory path to date suffix. 148 149 @raise ValueError: Under many generic error conditions 150 @raise IOError: If there is a problem writing the image to disc. 151 """ 152 writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs)
153 154 155 ################################# 156 # writeImageBlankSafe() function 157 ################################# 158
159 -def writeImageBlankSafe(config, rebuildMedia, todayIsStart, blankBehavior, stagingDirs):
160 """ 161 Builds and writes an ISO image containing the indicated stage directories. 162 163 The generated image will contain each of the staging directories listed in 164 C{stagingDirs}. The directories will be placed into the image at the root by 165 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the 166 disc at C{/2005/02/10}. The media will always be written with a media 167 label specific to Cedar Backup. 168 169 This function is similar to L{writeImage}, but tries to implement a smarter 170 blanking strategy. 171 172 First, the media is always blanked if the C{rebuildMedia} flag is true. 173 Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart} 174 come into effect:: 175 176 If no blanking behavior is specified, and it is the start of the week, 177 the disc will be blanked 178 179 If blanking behavior is specified, and either the blank mode is "daily" 180 or the blank mode is "weekly" and it is the start of the week, then 181 the disc will be blanked if it looks like the weekly backup will not 182 fit onto the media. 183 184 Otherwise, the disc will not be blanked 185 186 How do we decide whether the weekly backup will fit onto the media? That is 187 what the blanking factor is used for. The following formula is used:: 188 189 will backup fit? = (bytes available / (1 + bytes required) <= blankFactor 190 191 The blanking factor will vary from setup to setup, and will probably 192 require some experimentation to get it right. 193 194 @param config: Config object. 195 @param rebuildMedia: Indicates whether media should be rebuilt 196 @param todayIsStart: Indicates whether today is the starting day of the week 197 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior 198 @param stagingDirs: Dictionary mapping directory path to date suffix. 199 200 @raise ValueError: Under many generic error conditions 201 @raise IOError: If there is a problem writing the image to disc. 202 """ 203 mediaLabel = buildMediaLabel() 204 writer = createWriter(config) 205 writer.initializeImage(True, config.options.workingDir, mediaLabel) # default value for newDisc 206 for stageDir in list(stagingDirs.keys()): 207 logger.debug("Adding stage directory [%s].", stageDir) 208 dateSuffix = stagingDirs[stageDir] 209 writer.addImageEntry(stageDir, dateSuffix) 210 newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior) 211 writer.setImageNewDisc(newDisc) 212 writer.writeImage()
213
214 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior):
215 """ 216 Gets a value for the newDisc flag based on blanking factor rules. 217 218 The blanking factor rules are described above by L{writeImageBlankSafe}. 219 220 @param writer: Previously configured image writer containing image entries 221 @param rebuildMedia: Indicates whether media should be rebuilt 222 @param todayIsStart: Indicates whether today is the starting day of the week 223 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior 224 225 @return: newDisc flag to be set on writer. 226 """ 227 newDisc = False 228 if rebuildMedia: 229 newDisc = True 230 logger.debug("Setting new disc flag based on rebuildMedia flag.") 231 else: 232 if blankBehavior is None: 233 logger.debug("Default media blanking behavior is in effect.") 234 if todayIsStart: 235 newDisc = True 236 logger.debug("Setting new disc flag based on todayIsStart.") 237 else: 238 # note: validation says we can assume that behavior is fully filled in if it exists at all 239 logger.debug("Optimized media blanking behavior is in effect based on configuration.") 240 if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart): 241 logger.debug("New disc flag will be set based on blank factor calculation.") 242 blankFactor = float(blankBehavior.blankFactor) 243 logger.debug("Configured blanking factor: %.2f", blankFactor) 244 available = writer.retrieveCapacity().bytesAvailable 245 logger.debug("Bytes available: %s", displayBytes(available)) 246 required = writer.getEstimatedImageSize() 247 logger.debug("Bytes required: %s", displayBytes(required)) 248 ratio = available / (1.0 + required) 249 logger.debug("Calculated ratio: %.2f", ratio) 250 newDisc = (ratio <= blankFactor) 251 logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc) 252 else: 253 logger.debug("No blank factor calculation is required based on configuration.") 254 logger.debug("New disc flag [%s].", newDisc) 255 return newDisc
256 257 258 ################################# 259 # writeStoreIndicator() function 260 ################################# 261
262 -def writeStoreIndicator(config, stagingDirs):
263 """ 264 Writes a store indicator file into staging directories. 265 266 The store indicator is written into each of the staging directories when 267 either a store or rebuild action has written the staging directory to disc. 268 269 @param config: Config object. 270 @param stagingDirs: Dictionary mapping directory path to date suffix. 271 """ 272 for stagingDir in list(stagingDirs.keys()): 273 writeIndicatorFile(stagingDir, STORE_INDICATOR, 274 config.options.backupUser, 275 config.options.backupGroup)
276 277 278 ############################## 279 # consistencyCheck() function 280 ############################## 281
282 -def consistencyCheck(config, stagingDirs):
283 """ 284 Runs a consistency check against media in the backup device. 285 286 It seems that sometimes, it's possible to create a corrupted multisession 287 disc (i.e. one that cannot be read) although no errors were encountered 288 while writing the disc. This consistency check makes sure that the data 289 read from disc matches the data that was used to create the disc. 290 291 The function mounts the device at a temporary mount point in the working 292 directory, and then compares the indicated staging directories in the 293 staging directory and on the media. The comparison is done via 294 functionality in C{filesystem.py}. 295 296 If no exceptions are thrown, there were no problems with the consistency 297 check. A positive confirmation of "no problems" is also written to the log 298 with C{info} priority. 299 300 @warning: The implementation of this function is very UNIX-specific. 301 302 @param config: Config object. 303 @param stagingDirs: Dictionary mapping directory path to date suffix. 304 305 @raise ValueError: If the two directories are not equivalent. 306 @raise IOError: If there is a problem working with the media. 307 """ 308 logger.debug("Running consistency check.") 309 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir) 310 try: 311 mount(config.store.devicePath, mountPoint, "iso9660") 312 for stagingDir in list(stagingDirs.keys()): 313 discDir = os.path.join(mountPoint, stagingDirs[stagingDir]) 314 logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir) 315 compareContents(stagingDir, discDir, verbose=True) 316 logger.info("Consistency check completed for [%s]. No problems found.", stagingDir) 317 finally: 318 unmount(mountPoint, True, 5, 1) # try 5 times, and remove mount point when done
319 320 321 ######################################################################## 322 # Private utility functions 323 ######################################################################## 324 325 ######################### 326 # _findCorrectDailyDir() 327 ######################### 328
329 -def _findCorrectDailyDir(options, config):
330 """ 331 Finds the correct daily staging directory to be written to disk. 332 333 In Cedar Backup v1.0, we assumed that the correct staging directory matched 334 the current date. However, that has problems. In particular, it breaks 335 down if collect is on one side of midnite and stage is on the other, or if 336 certain processes span midnite. 337 338 For v2.0, I'm trying to be smarter. I'll first check the current day. If 339 that directory is found, it's good enough. If it's not found, I'll look for 340 a valid directory from the day before or day after I{which has not yet been 341 staged, according to the stage indicator file}. The first one I find, I'll 342 use. If I use a directory other than for the current day I{and} 343 C{config.store.warnMidnite} is set, a warning will be put in the log. 344 345 There is one exception to this rule. If the C{options.full} flag is set, 346 then the special "span midnite" logic will be disabled and any existing 347 store indicator will be ignored. I did this because I think that most users 348 who run C{cback3 --full store} twice in a row expect the command to generate 349 two identical discs. With the other rule in place, running that command 350 twice in a row could result in an error ("no unstored directory exists") or 351 could even cause a completely unexpected directory to be written to disc (if 352 some previous day's contents had not yet been written). 353 354 @note: This code is probably longer and more verbose than it needs to be, 355 but at least it's straightforward. 356 357 @param options: Options object. 358 @param config: Config object. 359 360 @return: Correct staging dir, as a dict mapping directory to date suffix. 361 @raise IOError: If the staging directory cannot be found. 362 """ 363 oneDay = datetime.timedelta(days=1) 364 today = datetime.date.today() 365 yesterday = today - oneDay 366 tomorrow = today + oneDay 367 todayDate = today.strftime(DIR_TIME_FORMAT) 368 yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT) 369 tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT) 370 todayPath = os.path.join(config.stage.targetDir, todayDate) 371 yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate) 372 tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate) 373 todayStageInd = os.path.join(todayPath, STAGE_INDICATOR) 374 yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR) 375 tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR) 376 todayStoreInd = os.path.join(todayPath, STORE_INDICATOR) 377 yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR) 378 tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR) 379 if options.full: 380 if os.path.isdir(todayPath) and os.path.exists(todayStageInd): 381 logger.info("Store process will use current day's stage directory [%s]", todayPath) 382 return { todayPath:todayDate } 383 raise IOError("Unable to find staging directory to store (only tried today due to full option).") 384 else: 385 if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd): 386 logger.info("Store process will use current day's stage directory [%s]", todayPath) 387 return { todayPath:todayDate } 388 elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd): 389 logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath) 390 if config.store.warnMidnite: 391 logger.warning("Warning: store process crossed midnite boundary to find data.") 392 return { yesterdayPath:yesterdayDate } 393 elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd): 394 logger.info("Store process will use next day's stage directory [%s]", tomorrowPath) 395 if config.store.warnMidnite: 396 logger.warning("Warning: store process crossed midnite boundary to find data.") 397 return { tomorrowPath:tomorrowDate } 398 raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).")
399