Package CedarBackup3 :: Package actions :: Module store
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.actions.store

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2004-2007,2010,2015 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python 3 (>= 3.4) 
 29  # Project  : Cedar Backup, release 3 
 30  # Purpose  : Implements the standard 'store' action. 
 31  # 
 32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 33   
 34  ######################################################################## 
 35  # Module documentation 
 36  ######################################################################## 
 37   
 38  """ 
 39  Implements the standard 'store' action. 
 40  @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck 
 41  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 42  @author: Dmitry Rutsky <rutsky@inbox.ru> 
 43  """ 
 44   
 45   
 46  ######################################################################## 
 47  # Imported modules 
 48  ######################################################################## 
 49   
 50  # System modules 
 51  import sys 
 52  import os 
 53  import logging 
 54  import datetime 
 55  import tempfile 
 56   
 57  # Cedar Backup modules 
 58  from CedarBackup3.filesystem import compareContents 
 59  from CedarBackup3.util import isStartOfWeek 
 60  from CedarBackup3.util import mount, unmount, displayBytes 
 61  from CedarBackup3.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile 
 62  from CedarBackup3.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR 
 63   
 64   
 65  ######################################################################## 
 66  # Module-wide constants and variables 
 67  ######################################################################## 
 68   
 69  logger = logging.getLogger("CedarBackup3.log.actions.store") 
 70   
 71   
 72  ######################################################################## 
 73  # Public functions 
 74  ######################################################################## 
 75   
 76  ########################## 
 77  # executeStore() function 
 78  ########################## 
 79   
80 -def executeStore(configPath, options, config):
81 """ 82 Executes the store backup action. 83 84 @note: The rebuild action and the store action are very similar. The 85 main difference is that while store only stores a single day's staging 86 directory, the rebuild action operates on multiple staging directories. 87 88 @note: When the store action is complete, we will write a store indicator to 89 the daily staging directory we used, so it's obvious that the store action 90 has completed. 91 92 @param configPath: Path to configuration file on disk. 93 @type configPath: String representing a path on disk. 94 95 @param options: Program command-line options. 96 @type options: Options object. 97 98 @param config: Program configuration. 99 @type config: Config object. 100 101 @raise ValueError: Under many generic error conditions 102 @raise IOError: If there are problems reading or writing files. 103 """ 104 logger.debug("Executing the 'store' action.") 105 if sys.platform == "darwin": 106 logger.warn("Warning: the store action is not fully supported on Mac OS X.") 107 logger.warn("See the Cedar Backup software manual for further information.") 108 if config.options is None or config.store is None: 109 raise ValueError("Store configuration is not properly filled in.") 110 if config.store.checkMedia: 111 checkMediaState(config.store) # raises exception if media is not initialized 112 rebuildMedia = options.full 113 logger.debug("Rebuild media flag [%s]", rebuildMedia) 114 todayIsStart = isStartOfWeek(config.options.startingDay) 115 stagingDirs = _findCorrectDailyDir(options, config) 116 writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs) 117 if config.store.checkData: 118 if sys.platform == "darwin": 119 logger.warn("Warning: consistency check cannot be run successfully on Mac OS X.") 120 logger.warn("See the Cedar Backup software manual for further information.") 121 else: 122 logger.debug("Running consistency check of media.") 123 consistencyCheck(config, stagingDirs) 124 writeStoreIndicator(config, stagingDirs) 125 logger.info("Executed the 'store' action successfully.")
126 127 128 ######################## 129 # writeImage() function 130 ######################## 131
132 -def writeImage(config, newDisc, stagingDirs):
133 """ 134 Builds and writes an ISO image containing the indicated stage directories. 135 136 The generated image will contain each of the staging directories listed in 137 C{stagingDirs}. The directories will be placed into the image at the root by 138 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the 139 disc at C{/2005/02/10}. 140 141 @note: This function is implemented in terms of L{writeImageBlankSafe}. The 142 C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}. 143 144 @param config: Config object. 145 @param newDisc: Indicates whether the disc should be re-initialized 146 @param stagingDirs: Dictionary mapping directory path to date suffix. 147 148 @raise ValueError: Under many generic error conditions 149 @raise IOError: If there is a problem writing the image to disc. 150 """ 151 writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs)
152 153 154 ################################# 155 # writeImageBlankSafe() function 156 ################################# 157
158 -def writeImageBlankSafe(config, rebuildMedia, todayIsStart, blankBehavior, stagingDirs):
159 """ 160 Builds and writes an ISO image containing the indicated stage directories. 161 162 The generated image will contain each of the staging directories listed in 163 C{stagingDirs}. The directories will be placed into the image at the root by 164 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the 165 disc at C{/2005/02/10}. The media will always be written with a media 166 label specific to Cedar Backup. 167 168 This function is similar to L{writeImage}, but tries to implement a smarter 169 blanking strategy. 170 171 First, the media is always blanked if the C{rebuildMedia} flag is true. 172 Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart} 173 come into effect:: 174 175 If no blanking behavior is specified, and it is the start of the week, 176 the disc will be blanked 177 178 If blanking behavior is specified, and either the blank mode is "daily" 179 or the blank mode is "weekly" and it is the start of the week, then 180 the disc will be blanked if it looks like the weekly backup will not 181 fit onto the media. 182 183 Otherwise, the disc will not be blanked 184 185 How do we decide whether the weekly backup will fit onto the media? That is 186 what the blanking factor is used for. The following formula is used:: 187 188 will backup fit? = (bytes available / (1 + bytes required) <= blankFactor 189 190 The blanking factor will vary from setup to setup, and will probably 191 require some experimentation to get it right. 192 193 @param config: Config object. 194 @param rebuildMedia: Indicates whether media should be rebuilt 195 @param todayIsStart: Indicates whether today is the starting day of the week 196 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior 197 @param stagingDirs: Dictionary mapping directory path to date suffix. 198 199 @raise ValueError: Under many generic error conditions 200 @raise IOError: If there is a problem writing the image to disc. 201 """ 202 mediaLabel = buildMediaLabel() 203 writer = createWriter(config) 204 writer.initializeImage(True, config.options.workingDir, mediaLabel) # default value for newDisc 205 for stageDir in list(stagingDirs.keys()): 206 logger.debug("Adding stage directory [%s].", stageDir) 207 dateSuffix = stagingDirs[stageDir] 208 writer.addImageEntry(stageDir, dateSuffix) 209 newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior) 210 writer.setImageNewDisc(newDisc) 211 writer.writeImage()
212
213 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior):
214 """ 215 Gets a value for the newDisc flag based on blanking factor rules. 216 217 The blanking factor rules are described above by L{writeImageBlankSafe}. 218 219 @param writer: Previously configured image writer containing image entries 220 @param rebuildMedia: Indicates whether media should be rebuilt 221 @param todayIsStart: Indicates whether today is the starting day of the week 222 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior 223 224 @return: newDisc flag to be set on writer. 225 """ 226 newDisc = False 227 if rebuildMedia: 228 newDisc = True 229 logger.debug("Setting new disc flag based on rebuildMedia flag.") 230 else: 231 if blankBehavior is None: 232 logger.debug("Default media blanking behavior is in effect.") 233 if todayIsStart: 234 newDisc = True 235 logger.debug("Setting new disc flag based on todayIsStart.") 236 else: 237 # note: validation says we can assume that behavior is fully filled in if it exists at all 238 logger.debug("Optimized media blanking behavior is in effect based on configuration.") 239 if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart): 240 logger.debug("New disc flag will be set based on blank factor calculation.") 241 blankFactor = float(blankBehavior.blankFactor) 242 logger.debug("Configured blanking factor: %.2f", blankFactor) 243 available = writer.retrieveCapacity().bytesAvailable 244 logger.debug("Bytes available: %s", displayBytes(available)) 245 required = writer.getEstimatedImageSize() 246 logger.debug("Bytes required: %s", displayBytes(required)) 247 ratio = available / (1.0 + required) 248 logger.debug("Calculated ratio: %.2f", ratio) 249 newDisc = (ratio <= blankFactor) 250 logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc) 251 else: 252 logger.debug("No blank factor calculation is required based on configuration.") 253 logger.debug("New disc flag [%s].", newDisc) 254 return newDisc
255 256 257 ################################# 258 # writeStoreIndicator() function 259 ################################# 260
261 -def writeStoreIndicator(config, stagingDirs):
262 """ 263 Writes a store indicator file into staging directories. 264 265 The store indicator is written into each of the staging directories when 266 either a store or rebuild action has written the staging directory to disc. 267 268 @param config: Config object. 269 @param stagingDirs: Dictionary mapping directory path to date suffix. 270 """ 271 for stagingDir in list(stagingDirs.keys()): 272 writeIndicatorFile(stagingDir, STORE_INDICATOR, 273 config.options.backupUser, 274 config.options.backupGroup)
275 276 277 ############################## 278 # consistencyCheck() function 279 ############################## 280
281 -def consistencyCheck(config, stagingDirs):
282 """ 283 Runs a consistency check against media in the backup device. 284 285 It seems that sometimes, it's possible to create a corrupted multisession 286 disc (i.e. one that cannot be read) although no errors were encountered 287 while writing the disc. This consistency check makes sure that the data 288 read from disc matches the data that was used to create the disc. 289 290 The function mounts the device at a temporary mount point in the working 291 directory, and then compares the indicated staging directories in the 292 staging directory and on the media. The comparison is done via 293 functionality in C{filesystem.py}. 294 295 If no exceptions are thrown, there were no problems with the consistency 296 check. A positive confirmation of "no problems" is also written to the log 297 with C{info} priority. 298 299 @warning: The implementation of this function is very UNIX-specific. 300 301 @param config: Config object. 302 @param stagingDirs: Dictionary mapping directory path to date suffix. 303 304 @raise ValueError: If the two directories are not equivalent. 305 @raise IOError: If there is a problem working with the media. 306 """ 307 logger.debug("Running consistency check.") 308 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir) 309 try: 310 mount(config.store.devicePath, mountPoint, "iso9660") 311 for stagingDir in list(stagingDirs.keys()): 312 discDir = os.path.join(mountPoint, stagingDirs[stagingDir]) 313 logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir) 314 compareContents(stagingDir, discDir, verbose=True) 315 logger.info("Consistency check completed for [%s]. No problems found.", stagingDir) 316 finally: 317 unmount(mountPoint, True, 5, 1) # try 5 times, and remove mount point when done
318 319 320 ######################################################################## 321 # Private utility functions 322 ######################################################################## 323 324 ######################### 325 # _findCorrectDailyDir() 326 ######################### 327
328 -def _findCorrectDailyDir(options, config):
329 """ 330 Finds the correct daily staging directory to be written to disk. 331 332 In Cedar Backup v1.0, we assumed that the correct staging directory matched 333 the current date. However, that has problems. In particular, it breaks 334 down if collect is on one side of midnite and stage is on the other, or if 335 certain processes span midnite. 336 337 For v2.0, I'm trying to be smarter. I'll first check the current day. If 338 that directory is found, it's good enough. If it's not found, I'll look for 339 a valid directory from the day before or day after I{which has not yet been 340 staged, according to the stage indicator file}. The first one I find, I'll 341 use. If I use a directory other than for the current day I{and} 342 C{config.store.warnMidnite} is set, a warning will be put in the log. 343 344 There is one exception to this rule. If the C{options.full} flag is set, 345 then the special "span midnite" logic will be disabled and any existing 346 store indicator will be ignored. I did this because I think that most users 347 who run C{cback3 --full store} twice in a row expect the command to generate 348 two identical discs. With the other rule in place, running that command 349 twice in a row could result in an error ("no unstored directory exists") or 350 could even cause a completely unexpected directory to be written to disc (if 351 some previous day's contents had not yet been written). 352 353 @note: This code is probably longer and more verbose than it needs to be, 354 but at least it's straightforward. 355 356 @param options: Options object. 357 @param config: Config object. 358 359 @return: Correct staging dir, as a dict mapping directory to date suffix. 360 @raise IOError: If the staging directory cannot be found. 361 """ 362 oneDay = datetime.timedelta(days=1) 363 today = datetime.date.today() 364 yesterday = today - oneDay 365 tomorrow = today + oneDay 366 todayDate = today.strftime(DIR_TIME_FORMAT) 367 yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT) 368 tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT) 369 todayPath = os.path.join(config.stage.targetDir, todayDate) 370 yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate) 371 tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate) 372 todayStageInd = os.path.join(todayPath, STAGE_INDICATOR) 373 yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR) 374 tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR) 375 todayStoreInd = os.path.join(todayPath, STORE_INDICATOR) 376 yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR) 377 tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR) 378 if options.full: 379 if os.path.isdir(todayPath) and os.path.exists(todayStageInd): 380 logger.info("Store process will use current day's stage directory [%s]", todayPath) 381 return { todayPath:todayDate } 382 raise IOError("Unable to find staging directory to store (only tried today due to full option).") 383 else: 384 if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd): 385 logger.info("Store process will use current day's stage directory [%s]", todayPath) 386 return { todayPath:todayDate } 387 elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd): 388 logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath) 389 if config.store.warnMidnite: 390 logger.warn("Warning: store process crossed midnite boundary to find data.") 391 return { yesterdayPath:yesterdayDate } 392 elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd): 393 logger.info("Store process will use next day's stage directory [%s]", tomorrowPath) 394 if config.store.warnMidnite: 395 logger.warn("Warning: store process crossed midnite boundary to find data.") 396 return { tomorrowPath:tomorrowDate } 397 raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).")
398