Package CedarBackup3 :: Package extend :: Module split
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.extend.split

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2007,2010,2013,2015 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python 3 (>= 3.4) 
 29  # Project  : Official Cedar Backup Extensions 
 30  # Purpose  : Provides an extension to split up large files in staging directories. 
 31  # 
 32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 33   
 34  ######################################################################## 
 35  # Module documentation 
 36  ######################################################################## 
 37   
 38  """ 
 39  Provides an extension to split up large files in staging directories. 
 40   
 41  When this extension is executed, it will look through the configured Cedar 
 42  Backup staging directory for files exceeding a specified size limit, and split 
 43  them down into smaller files using the 'split' utility.  Any directory which 
 44  has already been split (as indicated by the C{cback.split} file) will be 
 45  ignored. 
 46   
 47  This extension requires a new configuration section <split> and is intended 
 48  to be run immediately after the standard stage action or immediately before the 
 49  standard store action.  Aside from its own configuration, it requires the 
 50  options and staging configuration sections in the standard Cedar Backup 
 51  configuration file. 
 52   
 53  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 54  """ 
 55   
 56  ######################################################################## 
 57  # Imported modules 
 58  ######################################################################## 
 59   
 60  # System modules 
 61  import os 
 62  import re 
 63  import logging 
 64  from functools import total_ordering 
 65   
 66  # Cedar Backup modules 
 67  from CedarBackup3.util import resolveCommand, executeCommand, changeOwnership 
 68  from CedarBackup3.xmlutil import createInputDom, addContainerNode 
 69  from CedarBackup3.xmlutil import readFirstChild 
 70  from CedarBackup3.actions.util import findDailyDirs, writeIndicatorFile, getBackupFiles 
 71  from CedarBackup3.config import ByteQuantity, readByteQuantity, addByteQuantityNode 
 72   
 73   
 74  ######################################################################## 
 75  # Module-wide constants and variables 
 76  ######################################################################## 
 77   
 78  logger = logging.getLogger("CedarBackup3.log.extend.split") 
 79   
 80  SPLIT_COMMAND = [ "split", ] 
 81  SPLIT_INDICATOR = "cback.split" 
82 83 84 ######################################################################## 85 # SplitConfig class definition 86 ######################################################################## 87 88 @total_ordering 89 -class SplitConfig(object):
90 91 """ 92 Class representing split configuration. 93 94 Split configuration is used for splitting staging directories. 95 96 The following restrictions exist on data in this class: 97 98 - The size limit must be a ByteQuantity 99 - The split size must be a ByteQuantity 100 101 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, 102 sizeLimit, splitSize 103 """ 104
105 - def __init__(self, sizeLimit=None, splitSize=None):
106 """ 107 Constructor for the C{SplitCOnfig} class. 108 109 @param sizeLimit: Size limit of the files, in bytes 110 @param splitSize: Size that files exceeding the limit will be split into, in bytes 111 112 @raise ValueError: If one of the values is invalid. 113 """ 114 self._sizeLimit = None 115 self._splitSize = None 116 self.sizeLimit = sizeLimit 117 self.splitSize = splitSize
118
119 - def __repr__(self):
120 """ 121 Official string representation for class instance. 122 """ 123 return "SplitConfig(%s, %s)" % (self.sizeLimit, self.splitSize)
124
125 - def __str__(self):
126 """ 127 Informal string representation for class instance. 128 """ 129 return self.__repr__()
130
131 - def __eq__(self, other):
132 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 133 return self.__cmp__(other) == 0
134
135 - def __lt__(self, other):
136 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 137 return self.__cmp__(other) < 0
138
139 - def __gt__(self, other):
140 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 141 return self.__cmp__(other) > 0
142
143 - def __cmp__(self, other):
144 """ 145 Original Python 2 comparison operator. 146 Lists within this class are "unordered" for equality comparisons. 147 @param other: Other object to compare to. 148 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 149 """ 150 if other is None: 151 return 1 152 if self.sizeLimit != other.sizeLimit: 153 if (self.sizeLimit or ByteQuantity()) < (other.sizeLimit or ByteQuantity()): 154 return -1 155 else: 156 return 1 157 if self.splitSize != other.splitSize: 158 if (self.splitSize or ByteQuantity()) < (other.splitSize or ByteQuantity()): 159 return -1 160 else: 161 return 1 162 return 0
163
164 - def _setSizeLimit(self, value):
165 """ 166 Property target used to set the size limit. 167 If not C{None}, the value must be a C{ByteQuantity} object. 168 @raise ValueError: If the value is not a C{ByteQuantity} 169 """ 170 if value is None: 171 self._sizeLimit = None 172 else: 173 if not isinstance(value, ByteQuantity): 174 raise ValueError("Value must be a C{ByteQuantity} object.") 175 self._sizeLimit = value
176
177 - def _getSizeLimit(self):
178 """ 179 Property target used to get the size limit. 180 """ 181 return self._sizeLimit
182
183 - def _setSplitSize(self, value):
184 """ 185 Property target used to set the split size. 186 If not C{None}, the value must be a C{ByteQuantity} object. 187 @raise ValueError: If the value is not a C{ByteQuantity} 188 """ 189 if value is None: 190 self._splitSize = None 191 else: 192 if not isinstance(value, ByteQuantity): 193 raise ValueError("Value must be a C{ByteQuantity} object.") 194 self._splitSize = value
195
196 - def _getSplitSize(self):
197 """ 198 Property target used to get the split size. 199 """ 200 return self._splitSize
201 202 sizeLimit = property(_getSizeLimit, _setSizeLimit, None, doc="Size limit, as a ByteQuantity") 203 splitSize = property(_getSplitSize, _setSplitSize, None, doc="Split size, as a ByteQuantity")
204
205 206 ######################################################################## 207 # LocalConfig class definition 208 ######################################################################## 209 210 @total_ordering 211 -class LocalConfig(object):
212 213 """ 214 Class representing this extension's configuration document. 215 216 This is not a general-purpose configuration object like the main Cedar 217 Backup configuration object. Instead, it just knows how to parse and emit 218 split-specific configuration values. Third parties who need to read and 219 write configuration related to this extension should access it through the 220 constructor, C{validate} and C{addConfig} methods. 221 222 @note: Lists within this class are "unordered" for equality comparisons. 223 224 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, split, 225 validate, addConfig 226 """ 227
228 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
229 """ 230 Initializes a configuration object. 231 232 If you initialize the object without passing either C{xmlData} or 233 C{xmlPath} then configuration will be empty and will be invalid until it 234 is filled in properly. 235 236 No reference to the original XML data or original path is saved off by 237 this class. Once the data has been parsed (successfully or not) this 238 original information is discarded. 239 240 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 241 method will be called (with its default arguments) against configuration 242 after successfully parsing any passed-in XML. Keep in mind that even if 243 C{validate} is C{False}, it might not be possible to parse the passed-in 244 XML document if lower-level validations fail. 245 246 @note: It is strongly suggested that the C{validate} option always be set 247 to C{True} (the default) unless there is a specific need to read in 248 invalid configuration from disk. 249 250 @param xmlData: XML data representing configuration. 251 @type xmlData: String data. 252 253 @param xmlPath: Path to an XML file on disk. 254 @type xmlPath: Absolute path to a file on disk. 255 256 @param validate: Validate the document after parsing it. 257 @type validate: Boolean true/false. 258 259 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 260 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 261 @raise ValueError: If the parsed configuration document is not valid. 262 """ 263 self._split = None 264 self.split = None 265 if xmlData is not None and xmlPath is not None: 266 raise ValueError("Use either xmlData or xmlPath, but not both.") 267 if xmlData is not None: 268 self._parseXmlData(xmlData) 269 if validate: 270 self.validate() 271 elif xmlPath is not None: 272 with open(xmlPath) as f: 273 xmlData = f.read() 274 self._parseXmlData(xmlData) 275 if validate: 276 self.validate()
277
278 - def __repr__(self):
279 """ 280 Official string representation for class instance. 281 """ 282 return "LocalConfig(%s)" % (self.split)
283
284 - def __str__(self):
285 """ 286 Informal string representation for class instance. 287 """ 288 return self.__repr__()
289
290 - def __eq__(self, other):
291 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 292 return self.__cmp__(other) == 0
293
294 - def __lt__(self, other):
295 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 296 return self.__cmp__(other) < 0
297
298 - def __gt__(self, other):
299 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 300 return self.__cmp__(other) > 0
301
302 - def __cmp__(self, other):
303 """ 304 Original Python 2 comparison operator. 305 Lists within this class are "unordered" for equality comparisons. 306 @param other: Other object to compare to. 307 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 308 """ 309 if other is None: 310 return 1 311 if self.split != other.split: 312 if self.split < other.split: 313 return -1 314 else: 315 return 1 316 return 0
317
318 - def _setSplit(self, value):
319 """ 320 Property target used to set the split configuration value. 321 If not C{None}, the value must be a C{SplitConfig} object. 322 @raise ValueError: If the value is not a C{SplitConfig} 323 """ 324 if value is None: 325 self._split = None 326 else: 327 if not isinstance(value, SplitConfig): 328 raise ValueError("Value must be a C{SplitConfig} object.") 329 self._split = value
330
331 - def _getSplit(self):
332 """ 333 Property target used to get the split configuration value. 334 """ 335 return self._split
336 337 split = property(_getSplit, _setSplit, None, "Split configuration in terms of a C{SplitConfig} object.") 338
339 - def validate(self):
340 """ 341 Validates configuration represented by the object. 342 343 Split configuration must be filled in. Within that, both the size limit 344 and split size must be filled in. 345 346 @raise ValueError: If one of the validations fails. 347 """ 348 if self.split is None: 349 raise ValueError("Split section is required.") 350 if self.split.sizeLimit is None: 351 raise ValueError("Size limit must be set.") 352 if self.split.splitSize is None: 353 raise ValueError("Split size must be set.")
354
355 - def addConfig(self, xmlDom, parentNode):
356 """ 357 Adds a <split> configuration section as the next child of a parent. 358 359 Third parties should use this function to write configuration related to 360 this extension. 361 362 We add the following fields to the document:: 363 364 sizeLimit //cb_config/split/size_limit 365 splitSize //cb_config/split/split_size 366 367 @param xmlDom: DOM tree as from C{impl.createDocument()}. 368 @param parentNode: Parent that the section should be appended to. 369 """ 370 if self.split is not None: 371 sectionNode = addContainerNode(xmlDom, parentNode, "split") 372 addByteQuantityNode(xmlDom, sectionNode, "size_limit", self.split.sizeLimit) 373 addByteQuantityNode(xmlDom, sectionNode, "split_size", self.split.splitSize)
374
375 - def _parseXmlData(self, xmlData):
376 """ 377 Internal method to parse an XML string into the object. 378 379 This method parses the XML document into a DOM tree (C{xmlDom}) and then 380 calls a static method to parse the split configuration section. 381 382 @param xmlData: XML data to be parsed 383 @type xmlData: String data 384 385 @raise ValueError: If the XML cannot be successfully parsed. 386 """ 387 (xmlDom, parentNode) = createInputDom(xmlData) 388 self._split = LocalConfig._parseSplit(parentNode)
389 390 @staticmethod
391 - def _parseSplit(parent):
392 """ 393 Parses an split configuration section. 394 395 We read the following individual fields:: 396 397 sizeLimit //cb_config/split/size_limit 398 splitSize //cb_config/split/split_size 399 400 @param parent: Parent node to search beneath. 401 402 @return: C{EncryptConfig} object or C{None} if the section does not exist. 403 @raise ValueError: If some filled-in value is invalid. 404 """ 405 split = None 406 section = readFirstChild(parent, "split") 407 if section is not None: 408 split = SplitConfig() 409 split.sizeLimit = readByteQuantity(section, "size_limit") 410 split.splitSize = readByteQuantity(section, "split_size") 411 return split
412
413 414 ######################################################################## 415 # Public functions 416 ######################################################################## 417 418 ########################### 419 # executeAction() function 420 ########################### 421 422 # pylint: disable=W0613 423 -def executeAction(configPath, options, config):
424 """ 425 Executes the split backup action. 426 427 @param configPath: Path to configuration file on disk. 428 @type configPath: String representing a path on disk. 429 430 @param options: Program command-line options. 431 @type options: Options object. 432 433 @param config: Program configuration. 434 @type config: Config object. 435 436 @raise ValueError: Under many generic error conditions 437 @raise IOError: If there are I/O problems reading or writing files 438 """ 439 logger.debug("Executing split extended action.") 440 if config.options is None or config.stage is None: 441 raise ValueError("Cedar Backup configuration is not properly filled in.") 442 local = LocalConfig(xmlPath=configPath) 443 dailyDirs = findDailyDirs(config.stage.targetDir, SPLIT_INDICATOR) 444 for dailyDir in dailyDirs: 445 _splitDailyDir(dailyDir, local.split.sizeLimit, local.split.splitSize, 446 config.options.backupUser, config.options.backupGroup) 447 writeIndicatorFile(dailyDir, SPLIT_INDICATOR, config.options.backupUser, config.options.backupGroup) 448 logger.info("Executed the split extended action successfully.")
449
450 451 ############################## 452 # _splitDailyDir() function 453 ############################## 454 455 -def _splitDailyDir(dailyDir, sizeLimit, splitSize, backupUser, backupGroup):
456 """ 457 Splits large files in a daily staging directory. 458 459 Files that match INDICATOR_PATTERNS (i.e. C{"cback.store"}, 460 C{"cback.stage"}, etc.) are assumed to be indicator files and are ignored. 461 All other files are split. 462 463 @param dailyDir: Daily directory to encrypt 464 @param sizeLimit: Size limit, in bytes 465 @param splitSize: Split size, in bytes 466 @param backupUser: User that target files should be owned by 467 @param backupGroup: Group that target files should be owned by 468 469 @raise ValueError: If the encrypt mode is not supported. 470 @raise ValueError: If the daily staging directory does not exist. 471 """ 472 logger.debug("Begin splitting contents of [%s].", dailyDir) 473 fileList = getBackupFiles(dailyDir) # ignores indicator files 474 for path in fileList: 475 size = float(os.stat(path).st_size) 476 if size > sizeLimit: 477 _splitFile(path, splitSize, backupUser, backupGroup, removeSource=True) 478 logger.debug("Completed splitting contents of [%s].", dailyDir)
479
480 481 ######################## 482 # _splitFile() function 483 ######################## 484 485 -def _splitFile(sourcePath, splitSize, backupUser, backupGroup, removeSource=False):
486 """ 487 Splits the source file into chunks of the indicated size. 488 489 The split files will be owned by the indicated backup user and group. If 490 C{removeSource} is C{True}, then the source file will be removed after it is 491 successfully split. 492 493 @param sourcePath: Absolute path of the source file to split 494 @param splitSize: Encryption mode (only "gpg" is allowed) 495 @param backupUser: User that target files should be owned by 496 @param backupGroup: Group that target files should be owned by 497 @param removeSource: Indicates whether to remove the source file 498 499 @raise IOError: If there is a problem accessing, splitting or removing the source file. 500 """ 501 cwd = os.getcwd() 502 try: 503 if not os.path.exists(sourcePath): 504 raise ValueError("Source path [%s] does not exist." % sourcePath) 505 dirname = os.path.dirname(sourcePath) 506 filename = os.path.basename(sourcePath) 507 prefix = "%s_" % filename 508 bytes = int(splitSize.bytes) # pylint: disable=W0622 509 os.chdir(dirname) # need to operate from directory that we want files written to 510 command = resolveCommand(SPLIT_COMMAND) 511 args = [ "--verbose", "--numeric-suffixes", "--suffix-length=5", "--bytes=%d" % bytes, filename, prefix, ] 512 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=False) 513 if result != 0: 514 raise IOError("Error [%d] calling split for [%s]." % (result, sourcePath)) 515 pattern = re.compile(r"(creating file [`'])(%s)(.*)(')" % prefix) 516 match = pattern.search(output[-1:][0]) 517 if match is None: 518 raise IOError("Unable to parse output from split command.") 519 value = int(match.group(3).strip()) 520 for index in range(0, value): 521 path = "%s%05d" % (prefix, index) 522 if not os.path.exists(path): 523 raise IOError("After call to split, expected file [%s] does not exist." % path) 524 changeOwnership(path, backupUser, backupGroup) 525 if removeSource: 526 if os.path.exists(sourcePath): 527 try: 528 os.remove(sourcePath) 529 logger.debug("Completed removing old file [%s].", sourcePath) 530 except: 531 raise IOError("Failed to remove file [%s] after splitting it." % (sourcePath)) 532 finally: 533 os.chdir(cwd)
534