Package CedarBackup3 :: Module util
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.util

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2004-2008,2010,2015 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # Portions copyright (c) 2001, 2002 Python Software Foundation. 
  15  # All Rights Reserved. 
  16  # 
  17  # This program is free software; you can redistribute it and/or 
  18  # modify it under the terms of the GNU General Public License, 
  19  # Version 2, as published by the Free Software Foundation. 
  20  # 
  21  # This program is distributed in the hope that it will be useful, 
  22  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  23  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  24  # 
  25  # Copies of the GNU General Public License are available from 
  26  # the Free Software Foundation website, http://www.gnu.org/. 
  27  # 
  28  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  29  # 
  30  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  31  # Language : Python 3 (>= 3.4) 
  32  # Project  : Cedar Backup, release 3 
  33  # Purpose  : Provides general-purpose utilities. 
  34  # 
  35  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  36   
  37  ######################################################################## 
  38  # Module documentation 
  39  ######################################################################## 
  40   
  41  """ 
  42  Provides general-purpose utilities. 
  43   
  44  @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList, 
  45         RegexList, _Vertex, DirectedGraph, PathResolverSingleton, 
  46         sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine, 
  47         resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice, 
  48         deriveDayOfWeek, isStartOfWeek, buildNormalizedPath, 
  49         ISO_SECTOR_SIZE, BYTES_PER_SECTOR, 
  50         BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE, 
  51         SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY, 
  52         UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS 
  53   
  54  @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes. 
  55  @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector. 
  56  @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB). 
  57  @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB). 
  58  @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB). 
  59  @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB). 
  60  @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB). 
  61  @var SECONDS_PER_MINUTE: Number of seconds per minute. 
  62  @var MINUTES_PER_HOUR: Number of minutes per hour. 
  63  @var HOURS_PER_DAY: Number of hours per day. 
  64  @var SECONDS_PER_DAY: Number of seconds per day. 
  65  @var UNIT_BYTES: Constant representing the byte (B) unit for conversion. 
  66  @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion. 
  67  @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion. 
  68  @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion. 
  69  @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion. 
  70   
  71  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  72  """ 
  73   
  74   
  75  ######################################################################## 
  76  # Imported modules 
  77  ######################################################################## 
  78   
  79  import sys 
  80  import math 
  81  import os 
  82  import re 
  83  import time 
  84  import logging 
  85  from subprocess import Popen, STDOUT, PIPE 
  86  from functools import total_ordering 
  87  from numbers import Real 
  88  from decimal import Decimal 
  89  import collections 
  90   
  91  try: 
  92     import pwd 
  93     import grp 
  94     _UID_GID_AVAILABLE = True 
  95  except ImportError: 
  96     _UID_GID_AVAILABLE = False 
  97   
  98  from CedarBackup3.release import VERSION, DATE 
  99   
 100   
 101  ######################################################################## 
 102  # Module-wide constants and variables 
 103  ######################################################################## 
 104   
 105  logger = logging.getLogger("CedarBackup3.log.util") 
 106  outputLogger = logging.getLogger("CedarBackup3.output") 
 107   
 108  ISO_SECTOR_SIZE    = 2048.0   # in bytes 
 109  BYTES_PER_SECTOR   = ISO_SECTOR_SIZE 
 110   
 111  BYTES_PER_KBYTE    = 1024.0 
 112  KBYTES_PER_MBYTE   = 1024.0 
 113  MBYTES_PER_GBYTE   = 1024.0 
 114  BYTES_PER_MBYTE    = BYTES_PER_KBYTE * KBYTES_PER_MBYTE 
 115  BYTES_PER_GBYTE    = BYTES_PER_MBYTE * MBYTES_PER_GBYTE 
 116   
 117  SECONDS_PER_MINUTE = 60.0 
 118  MINUTES_PER_HOUR   = 60.0 
 119  HOURS_PER_DAY      = 24.0 
 120  SECONDS_PER_DAY    = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY 
 121   
 122  UNIT_BYTES         = 0 
 123  UNIT_KBYTES        = 1 
 124  UNIT_MBYTES        = 2 
 125  UNIT_GBYTES        = 4 
 126  UNIT_SECTORS       = 3 
 127   
 128  MTAB_FILE          = "/etc/mtab" 
 129   
 130  MOUNT_COMMAND      = [ "mount", ] 
 131  UMOUNT_COMMAND     = [ "umount", ] 
 132   
 133  DEFAULT_LANGUAGE   = "C" 
 134  LANG_VAR           = "LANG" 
 135  LOCALE_VARS        = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE", 
 136                         "LC_CTYPE", "LC_IDENTIFICATION", 
 137                         "LC_MEASUREMENT", "LC_MESSAGES", 
 138                         "LC_MONETARY", "LC_NAME", "LC_NUMERIC", 
 139                         "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ] 
140 141 142 ######################################################################## 143 # UnorderedList class definition 144 ######################################################################## 145 146 -class UnorderedList(list):
147 148 """ 149 Class representing an "unordered list". 150 151 An "unordered list" is a list in which only the contents matter, not the 152 order in which the contents appear in the list. 153 154 For instance, we might be keeping track of set of paths in a list, because 155 it's convenient to have them in that form. However, for comparison 156 purposes, we would only care that the lists contain exactly the same 157 contents, regardless of order. 158 159 I have come up with two reasonable ways of doing this, plus a couple more 160 that would work but would be a pain to implement. My first method is to 161 copy and sort each list, comparing the sorted versions. This will only work 162 if two lists with exactly the same members are guaranteed to sort in exactly 163 the same order. The second way would be to create two Sets and then compare 164 the sets. However, this would lose information about any duplicates in 165 either list. I've decided to go with option #1 for now. I'll modify this 166 code if I run into problems in the future. 167 168 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__}, 169 C{__le__} and C{__lt__} list methods to change the definition of the various 170 comparison operators. In all cases, the comparison is changed to return the 171 result of the original operation I{but instead comparing sorted lists}. 172 This is going to be quite a bit slower than a normal list, so you probably 173 only want to use it on small lists. 174 """ 175
176 - def __eq__(self, other):
177 """ 178 Definition of C{==} operator for this class. 179 @param other: Other object to compare to. 180 @return: True/false depending on whether C{self == other}. 181 """ 182 if other is None: 183 return False 184 selfSorted = UnorderedList.mixedsort(self[:]) 185 otherSorted = UnorderedList.mixedsort(other[:]) 186 return selfSorted.__eq__(otherSorted)
187
188 - def __ne__(self, other):
189 """ 190 Definition of C{!=} operator for this class. 191 @param other: Other object to compare to. 192 @return: True/false depending on whether C{self != other}. 193 """ 194 if other is None: 195 return True 196 selfSorted = UnorderedList.mixedsort(self[:]) 197 otherSorted = UnorderedList.mixedsort(other[:]) 198 return selfSorted.__ne__(otherSorted)
199
200 - def __ge__(self, other):
201 """ 202 Definition of S{>=} operator for this class. 203 @param other: Other object to compare to. 204 @return: True/false depending on whether C{self >= other}. 205 """ 206 if other is None: 207 return True 208 selfSorted = UnorderedList.mixedsort(self[:]) 209 otherSorted = UnorderedList.mixedsort(other[:]) 210 return selfSorted.__ge__(otherSorted)
211
212 - def __gt__(self, other):
213 """ 214 Definition of C{>} operator for this class. 215 @param other: Other object to compare to. 216 @return: True/false depending on whether C{self > other}. 217 """ 218 if other is None: 219 return True 220 selfSorted = UnorderedList.mixedsort(self[:]) 221 otherSorted = UnorderedList.mixedsort(other[:]) 222 return selfSorted.__gt__(otherSorted)
223
224 - def __le__(self, other):
225 """ 226 Definition of S{<=} operator for this class. 227 @param other: Other object to compare to. 228 @return: True/false depending on whether C{self <= other}. 229 """ 230 if other is None: 231 return False 232 selfSorted = UnorderedList.mixedsort(self[:]) 233 otherSorted = UnorderedList.mixedsort(other[:]) 234 return selfSorted.__le__(otherSorted)
235
236 - def __lt__(self, other):
237 """ 238 Definition of C{<} operator for this class. 239 @param other: Other object to compare to. 240 @return: True/false depending on whether C{self < other}. 241 """ 242 if other is None: 243 return False 244 selfSorted = UnorderedList.mixedsort(self[:]) 245 otherSorted = UnorderedList.mixedsort(other[:]) 246 return selfSorted.__lt__(otherSorted)
247 248 @staticmethod
249 - def mixedsort(value):
250 """ 251 Sort a list, making sure we don't blow up if the list happens to include mixed values. 252 @see: http://stackoverflow.com/questions/26575183/how-can-i-get-2-x-like-sorting-behaviour-in-python-3-x 253 """ 254 return sorted(value, key=UnorderedList.mixedkey)
255 256 @staticmethod 257 #pylint: disable=R0204
258 - def mixedkey(value):
259 """Provide a key for use by mixedsort()""" 260 numeric = Real, Decimal 261 if isinstance(value, numeric): 262 typeinfo = numeric 263 else: 264 typeinfo = type(value) 265 try: 266 x = value < value 267 except TypeError: 268 value = repr(value) 269 return repr(typeinfo), value
270
271 272 ######################################################################## 273 # AbsolutePathList class definition 274 ######################################################################## 275 276 -class AbsolutePathList(UnorderedList):
277 278 """ 279 Class representing a list of absolute paths. 280 281 This is an unordered list. 282 283 We override the C{append}, C{insert} and C{extend} methods to ensure that 284 any item added to the list is an absolute path. 285 286 Each item added to the list is encoded using L{encodePath}. If we don't do 287 this, we have problems trying certain operations between strings and unicode 288 objects, particularly for "odd" filenames that can't be encoded in standard 289 ASCII. 290 """ 291
292 - def append(self, item):
293 """ 294 Overrides the standard C{append} method. 295 @raise ValueError: If item is not an absolute path. 296 """ 297 if not os.path.isabs(item): 298 raise ValueError("Not an absolute path: [%s]" % item) 299 list.append(self, encodePath(item))
300
301 - def insert(self, index, item):
302 """ 303 Overrides the standard C{insert} method. 304 @raise ValueError: If item is not an absolute path. 305 """ 306 if not os.path.isabs(item): 307 raise ValueError("Not an absolute path: [%s]" % item) 308 list.insert(self, index, encodePath(item))
309
310 - def extend(self, seq):
311 """ 312 Overrides the standard C{insert} method. 313 @raise ValueError: If any item is not an absolute path. 314 """ 315 for item in seq: 316 if not os.path.isabs(item): 317 raise ValueError("Not an absolute path: [%s]" % item) 318 for item in seq: 319 list.append(self, encodePath(item))
320
321 322 ######################################################################## 323 # ObjectTypeList class definition 324 ######################################################################## 325 326 -class ObjectTypeList(UnorderedList):
327 328 """ 329 Class representing a list containing only objects with a certain type. 330 331 This is an unordered list. 332 333 We override the C{append}, C{insert} and C{extend} methods to ensure that 334 any item added to the list matches the type that is requested. The 335 comparison uses the built-in C{isinstance}, which should allow subclasses of 336 of the requested type to be added to the list as well. 337 338 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a 339 CollectDir object."} if C{objectName} is C{"CollectDir"}. 340 """ 341
342 - def __init__(self, objectType, objectName):
343 """ 344 Initializes a typed list for a particular type. 345 @param objectType: Type that the list elements must match. 346 @param objectName: Short string containing the "name" of the type. 347 """ 348 super(ObjectTypeList, self).__init__() 349 self.objectType = objectType 350 self.objectName = objectName
351
352 - def append(self, item):
353 """ 354 Overrides the standard C{append} method. 355 @raise ValueError: If item does not match requested type. 356 """ 357 if not isinstance(item, self.objectType): 358 raise ValueError("Item must be a %s object." % self.objectName) 359 list.append(self, item)
360
361 - def insert(self, index, item):
362 """ 363 Overrides the standard C{insert} method. 364 @raise ValueError: If item does not match requested type. 365 """ 366 if not isinstance(item, self.objectType): 367 raise ValueError("Item must be a %s object." % self.objectName) 368 list.insert(self, index, item)
369
370 - def extend(self, seq):
371 """ 372 Overrides the standard C{insert} method. 373 @raise ValueError: If item does not match requested type. 374 """ 375 for item in seq: 376 if not isinstance(item, self.objectType): 377 raise ValueError("All items must be %s objects." % self.objectName) 378 list.extend(self, seq)
379
380 381 ######################################################################## 382 # RestrictedContentList class definition 383 ######################################################################## 384 385 -class RestrictedContentList(UnorderedList):
386 387 """ 388 Class representing a list containing only object with certain values. 389 390 This is an unordered list. 391 392 We override the C{append}, C{insert} and C{extend} methods to ensure that 393 any item added to the list is among the valid values. We use a standard 394 comparison, so pretty much anything can be in the list of valid values. 395 396 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be 397 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}. 398 399 @note: This class doesn't make any attempt to trap for nonsensical 400 arguments. All of the values in the values list should be of the same type 401 (i.e. strings). Then, all list operations also need to be of that type 402 (i.e. you should always insert or append just strings). If you mix types -- 403 for instance lists and strings -- you will likely see AttributeError 404 exceptions or other problems. 405 """ 406
407 - def __init__(self, valuesList, valuesDescr, prefix=None):
408 """ 409 Initializes a list restricted to containing certain values. 410 @param valuesList: List of valid values. 411 @param valuesDescr: Short string describing list of values. 412 @param prefix: Prefix to use in error messages (None results in prefix "Item") 413 """ 414 super(RestrictedContentList, self).__init__() 415 self.prefix = "Item" 416 if prefix is not None: self.prefix = prefix 417 self.valuesList = valuesList 418 self.valuesDescr = valuesDescr
419
420 - def append(self, item):
421 """ 422 Overrides the standard C{append} method. 423 @raise ValueError: If item is not in the values list. 424 """ 425 if item not in self.valuesList: 426 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 427 list.append(self, item)
428
429 - def insert(self, index, item):
430 """ 431 Overrides the standard C{insert} method. 432 @raise ValueError: If item is not in the values list. 433 """ 434 if item not in self.valuesList: 435 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 436 list.insert(self, index, item)
437
438 - def extend(self, seq):
439 """ 440 Overrides the standard C{insert} method. 441 @raise ValueError: If item is not in the values list. 442 """ 443 for item in seq: 444 if item not in self.valuesList: 445 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 446 list.extend(self, seq)
447
448 449 ######################################################################## 450 # RegexMatchList class definition 451 ######################################################################## 452 453 -class RegexMatchList(UnorderedList):
454 455 """ 456 Class representing a list containing only strings that match a regular expression. 457 458 If C{emptyAllowed} is passed in as C{False}, then empty strings are 459 explicitly disallowed, even if they happen to match the regular expression. 460 (C{None} values are always disallowed, since string operations are not 461 permitted on C{None}.) 462 463 This is an unordered list. 464 465 We override the C{append}, C{insert} and C{extend} methods to ensure that 466 any item added to the list matches the indicated regular expression. 467 468 @note: If you try to put values that are not strings into the list, you will 469 likely get either TypeError or AttributeError exceptions as a result. 470 """ 471
472 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
473 """ 474 Initializes a list restricted to containing certain values. 475 @param valuesRegex: Regular expression that must be matched, as a string 476 @param emptyAllowed: Indicates whether empty or None values are allowed. 477 @param prefix: Prefix to use in error messages (None results in prefix "Item") 478 """ 479 super(RegexMatchList, self).__init__() 480 self.prefix = "Item" 481 if prefix is not None: self.prefix = prefix 482 self.valuesRegex = valuesRegex 483 self.emptyAllowed = emptyAllowed 484 self.pattern = re.compile(self.valuesRegex)
485
486 - def append(self, item):
487 """ 488 Overrides the standard C{append} method. 489 @raise ValueError: If item is None 490 @raise ValueError: If item is empty and empty values are not allowed 491 @raise ValueError: If item does not match the configured regular expression 492 """ 493 if item is None or (not self.emptyAllowed and item == ""): 494 raise ValueError("%s cannot be empty." % self.prefix) 495 if not self.pattern.search(item): 496 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 497 list.append(self, item)
498
499 - def insert(self, index, item):
500 """ 501 Overrides the standard C{insert} method. 502 @raise ValueError: If item is None 503 @raise ValueError: If item is empty and empty values are not allowed 504 @raise ValueError: If item does not match the configured regular expression 505 """ 506 if item is None or (not self.emptyAllowed and item == ""): 507 raise ValueError("%s cannot be empty." % self.prefix) 508 if not self.pattern.search(item): 509 raise ValueError("%s is not valid [%s]" % (self.prefix, item)) 510 list.insert(self, index, item)
511
512 - def extend(self, seq):
513 """ 514 Overrides the standard C{insert} method. 515 @raise ValueError: If any item is None 516 @raise ValueError: If any item is empty and empty values are not allowed 517 @raise ValueError: If any item does not match the configured regular expression 518 """ 519 for item in seq: 520 if item is None or (not self.emptyAllowed and item == ""): 521 raise ValueError("%s cannot be empty." % self.prefix) 522 if not self.pattern.search(item): 523 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 524 list.extend(self, seq)
525
526 527 ######################################################################## 528 # RegexList class definition 529 ######################################################################## 530 531 -class RegexList(UnorderedList):
532 533 """ 534 Class representing a list of valid regular expression strings. 535 536 This is an unordered list. 537 538 We override the C{append}, C{insert} and C{extend} methods to ensure that 539 any item added to the list is a valid regular expression. 540 """ 541
542 - def append(self, item):
543 """ 544 Overrides the standard C{append} method. 545 @raise ValueError: If item is not an absolute path. 546 """ 547 try: 548 re.compile(item) 549 except re.error: 550 raise ValueError("Not a valid regular expression: [%s]" % item) 551 list.append(self, item)
552
553 - def insert(self, index, item):
554 """ 555 Overrides the standard C{insert} method. 556 @raise ValueError: If item is not an absolute path. 557 """ 558 try: 559 re.compile(item) 560 except re.error: 561 raise ValueError("Not a valid regular expression: [%s]" % item) 562 list.insert(self, index, item)
563
564 - def extend(self, seq):
565 """ 566 Overrides the standard C{insert} method. 567 @raise ValueError: If any item is not an absolute path. 568 """ 569 for item in seq: 570 try: 571 re.compile(item) 572 except re.error: 573 raise ValueError("Not a valid regular expression: [%s]" % item) 574 for item in seq: 575 list.append(self, item)
576
577 578 ######################################################################## 579 # Directed graph implementation 580 ######################################################################## 581 582 -class _Vertex(object):
583 584 """ 585 Represents a vertex (or node) in a directed graph. 586 """ 587
588 - def __init__(self, name):
589 """ 590 Constructor. 591 @param name: Name of this graph vertex. 592 @type name: String value. 593 """ 594 self.name = name 595 self.endpoints = [] 596 self.state = None
597
598 @total_ordering 599 -class DirectedGraph(object):
600 601 """ 602 Represents a directed graph. 603 604 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set 605 B{E} of vertex pairs or edges. In a directed graph, each edge also has an 606 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph} 607 object provides a way to construct a directed graph and execute a depth- 608 first search. 609 610 This data structure was designed based on the graphing chapter in 611 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>}, 612 by Steven S. Skiena. 613 614 This class is intended to be used by Cedar Backup for dependency ordering. 615 Because of this, it's not quite general-purpose. Unlike a "general" graph, 616 every vertex in this graph has at least one edge pointing to it, from a 617 special "start" vertex. This is so no vertices get "lost" either because 618 they have no dependencies or because nothing depends on them. 619 """ 620 621 _UNDISCOVERED = 0 622 _DISCOVERED = 1 623 _EXPLORED = 2 624
625 - def __init__(self, name):
626 """ 627 Directed graph constructor. 628 629 @param name: Name of this graph. 630 @type name: String value. 631 """ 632 if name is None or name == "": 633 raise ValueError("Graph name must be non-empty.") 634 self._name = name 635 self._vertices = {} 636 self._startVertex = _Vertex(None) # start vertex is only vertex with no name
637
638 - def __repr__(self):
639 """ 640 Official string representation for class instance. 641 """ 642 return "DirectedGraph(%s)" % self.name
643
644 - def __str__(self):
645 """ 646 Informal string representation for class instance. 647 """ 648 return self.__repr__()
649
650 - def __eq__(self, other):
651 """Equals operator, implemented in terms of original Python 2 compare operator.""" 652 return self.__cmp__(other) == 0
653
654 - def __lt__(self, other):
655 """Less-than operator, implemented in terms of original Python 2 compare operator.""" 656 return self.__cmp__(other) < 0
657
658 - def __gt__(self, other):
659 """Greater-than operator, implemented in terms of original Python 2 compare operator.""" 660 return self.__cmp__(other) > 0
661
662 - def __cmp__(self, other):
663 """ 664 Original Python 2 comparison operator. 665 @param other: Other object to compare to. 666 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 667 """ 668 # pylint: disable=W0212 669 if other is None: 670 return 1 671 if self.name != other.name: 672 if str(self.name or "") < str(other.name or ""): 673 return -1 674 else: 675 return 1 676 if self._vertices != other._vertices: 677 if self._vertices < other._vertices: 678 return -1 679 else: 680 return 1 681 return 0
682
683 - def _getName(self):
684 """ 685 Property target used to get the graph name. 686 """ 687 return self._name
688 689 name = property(_getName, None, None, "Name of the graph.") 690
691 - def createVertex(self, name):
692 """ 693 Creates a named vertex. 694 @param name: vertex name 695 @raise ValueError: If the vertex name is C{None} or empty. 696 """ 697 if name is None or name == "": 698 raise ValueError("Vertex name must be non-empty.") 699 vertex = _Vertex(name) 700 self._startVertex.endpoints.append(vertex) # so every vertex is connected at least once 701 self._vertices[name] = vertex
702
703 - def createEdge(self, start, finish):
704 """ 705 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex. 706 @param start: Name of start vertex. 707 @param finish: Name of finish vertex. 708 @raise ValueError: If one of the named vertices is unknown. 709 """ 710 try: 711 startVertex = self._vertices[start] 712 finishVertex = self._vertices[finish] 713 startVertex.endpoints.append(finishVertex) 714 except KeyError as e: 715 raise ValueError("Vertex [%s] could not be found." % e)
716
717 - def topologicalSort(self):
718 """ 719 Implements a topological sort of the graph. 720 721 This method also enforces that the graph is a directed acyclic graph, 722 which is a requirement of a topological sort. 723 724 A directed acyclic graph (or "DAG") is a directed graph with no directed 725 cycles. A topological sort of a DAG is an ordering on the vertices such 726 that all edges go from left to right. Only an acyclic graph can have a 727 topological sort, but any DAG has at least one topological sort. 728 729 Since a topological sort only makes sense for an acyclic graph, this 730 method throws an exception if a cycle is found. 731 732 A depth-first search only makes sense if the graph is acyclic. If the 733 graph contains any cycles, it is not possible to determine a consistent 734 ordering for the vertices. 735 736 @note: If a particular vertex has no edges, then its position in the 737 final list depends on the order in which the vertices were created in the 738 graph. If you're using this method to determine a dependency order, this 739 makes sense: a vertex with no dependencies can go anywhere (and will). 740 741 @return: Ordering on the vertices so that all edges go from left to right. 742 743 @raise ValueError: If a cycle is found in the graph. 744 """ 745 ordering = [] 746 for key in self._vertices: 747 vertex = self._vertices[key] 748 vertex.state = self._UNDISCOVERED 749 for key in self._vertices: 750 vertex = self._vertices[key] 751 if vertex.state == self._UNDISCOVERED: 752 self._topologicalSort(self._startVertex, ordering) 753 return ordering
754
755 - def _topologicalSort(self, vertex, ordering):
756 """ 757 Recursive depth first search function implementing topological sort. 758 @param vertex: Vertex to search 759 @param ordering: List of vertices in proper order 760 """ 761 vertex.state = self._DISCOVERED 762 for endpoint in vertex.endpoints: 763 if endpoint.state == self._UNDISCOVERED: 764 self._topologicalSort(endpoint, ordering) 765 elif endpoint.state != self._EXPLORED: 766 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name)) 767 if vertex.name is not None: 768 ordering.insert(0, vertex.name) 769 vertex.state = self._EXPLORED
770
771 772 ######################################################################## 773 # PathResolverSingleton class definition 774 ######################################################################## 775 776 -class PathResolverSingleton(object):
777 778 """ 779 Singleton used for resolving executable paths. 780 781 Various functions throughout Cedar Backup (including extensions) need a way 782 to resolve the path of executables that they use. For instance, the image 783 functionality needs to find the C{mkisofs} executable, and the Subversion 784 extension needs to find the C{svnlook} executable. Cedar Backup's original 785 behavior was to assume that the simple name (C{"svnlook"} or whatever) was 786 available on the caller's C{$PATH}, and to fail otherwise. However, this 787 turns out to be less than ideal, since for instance the root user might not 788 always have executables like C{svnlook} in its path. 789 790 One solution is to specify a path (either via an absolute path or some sort 791 of path insertion or path appending mechanism) that would apply to the 792 C{executeCommand()} function. This is not difficult to implement, but it 793 seem like kind of a "big hammer" solution. Besides that, it might also 794 represent a security flaw (for instance, I prefer not to mess with root's 795 C{$PATH} on the application level if I don't have to). 796 797 The alternative is to set up some sort of configuration for the path to 798 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or 799 whatever. This PathResolverSingleton aims to provide a good solution to the 800 mapping problem. Callers of all sorts (extensions or not) can get an 801 instance of the singleton. Then, they call the C{lookup} method to try and 802 resolve the executable they are looking for. Through the C{lookup} method, 803 the caller can also specify a default to use if a mapping is not found. 804 This way, with no real effort on the part of the caller, behavior can neatly 805 degrade to something equivalent to the current behavior if there is no 806 special mapping or if the singleton was never initialized in the first 807 place. 808 809 Even better, extensions automagically get access to the same resolver 810 functionality, and they don't even need to understand how the mapping 811 happens. All extension authors need to do is document what executables 812 their code requires, and the standard resolver configuration section will 813 meet their needs. 814 815 The class should be initialized once through the constructor somewhere in 816 the main routine. Then, the main routine should call the L{fill} method to 817 fill in the resolver's internal structures. Everyone else who needs to 818 resolve a path will get an instance of the class using L{getInstance} and 819 will then just call the L{lookup} method. 820 821 @cvar _instance: Holds a reference to the singleton 822 @ivar _mapping: Internal mapping from resource name to path. 823 """ 824 825 _instance = None # Holds a reference to singleton instance 826
827 - class _Helper:
828 """Helper class to provide a singleton factory method."""
829 - def __init__(self):
830 pass
831 - def __call__(self, *args, **kw):
832 # pylint: disable=W0212,R0201 833 if PathResolverSingleton._instance is None: 834 obj = PathResolverSingleton() 835 PathResolverSingleton._instance = obj 836 return PathResolverSingleton._instance
837 838 getInstance = _Helper() # Method that callers will use to get an instance 839
840 - def __init__(self, ):
841 """Singleton constructor, which just creates the singleton instance.""" 842 PathResolverSingleton._instance = self 843 self._mapping = { }
844
845 - def lookup(self, name, default=None):
846 """ 847 Looks up name and returns the resolved path associated with the name. 848 @param name: Name of the path resource to resolve. 849 @param default: Default to return if resource cannot be resolved. 850 @return: Resolved path associated with name, or default if name can't be resolved. 851 """ 852 value = default 853 if name in list(self._mapping.keys()): 854 value = self._mapping[name] 855 logger.debug("Resolved command [%s] to [%s].", name, value) 856 return value
857
858 - def fill(self, mapping):
859 """ 860 Fills in the singleton's internal mapping from name to resource. 861 @param mapping: Mapping from resource name to path. 862 @type mapping: Dictionary mapping name to path, both as strings. 863 """ 864 self._mapping = { } 865 for key in list(mapping.keys()): 866 self._mapping[key] = mapping[key]
867
868 869 ######################################################################## 870 # Pipe class definition 871 ######################################################################## 872 873 -class Pipe(Popen):
874 """ 875 Specialized pipe class for use by C{executeCommand}. 876 877 The L{executeCommand} function needs a specialized way of interacting 878 with a pipe. First, C{executeCommand} only reads from the pipe, and 879 never writes to it. Second, C{executeCommand} needs a way to discard all 880 output written to C{stderr}, as a means of simulating the shell 881 C{2>/dev/null} construct. 882 """
883 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
884 stderr = STDOUT 885 if ignoreStderr: 886 devnull = nullDevice() 887 stderr = os.open(devnull, os.O_RDWR) 888 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
889
890 891 ######################################################################## 892 # Diagnostics class definition 893 ######################################################################## 894 895 -class Diagnostics(object):
896 897 """ 898 Class holding runtime diagnostic information. 899 900 Diagnostic information is information that is useful to get from users for 901 debugging purposes. I'm consolidating it all here into one object. 902 903 @sort: __init__, __repr__, __str__ 904 """ 905 # pylint: disable=R0201 906
907 - def __init__(self):
908 """ 909 Constructor for the C{Diagnostics} class. 910 """
911
912 - def __repr__(self):
913 """ 914 Official string representation for class instance. 915 """ 916 return "Diagnostics()"
917
918 - def __str__(self):
919 """ 920 Informal string representation for class instance. 921 """ 922 return self.__repr__()
923
924 - def getValues(self):
925 """ 926 Get a map containing all of the diagnostic values. 927 @return: Map from diagnostic name to diagnostic value. 928 """ 929 values = {} 930 values['version'] = self.version 931 values['interpreter'] = self.interpreter 932 values['platform'] = self.platform 933 values['encoding'] = self.encoding 934 values['locale'] = self.locale 935 values['timestamp'] = self.timestamp 936 return values
937
938 - def printDiagnostics(self, fd=sys.stdout, prefix=""):
939 """ 940 Pretty-print diagnostic information to a file descriptor. 941 @param fd: File descriptor used to print information. 942 @param prefix: Prefix string (if any) to place onto printed lines 943 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 944 """ 945 lines = self._buildDiagnosticLines(prefix) 946 for line in lines: 947 fd.write("%s\n" % line)
948
949 - def logDiagnostics(self, method, prefix=""):
950 """ 951 Pretty-print diagnostic information using a logger method. 952 @param method: Logger method to use for logging (i.e. logger.info) 953 @param prefix: Prefix string (if any) to place onto printed lines 954 """ 955 lines = self._buildDiagnosticLines(prefix) 956 for line in lines: 957 method("%s" % line)
958
959 - def _buildDiagnosticLines(self, prefix=""):
960 """ 961 Build a set of pretty-printed diagnostic lines. 962 @param prefix: Prefix string (if any) to place onto printed lines 963 @return: List of strings, not terminated by newlines. 964 """ 965 values = self.getValues() 966 keys = list(values.keys()) 967 keys.sort() 968 tmax = Diagnostics._getMaxLength(keys) + 3 # three extra dots in output 969 lines = [] 970 for key in keys: 971 title = key.title() 972 title += (tmax - len(title)) * '.' 973 value = values[key] 974 line = "%s%s: %s" % (prefix, title, value) 975 lines.append(line) 976 return lines
977 978 @staticmethod
979 - def _getMaxLength(values):
980 """ 981 Get the maximum length from among a list of strings. 982 """ 983 tmax = 0 984 for value in values: 985 if len(value) > tmax: 986 tmax = len(value) 987 return tmax
988
989 - def _getVersion(self):
990 """ 991 Property target to get the Cedar Backup version. 992 """ 993 return "Cedar Backup %s (%s)" % (VERSION, DATE)
994
995 - def _getInterpreter(self):
996 """ 997 Property target to get the Python interpreter version. 998 """ 999 version = sys.version_info 1000 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
1001
1002 - def _getEncoding(self):
1003 """ 1004 Property target to get the filesystem encoding. 1005 """ 1006 return sys.getfilesystemencoding() or sys.getdefaultencoding()
1007
1008 - def _getPlatform(self):
1009 """ 1010 Property target to get the operating system platform. 1011 """ 1012 try: 1013 uname = os.uname() 1014 sysname = uname[0] # i.e. Linux 1015 release = uname[2] # i.e. 2.16.18-2 1016 machine = uname[4] # i.e. i686 1017 return "%s (%s %s %s)" % (sys.platform, sysname, release, machine) 1018 except: 1019 return sys.platform
1020
1021 - def _getLocale(self):
1022 """ 1023 Property target to get the default locale that is in effect. 1024 """ 1025 try: 1026 import locale 1027 return locale.getdefaultlocale()[0] 1028 except: 1029 return "(unknown)"
1030
1031 - def _getTimestamp(self):
1032 """ 1033 Property target to get a current date/time stamp. 1034 """ 1035 try: 1036 import datetime 1037 return datetime.datetime.utcnow().ctime() + " UTC" 1038 except: 1039 return "(unknown)"
1040 1041 version = property(_getVersion, None, None, "Cedar Backup version.") 1042 interpreter = property(_getInterpreter, None, None, "Python interpreter version.") 1043 platform = property(_getPlatform, None, None, "Platform identifying information.") 1044 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.") 1045 locale = property(_getLocale, None, None, "Locale that is in effect.") 1046 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1047
1048 1049 ######################################################################## 1050 # General utility functions 1051 ######################################################################## 1052 1053 ###################### 1054 # sortDict() function 1055 ###################### 1056 1057 -def sortDict(d):
1058 """ 1059 Returns the keys of the dictionary sorted by value. 1060 @param d: Dictionary to operate on 1061 @return: List of dictionary keys sorted in order by dictionary value. 1062 """ 1063 items = list(d.items()) 1064 items.sort(key=lambda x: (x[1], x[0])) # sort by value and then by key 1065 return [key for key, value in items]
1066
1067 1068 ######################## 1069 # removeKeys() function 1070 ######################## 1071 1072 -def removeKeys(d, keys):
1073 """ 1074 Removes all of the keys from the dictionary. 1075 The dictionary is altered in-place. 1076 Each key must exist in the dictionary. 1077 @param d: Dictionary to operate on 1078 @param keys: List of keys to remove 1079 @raise KeyError: If one of the keys does not exist 1080 """ 1081 for key in keys: 1082 del d[key]
1083
1084 1085 ######################### 1086 # convertSize() function 1087 ######################### 1088 1089 -def convertSize(size, fromUnit, toUnit):
1090 """ 1091 Converts a size in one unit to a size in another unit. 1092 1093 This is just a convenience function so that the functionality can be 1094 implemented in just one place. Internally, we convert values to bytes and 1095 then to the final unit. 1096 1097 The available units are: 1098 1099 - C{UNIT_BYTES} - Bytes 1100 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B 1101 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB 1102 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB 1103 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B 1104 1105 @param size: Size to convert 1106 @type size: Integer or float value in units of C{fromUnit} 1107 1108 @param fromUnit: Unit to convert from 1109 @type fromUnit: One of the units listed above 1110 1111 @param toUnit: Unit to convert to 1112 @type toUnit: One of the units listed above 1113 1114 @return: Number converted to new unit, as a float. 1115 @raise ValueError: If one of the units is invalid. 1116 """ 1117 if size is None: 1118 raise ValueError("Cannot convert size of None.") 1119 if fromUnit == UNIT_BYTES: 1120 byteSize = float(size) 1121 elif fromUnit == UNIT_KBYTES: 1122 byteSize = float(size) * BYTES_PER_KBYTE 1123 elif fromUnit == UNIT_MBYTES: 1124 byteSize = float(size) * BYTES_PER_MBYTE 1125 elif fromUnit == UNIT_GBYTES: 1126 byteSize = float(size) * BYTES_PER_GBYTE 1127 elif fromUnit == UNIT_SECTORS: 1128 byteSize = float(size) * BYTES_PER_SECTOR 1129 else: 1130 raise ValueError("Unknown 'from' unit %s." % fromUnit) 1131 if toUnit == UNIT_BYTES: 1132 return byteSize 1133 elif toUnit == UNIT_KBYTES: 1134 return byteSize / BYTES_PER_KBYTE 1135 elif toUnit == UNIT_MBYTES: 1136 return byteSize / BYTES_PER_MBYTE 1137 elif toUnit == UNIT_GBYTES: 1138 return byteSize / BYTES_PER_GBYTE 1139 elif toUnit == UNIT_SECTORS: 1140 return byteSize / BYTES_PER_SECTOR 1141 else: 1142 raise ValueError("Unknown 'to' unit %s." % toUnit)
1143
1144 1145 ########################## 1146 # displayBytes() function 1147 ########################## 1148 1149 -def displayBytes(bytes, digits=2): # pylint: disable=W0622
1150 """ 1151 Format a byte quantity so it can be sensibly displayed. 1152 1153 It's rather difficult to look at a number like "72372224 bytes" and get any 1154 meaningful information out of it. It would be more useful to see something 1155 like "69.02 MB". That's what this function does. Any time you want to display 1156 a byte value, i.e.:: 1157 1158 print "Size: %s bytes" % bytes 1159 1160 Call this function instead:: 1161 1162 print "Size: %s" % displayBytes(bytes) 1163 1164 What comes out will be sensibly formatted. The indicated number of digits 1165 will be listed after the decimal point, rounded based on whatever rules are 1166 used by Python's standard C{%f} string format specifier. (Values less than 1 1167 kB will be listed in bytes and will not have a decimal point, since the 1168 concept of a fractional byte is nonsensical.) 1169 1170 @param bytes: Byte quantity. 1171 @type bytes: Integer number of bytes. 1172 1173 @param digits: Number of digits to display after the decimal point. 1174 @type digits: Integer value, typically 2-5. 1175 1176 @return: String, formatted for sensible display. 1177 """ 1178 if bytes is None: 1179 raise ValueError("Cannot display byte value of None.") 1180 bytes = float(bytes) 1181 if math.fabs(bytes) < BYTES_PER_KBYTE: 1182 fmt = "%.0f bytes" 1183 value = bytes 1184 elif math.fabs(bytes) < BYTES_PER_MBYTE: 1185 fmt = "%." + "%d" % digits + "f kB" 1186 value = bytes / BYTES_PER_KBYTE 1187 elif math.fabs(bytes) < BYTES_PER_GBYTE: 1188 fmt = "%." + "%d" % digits + "f MB" 1189 value = bytes / BYTES_PER_MBYTE 1190 else: 1191 fmt = "%." + "%d" % digits + "f GB" 1192 value = bytes / BYTES_PER_GBYTE 1193 return fmt % value 1194
1195 1196 ################################## 1197 # getFunctionReference() function 1198 ################################## 1199 1200 -def getFunctionReference(module, function):
1201 """ 1202 Gets a reference to a named function. 1203 1204 This does some hokey-pokey to get back a reference to a dynamically named 1205 function. For instance, say you wanted to get a reference to the 1206 C{os.path.isdir} function. You could use:: 1207 1208 myfunc = getFunctionReference("os.path", "isdir") 1209 1210 Although we won't bomb out directly, behavior is pretty much undefined if 1211 you pass in C{None} or C{""} for either C{module} or C{function}. 1212 1213 The only validation we enforce is that whatever we get back must be 1214 callable. 1215 1216 I derived this code based on the internals of the Python unittest 1217 implementation. I don't claim to completely understand how it works. 1218 1219 @param module: Name of module associated with function. 1220 @type module: Something like "os.path" or "CedarBackup3.util" 1221 1222 @param function: Name of function 1223 @type function: Something like "isdir" or "getUidGid" 1224 1225 @return: Reference to function associated with name. 1226 1227 @raise ImportError: If the function cannot be found. 1228 @raise ValueError: If the resulting reference is not callable. 1229 1230 @copyright: Some of this code, prior to customization, was originally part 1231 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python 1232 Software Foundation; All Rights Reserved. 1233 """ 1234 parts = [] 1235 if module is not None and module != "": 1236 parts = module.split(".") 1237 if function is not None and function != "": 1238 parts.append(function) 1239 copy = parts[:] 1240 while copy: 1241 try: 1242 module = __import__(".".join(copy)) 1243 break 1244 except ImportError: 1245 del copy[-1] 1246 if not copy: raise 1247 parts = parts[1:] 1248 obj = module 1249 for part in parts: 1250 obj = getattr(obj, part) 1251 if not isinstance(obj, collections.Callable): 1252 raise ValueError("Reference to %s.%s is not callable." % (module, function)) 1253 return obj
1254
1255 1256 ####################### 1257 # getUidGid() function 1258 ####################### 1259 1260 -def getUidGid(user, group):
1261 """ 1262 Get the uid/gid associated with a user/group pair 1263 1264 This is a no-op if user/group functionality is not available on the platform. 1265 1266 @param user: User name 1267 @type user: User name as a string 1268 1269 @param group: Group name 1270 @type group: Group name as a string 1271 1272 @return: Tuple C{(uid, gid)} matching passed-in user and group. 1273 @raise ValueError: If the ownership user/group values are invalid 1274 """ 1275 if _UID_GID_AVAILABLE: 1276 try: 1277 uid = pwd.getpwnam(user)[2] 1278 gid = grp.getgrnam(group)[2] 1279 return (uid, gid) 1280 except Exception as e: 1281 logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e) 1282 raise ValueError("Unable to lookup up uid and gid for passed in user/group.") 1283 else: 1284 return (0, 0)
1285
1286 1287 ############################# 1288 # changeOwnership() function 1289 ############################# 1290 1291 -def changeOwnership(path, user, group):
1292 """ 1293 Changes ownership of path to match the user and group. 1294 1295 This is a no-op if user/group functionality is not available on the 1296 platform, or if the either passed-in user or group is C{None}. Further, we 1297 won't even try to do it unless running as root, since it's unlikely to work. 1298 1299 @param path: Path whose ownership to change. 1300 @param user: User which owns file. 1301 @param group: Group which owns file. 1302 """ 1303 if _UID_GID_AVAILABLE: 1304 if user is None or group is None: 1305 logger.debug("User or group is None, so not attempting to change owner on [%s].", path) 1306 elif not isRunningAsRoot(): 1307 logger.debug("Not root, so not attempting to change owner on [%s].", path) 1308 else: 1309 try: 1310 (uid, gid) = getUidGid(user, group) 1311 os.chown(path, uid, gid) 1312 except Exception as e: 1313 logger.error("Error changing ownership of [%s]: %s", path, e)
1314
1315 1316 ############################# 1317 # isRunningAsRoot() function 1318 ############################# 1319 1320 -def isRunningAsRoot():
1321 """ 1322 Indicates whether the program is running as the root user. 1323 """ 1324 return os.getuid() == 0
1325
1326 1327 ############################## 1328 # splitCommandLine() function 1329 ############################## 1330 1331 -def splitCommandLine(commandLine):
1332 """ 1333 Splits a command line string into a list of arguments. 1334 1335 Unfortunately, there is no "standard" way to parse a command line string, 1336 and it's actually not an easy problem to solve portably (essentially, we 1337 have to emulate the shell argument-processing logic). This code only 1338 respects double quotes (C{"}) for grouping arguments, not single quotes 1339 (C{'}). Make sure you take this into account when building your command 1340 line. 1341 1342 Incidentally, I found this particular parsing method while digging around in 1343 Google Groups, and I tweaked it for my own use. 1344 1345 @param commandLine: Command line string 1346 @type commandLine: String, i.e. "cback3 --verbose stage store" 1347 1348 @return: List of arguments, suitable for passing to C{popen2}. 1349 1350 @raise ValueError: If the command line is None. 1351 """ 1352 if commandLine is None: 1353 raise ValueError("Cannot split command line of None.") 1354 fields = re.findall('[^ "]+|"[^"]+"', commandLine) 1355 fields = [field.replace('"', '') for field in fields] 1356 return fields
1357
1358 1359 ############################ 1360 # resolveCommand() function 1361 ############################ 1362 1363 -def resolveCommand(command):
1364 """ 1365 Resolves the real path to a command through the path resolver mechanism. 1366 1367 Both extensions and standard Cedar Backup functionality need a way to 1368 resolve the "real" location of various executables. Normally, they assume 1369 that these executables are on the system path, but some callers need to 1370 specify an alternate location. 1371 1372 Ideally, we want to handle this configuration in a central location. The 1373 Cedar Backup path resolver mechanism (a singleton called 1374 L{PathResolverSingleton}) provides the central location to store the 1375 mappings. This function wraps access to the singleton, and is what all 1376 functions (extensions or standard functionality) should call if they need to 1377 find a command. 1378 1379 The passed-in command must actually be a list, in the standard form used by 1380 all existing Cedar Backup code (something like C{["svnlook", ]}). The 1381 lookup will actually be done on the first element in the list, and the 1382 returned command will always be in list form as well. 1383 1384 If the passed-in command can't be resolved or no mapping exists, then the 1385 command itself will be returned unchanged. This way, we neatly fall back on 1386 default behavior if we have no sensible alternative. 1387 1388 @param command: Command to resolve. 1389 @type command: List form of command, i.e. C{["svnlook", ]}. 1390 1391 @return: Path to command or just command itself if no mapping exists. 1392 """ 1393 singleton = PathResolverSingleton.getInstance() 1394 name = command[0] 1395 result = command[:] 1396 result[0] = singleton.lookup(name, name) 1397 return result
1398
1399 1400 ############################ 1401 # executeCommand() function 1402 ############################ 1403 1404 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1405 """ 1406 Executes a shell command, hopefully in a safe way. 1407 1408 This function exists to replace direct calls to C{os.popen} in the Cedar 1409 Backup code. It's not safe to call a function such as C{os.popen()} with 1410 untrusted arguments, since that can cause problems if the string contains 1411 non-safe variables or other constructs (imagine that the argument is 1412 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/; 1413 echo"} in the current environment). 1414 1415 Instead, it's safer to pass a list of arguments in the style supported bt 1416 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe} 1417 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}. 1418 1419 Under the normal case, this function will return a tuple of C{(status, 1420 None)} where the status is the wait-encoded return status of the call per 1421 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as 1422 C{True}, the function will return a tuple of C{(status, output)} where 1423 C{output} is a list of strings, one entry per line in the output from the 1424 command. Output is always logged to the C{outputLogger.info()} target, 1425 regardless of whether it's returned. 1426 1427 By default, C{stdout} and C{stderr} will be intermingled in the output. 1428 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be 1429 included in the output. 1430 1431 The C{doNotLog} parameter exists so that callers can force the function to 1432 not log command output to the debug log. Normally, you would want to log. 1433 However, if you're using this function to write huge output files (i.e. 1434 database backups written to C{stdout}) then you might want to avoid putting 1435 all that information into the debug log. 1436 1437 The C{outputFile} parameter exists to make it easier for a caller to push 1438 output into a file, i.e. as a substitute for redirection to a file. If this 1439 value is passed in, each time a line of output is generated, it will be 1440 written to the file using C{outputFile.write()}. At the end, the file 1441 descriptor will be flushed using C{outputFile.flush()}. The caller 1442 maintains responsibility for closing the file object appropriately. 1443 1444 @note: I know that it's a bit confusing that the command and the arguments 1445 are both lists. I could have just required the caller to pass in one big 1446 list. However, I think it makes some sense to keep the command (the 1447 constant part of what we're executing, i.e. C{"scp -B"}) separate from its 1448 arguments, even if they both end up looking kind of similar. 1449 1450 @note: You cannot redirect output via shell constructs (i.e. C{>file}, 1451 C{2>/dev/null}, etc.) using this function. The redirection string would be 1452 passed to the command just like any other argument. However, you can 1453 implement the equivalent to redirection using C{ignoreStderr} and 1454 C{outputFile}, as discussed above. 1455 1456 @note: The operating system environment is partially sanitized before 1457 the command is invoked. See L{sanitizeEnvironment} for details. 1458 1459 @param command: Shell command to execute 1460 @type command: List of individual arguments that make up the command 1461 1462 @param args: List of arguments to the command 1463 @type args: List of additional arguments to the command 1464 1465 @param returnOutput: Indicates whether to return the output of the command 1466 @type returnOutput: Boolean C{True} or C{False} 1467 1468 @param ignoreStderr: Whether stderr should be discarded 1469 @type ignoreStderr: Boolean True or False 1470 1471 @param doNotLog: Indicates that output should not be logged. 1472 @type doNotLog: Boolean C{True} or C{False} 1473 1474 @param outputFile: File object that all output should be written to. 1475 @type outputFile: File object as returned from C{open()} or C{file()}, configured for binary write 1476 1477 @return: Tuple of C{(result, output)} as described above. 1478 """ 1479 logger.debug("Executing command %s with args %s.", command, args) 1480 outputLogger.info("Executing command %s with args %s.", command, args) 1481 if doNotLog: 1482 logger.debug("Note: output will not be logged, per the doNotLog flag.") 1483 outputLogger.info("Note: output will not be logged, per the doNotLog flag.") 1484 output = [] 1485 fields = command[:] # make sure to copy it so we don't destroy it 1486 fields.extend(args) 1487 try: 1488 sanitizeEnvironment() # make sure we have a consistent environment 1489 try: 1490 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1491 except OSError: 1492 # On some platforms (i.e. Cygwin) this intermittently fails the first time we do it. 1493 # So, we attempt it a second time and if that works, we just go on as usual. 1494 # The problem appears to be that we sometimes get a bad stderr file descriptor. 1495 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1496 while True: 1497 line = pipe.stdout.readline() 1498 if not line: break 1499 if returnOutput: output.append(line.decode("utf-8")) 1500 if outputFile is not None: outputFile.write(line) 1501 if not doNotLog: outputLogger.info(line.decode("utf-8")[:-1]) # this way the log will (hopefully) get updated in realtime 1502 if outputFile is not None: 1503 try: # note, not every file-like object can be flushed 1504 outputFile.flush() 1505 except: pass 1506 if returnOutput: 1507 return (pipe.wait(), output) 1508 else: 1509 return (pipe.wait(), None) 1510 except OSError as e: 1511 try: 1512 if returnOutput: 1513 if output != []: 1514 return (pipe.wait(), output) 1515 else: 1516 return (pipe.wait(), [ e, ]) 1517 else: 1518 return (pipe.wait(), None) 1519 except UnboundLocalError: # pipe not set 1520 if returnOutput: 1521 return (256, []) 1522 else: 1523 return (256, None)
1524
1525 1526 ############################## 1527 # calculateFileAge() function 1528 ############################## 1529 1530 -def calculateFileAge(path):
1531 """ 1532 Calculates the age (in days) of a file. 1533 1534 The "age" of a file is the amount of time since the file was last used, per 1535 the most recent of the file's C{st_atime} and C{st_mtime} values. 1536 1537 Technically, we only intend this function to work with files, but it will 1538 probably work with anything on the filesystem. 1539 1540 @param path: Path to a file on disk. 1541 1542 @return: Age of the file in days (possibly fractional). 1543 @raise OSError: If the file doesn't exist. 1544 """ 1545 currentTime = int(time.time()) 1546 fileStats = os.stat(path) 1547 lastUse = max(fileStats.st_atime, fileStats.st_mtime) # "most recent" is "largest" 1548 ageInSeconds = currentTime - lastUse 1549 ageInDays = ageInSeconds / SECONDS_PER_DAY 1550 return ageInDays
1551
1552 1553 ################### 1554 # mount() function 1555 ################### 1556 1557 -def mount(devicePath, mountPoint, fsType):
1558 """ 1559 Mounts the indicated device at the indicated mount point. 1560 1561 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount 1562 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any 1563 filesystem type that is supported by C{mount} on your platform. If the type 1564 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may 1565 not work on all systems. 1566 1567 @note: This only works on platforms that have a concept of "mounting" a 1568 filesystem through a command-line C{"mount"} command, like UNIXes. It 1569 won't work on Windows. 1570 1571 @param devicePath: Path of device to be mounted. 1572 @param mountPoint: Path that device should be mounted at. 1573 @param fsType: Type of the filesystem assumed to be available via the device. 1574 1575 @raise IOError: If the device cannot be mounted. 1576 """ 1577 if fsType is None: 1578 args = [ devicePath, mountPoint ] 1579 else: 1580 args = [ "-t", fsType, devicePath, mountPoint ] 1581 command = resolveCommand(MOUNT_COMMAND) 1582 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0] 1583 if result != 0: 1584 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1585
1586 1587 ##################### 1588 # unmount() function 1589 ##################### 1590 1591 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1592 """ 1593 Unmounts whatever device is mounted at the indicated mount point. 1594 1595 Sometimes, it might not be possible to unmount the mount point immediately, 1596 if there are still files open there. Use the C{attempts} and C{waitSeconds} 1597 arguments to indicate how many unmount attempts to make and how many seconds 1598 to wait between attempts. If you pass in zero attempts, no attempts will be 1599 made (duh). 1600 1601 If the indicated mount point is not really a mount point per 1602 C{os.path.ismount()}, then it will be ignored. This seems to be a safer 1603 check then looking through C{/etc/mtab}, since C{ismount()} is already in 1604 the Python standard library and is documented as working on all POSIX 1605 systems. 1606 1607 If C{removeAfter} is C{True}, then the mount point will be removed using 1608 C{os.rmdir()} after the unmount action succeeds. If for some reason the 1609 mount point is not a directory, then it will not be removed. 1610 1611 @note: This only works on platforms that have a concept of "mounting" a 1612 filesystem through a command-line C{"mount"} command, like UNIXes. It 1613 won't work on Windows. 1614 1615 @param mountPoint: Mount point to be unmounted. 1616 @param removeAfter: Remove the mount point after unmounting it. 1617 @param attempts: Number of times to attempt the unmount. 1618 @param waitSeconds: Number of seconds to wait between repeated attempts. 1619 1620 @raise IOError: If the mount point is still mounted after attempts are exhausted. 1621 """ 1622 if os.path.ismount(mountPoint): 1623 for attempt in range(0, attempts): 1624 logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint) 1625 command = resolveCommand(UMOUNT_COMMAND) 1626 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0] 1627 if result != 0: 1628 logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt) 1629 elif os.path.ismount(mountPoint): 1630 logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint) 1631 else: 1632 logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt) 1633 break # this will cause us to skip the loop else: clause 1634 if attempt+1 < attempts: # i.e. this isn't the last attempt 1635 if waitSeconds > 0: 1636 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds) 1637 time.sleep(waitSeconds) 1638 else: 1639 if os.path.ismount(mountPoint): 1640 raise IOError("Unable to unmount [%s] after %d attempts.", mountPoint, attempts) 1641 logger.info("Mount point [%s] seems to have finally gone away.", mountPoint) 1642 if os.path.isdir(mountPoint) and removeAfter: 1643 logger.debug("Removing mount point [%s].", mountPoint) 1644 os.rmdir(mountPoint)
1645
1646 1647 ########################### 1648 # deviceMounted() function 1649 ########################### 1650 1651 -def deviceMounted(devicePath):
1652 """ 1653 Indicates whether a specific filesystem device is currently mounted. 1654 1655 We determine whether the device is mounted by looking through the system's 1656 C{mtab} file. This file shows every currently-mounted filesystem, ordered 1657 by device. We only do the check if the C{mtab} file exists and is readable. 1658 Otherwise, we assume that the device is not mounted. 1659 1660 @note: This only works on platforms that have a concept of an mtab file 1661 to show mounted volumes, like UNIXes. It won't work on Windows. 1662 1663 @param devicePath: Path of device to be checked 1664 1665 @return: True if device is mounted, false otherwise. 1666 """ 1667 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK): 1668 realPath = os.path.realpath(devicePath) 1669 with open(MTAB_FILE) as f: 1670 lines = f.readlines() 1671 for line in lines: 1672 (mountDevice, mountPoint, remainder) = line.split(None, 2) 1673 if mountDevice in [ devicePath, realPath, ]: 1674 logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint) 1675 return True 1676 return False
1677
1678 1679 ######################## 1680 # encodePath() function 1681 ######################## 1682 1683 -def encodePath(path):
1684 """ 1685 Safely encodes a filesystem path as a Unicode string, converting bytes to fileystem encoding if necessary. 1686 @param path: Path to encode 1687 @return: Path, as a string, encoded appropriately 1688 @raise ValueError: If the path cannot be encoded properly. 1689 @see: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/ 1690 """ 1691 if path is None: 1692 return path 1693 try: 1694 if isinstance(path, bytes): 1695 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 1696 path = path.decode(encoding, "surrogateescape") # to match what os.listdir() does 1697 return path 1698 except UnicodeError as e: 1699 raise ValueError("Path could not be safely encoded as %s: %s" % (encoding, str(e)))
1700
1701 1702 ######################## 1703 # nullDevice() function 1704 ######################## 1705 1706 -def nullDevice():
1707 """ 1708 Attempts to portably return the null device on this system. 1709 1710 The null device is something like C{/dev/null} on a UNIX system. The name 1711 varies on other platforms. 1712 """ 1713 return os.devnull
1714
1715 1716 ############################## 1717 # deriveDayOfWeek() function 1718 ############################## 1719 1720 -def deriveDayOfWeek(dayName):
1721 """ 1722 Converts English day name to numeric day of week as from C{time.localtime}. 1723 1724 For instance, the day C{monday} would be converted to the number C{0}. 1725 1726 @param dayName: Day of week to convert 1727 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1728 1729 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible. 1730 """ 1731 if dayName.lower() == "monday": 1732 return 0 1733 elif dayName.lower() == "tuesday": 1734 return 1 1735 elif dayName.lower() == "wednesday": 1736 return 2 1737 elif dayName.lower() == "thursday": 1738 return 3 1739 elif dayName.lower() == "friday": 1740 return 4 1741 elif dayName.lower() == "saturday": 1742 return 5 1743 elif dayName.lower() == "sunday": 1744 return 6 1745 else: 1746 return -1 # What else can we do?? Thrown an exception, I guess.
1747
1748 1749 ########################### 1750 # isStartOfWeek() function 1751 ########################### 1752 1753 -def isStartOfWeek(startingDay):
1754 """ 1755 Indicates whether "today" is the backup starting day per configuration. 1756 1757 If the current day's English name matches the indicated starting day, then 1758 today is a starting day. 1759 1760 @param startingDay: Configured starting day. 1761 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1762 1763 @return: Boolean indicating whether today is the starting day. 1764 """ 1765 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay) 1766 if value: 1767 logger.debug("Today is the start of the week.") 1768 else: 1769 logger.debug("Today is NOT the start of the week.") 1770 return value
1771
1772 1773 ################################# 1774 # buildNormalizedPath() function 1775 ################################# 1776 1777 -def buildNormalizedPath(path):
1778 """ 1779 Returns a "normalized" path based on a path name. 1780 1781 A normalized path is a representation of a path that is also a valid file 1782 name. To make a valid file name out of a complete path, we have to convert 1783 or remove some characters that are significant to the filesystem -- in 1784 particular, the path separator and any leading C{'.'} character (which would 1785 cause the file to be hidden in a file listing). 1786 1787 Note that this is a one-way transformation -- you can't safely derive the 1788 original path from the normalized path. 1789 1790 To normalize a path, we begin by looking at the first character. If the 1791 first character is C{'/'} or C{'\\'}, it gets removed. If the first 1792 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the 1793 rest of the path and convert all remaining C{'/'} or C{'\\'} characters 1794 C{'-'}, and all remaining whitespace characters to C{'_'}. 1795 1796 As a special case, a path consisting only of a single C{'/'} or C{'\\'} 1797 character will be converted to C{'-'}. 1798 1799 @param path: Path to normalize 1800 1801 @return: Normalized path as described above. 1802 1803 @raise ValueError: If the path is None 1804 """ 1805 if path is None: 1806 raise ValueError("Cannot normalize path None.") 1807 elif len(path) == 0: 1808 return path 1809 elif path == "/" or path == "\\": 1810 return "-" 1811 else: 1812 normalized = path 1813 normalized = re.sub(r"^\/", "", normalized) # remove leading '/' 1814 normalized = re.sub(r"^\\", "", normalized) # remove leading '\' 1815 normalized = re.sub(r"^\.", "_", normalized) # convert leading '.' to '_' so file won't be hidden 1816 normalized = re.sub(r"\/", "-", normalized) # convert all '/' characters to '-' 1817 normalized = re.sub(r"\\", "-", normalized) # convert all '\' characters to '-' 1818 normalized = re.sub(r"\s", "_", normalized) # convert all whitespace to '_' 1819 return normalized
1820
1821 1822 ################################# 1823 # sanitizeEnvironment() function 1824 ################################# 1825 1826 -def sanitizeEnvironment():
1827 """ 1828 Sanitizes the operating system environment. 1829 1830 The operating system environment is contained in C{os.environ}. This method 1831 sanitizes the contents of that dictionary. 1832 1833 Currently, all it does is reset the locale (removing C{$LC_*}) and set the 1834 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count 1835 on consistent localization regardless of what the end-user has configured. 1836 This is important for code that needs to parse program output. 1837 1838 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already 1839 set to the proper value, it is not re-set, so we can avoid the memory leaks 1840 that are documented to occur on BSD-based systems. 1841 1842 @return: Copy of the sanitized environment. 1843 """ 1844 for var in LOCALE_VARS: 1845 if var in os.environ: 1846 del os.environ[var] 1847 if LANG_VAR in os.environ: 1848 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE: # no need to reset if it exists (avoid leaks on BSD systems) 1849 os.environ[LANG_VAR] = DEFAULT_LANGUAGE 1850 return os.environ.copy()
1851 1870
1871 1872 ######################### 1873 # checkUnique() function 1874 ######################### 1875 1876 -def checkUnique(prefix, values):
1877 """ 1878 Checks that all values are unique. 1879 1880 The values list is checked for duplicate values. If there are 1881 duplicates, an exception is thrown. All duplicate values are listed in 1882 the exception. 1883 1884 @param prefix: Prefix to use in the thrown exception 1885 @param values: List of values to check 1886 1887 @raise ValueError: If there are duplicates in the list 1888 """ 1889 values.sort() 1890 duplicates = [] 1891 for i in range(1, len(values)): 1892 if values[i-1] == values[i]: 1893 duplicates.append(values[i]) 1894 if duplicates: 1895 raise ValueError("%s %s" % (prefix, duplicates))
1896
1897 1898 ####################################### 1899 # parseCommaSeparatedString() function 1900 ####################################### 1901 1902 -def parseCommaSeparatedString(commaString):
1903 """ 1904 Parses a list of values out of a comma-separated string. 1905 1906 The items in the list are split by comma, and then have whitespace 1907 stripped. As a special case, if C{commaString} is C{None}, then C{None} 1908 will be returned. 1909 1910 @param commaString: List of values in comma-separated string format. 1911 @return: Values from commaString split into a list, or C{None}. 1912 """ 1913 if commaString is None: 1914 return None 1915 else: 1916 pass1 = commaString.split(",") 1917 pass2 = [] 1918 for item in pass1: 1919 item = item.strip() 1920 if len(item) > 0: 1921 pass2.append(item) 1922 return pass2
1923