1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up Subversion repositories.
40
41 This is a Cedar Backup extension used to back up Subversion repositories via
42 the Cedar Backup command line. Each Subversion repository can be backed using
43 the same collect modes allowed for filesystems in the standard Cedar Backup
44 collect action: weekly, daily, incremental.
45
46 This extension requires a new configuration section <subversion> and is
47 intended to be run either immediately before or immediately after the standard
48 collect action. Aside from its own configuration, it requires the options and
49 collect configuration sections in the standard Cedar Backup configuration file.
50
51 There are two different kinds of Subversion repositories at this writing: BDB
52 (Berkeley Database) and FSFS (a "filesystem within a filesystem"). Although
53 the repository type can be specified in configuration, that information is just
54 kept around for reference. It doesn't affect the backup. Both kinds of
55 repositories are backed up in the same way, using C{svnadmin dump} in an
56 incremental mode.
57
58 It turns out that FSFS repositories can also be backed up just like any
59 other filesystem directory. If you would rather do that, then use the normal
60 collect action. This is probably simpler, although it carries its own
61 advantages and disadvantages (plus you will have to be careful to exclude
62 the working directories Subversion uses when building an update to commit).
63 Check the Subversion documentation for more information.
64
65 @author: Kenneth J. Pronovici <pronovic@ieee.org>
66 """
67
68
69
70
71
72
73 import os
74 import logging
75 import pickle
76 from bz2 import BZ2File
77 from gzip import GzipFile
78 from functools import total_ordering
79
80
81 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode
82 from CedarBackup3.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
83 from CedarBackup3.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
84 from CedarBackup3.filesystem import FilesystemList
85 from CedarBackup3.util import UnorderedList, RegexList
86 from CedarBackup3.util import isStartOfWeek, buildNormalizedPath
87 from CedarBackup3.util import resolveCommand, executeCommand
88 from CedarBackup3.util import ObjectTypeList, encodePath, changeOwnership
89
90
91
92
93
94
95 logger = logging.getLogger("CedarBackup3.log.extend.subversion")
96
97 SVNLOOK_COMMAND = [ "svnlook", ]
98 SVNADMIN_COMMAND = [ "svnadmin", ]
99
100 REVISION_PATH_EXTENSION = "svnlast"
101
102
103
104
105
106
107 @total_ordering
108 -class RepositoryDir(object):
109
110 """
111 Class representing Subversion repository directory.
112
113 A repository directory is a directory that contains one or more Subversion
114 repositories.
115
116 The following restrictions exist on data in this class:
117
118 - The directory path must be absolute.
119 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
120 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
121
122 The repository type value is kept around just for reference. It doesn't
123 affect the behavior of the backup.
124
125 Relative exclusions are allowed here. However, there is no configured
126 ignore file, because repository dir backups are not recursive.
127
128 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
129 directoryPath, collectMode, compressMode
130 """
131
132 - def __init__(self, repositoryType=None, directoryPath=None, collectMode=None, compressMode=None,
133 relativeExcludePaths=None, excludePatterns=None):
134 """
135 Constructor for the C{RepositoryDir} class.
136
137 @param repositoryType: Type of repository, for reference
138 @param directoryPath: Absolute path of the Subversion parent directory
139 @param collectMode: Overridden collect mode for this directory.
140 @param compressMode: Overridden compression mode for this directory.
141 @param relativeExcludePaths: List of relative paths to exclude.
142 @param excludePatterns: List of regular expression patterns to exclude
143 """
144 self._repositoryType = None
145 self._directoryPath = None
146 self._collectMode = None
147 self._compressMode = None
148 self._relativeExcludePaths = None
149 self._excludePatterns = None
150 self.repositoryType = repositoryType
151 self.directoryPath = directoryPath
152 self.collectMode = collectMode
153 self.compressMode = compressMode
154 self.relativeExcludePaths = relativeExcludePaths
155 self.excludePatterns = excludePatterns
156
158 """
159 Official string representation for class instance.
160 """
161 return "RepositoryDir(%s, %s, %s, %s, %s, %s)" % (self.repositoryType, self.directoryPath, self.collectMode,
162 self.compressMode, self.relativeExcludePaths, self.excludePatterns)
163
165 """
166 Informal string representation for class instance.
167 """
168 return self.__repr__()
169
171 """Equals operator, iplemented in terms of original Python 2 compare operator."""
172 return self.__cmp__(other) == 0
173
175 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
176 return self.__cmp__(other) < 0
177
179 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
180 return self.__cmp__(other) > 0
181
221
223 """
224 Property target used to set the repository type.
225 There is no validation; this value is kept around just for reference.
226 """
227 self._repositoryType = value
228
230 """
231 Property target used to get the repository type.
232 """
233 return self._repositoryType
234
236 """
237 Property target used to set the directory path.
238 The value must be an absolute path if it is not C{None}.
239 It does not have to exist on disk at the time of assignment.
240 @raise ValueError: If the value is not an absolute path.
241 @raise ValueError: If the value cannot be encoded properly.
242 """
243 if value is not None:
244 if not os.path.isabs(value):
245 raise ValueError("Repository path must be an absolute path.")
246 self._directoryPath = encodePath(value)
247
249 """
250 Property target used to get the repository path.
251 """
252 return self._directoryPath
253
255 """
256 Property target used to set the collect mode.
257 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
258 @raise ValueError: If the value is not valid.
259 """
260 if value is not None:
261 if value not in VALID_COLLECT_MODES:
262 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
263 self._collectMode = value
264
266 """
267 Property target used to get the collect mode.
268 """
269 return self._collectMode
270
272 """
273 Property target used to set the compress mode.
274 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
275 @raise ValueError: If the value is not valid.
276 """
277 if value is not None:
278 if value not in VALID_COMPRESS_MODES:
279 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
280 self._compressMode = value
281
283 """
284 Property target used to get the compress mode.
285 """
286 return self._compressMode
287
289 """
290 Property target used to set the relative exclude paths list.
291 Elements do not have to exist on disk at the time of assignment.
292 """
293 if value is None:
294 self._relativeExcludePaths = None
295 else:
296 try:
297 saved = self._relativeExcludePaths
298 self._relativeExcludePaths = UnorderedList()
299 self._relativeExcludePaths.extend(value)
300 except Exception as e:
301 self._relativeExcludePaths = saved
302 raise e
303
305 """
306 Property target used to get the relative exclude paths list.
307 """
308 return self._relativeExcludePaths
309
311 """
312 Property target used to set the exclude patterns list.
313 """
314 if value is None:
315 self._excludePatterns = None
316 else:
317 try:
318 saved = self._excludePatterns
319 self._excludePatterns = RegexList()
320 self._excludePatterns.extend(value)
321 except Exception as e:
322 self._excludePatterns = saved
323 raise e
324
326 """
327 Property target used to get the exclude patterns list.
328 """
329 return self._excludePatterns
330
331 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
332 directoryPath = property(_getDirectoryPath, _setDirectoryPath, None, doc="Absolute path of the Subversion parent directory.")
333 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
334 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
335 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
336 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
337
338
339
340
341
342
343 @total_ordering
344 -class Repository(object):
345
346 """
347 Class representing generic Subversion repository configuration..
348
349 The following restrictions exist on data in this class:
350
351 - The respository path must be absolute.
352 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
353 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
354
355 The repository type value is kept around just for reference. It doesn't
356 affect the behavior of the backup.
357
358 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
359 repositoryPath, collectMode, compressMode
360 """
361
362 - def __init__(self, repositoryType=None, repositoryPath=None, collectMode=None, compressMode=None):
363 """
364 Constructor for the C{Repository} class.
365
366 @param repositoryType: Type of repository, for reference
367 @param repositoryPath: Absolute path to a Subversion repository on disk.
368 @param collectMode: Overridden collect mode for this directory.
369 @param compressMode: Overridden compression mode for this directory.
370 """
371 self._repositoryType = None
372 self._repositoryPath = None
373 self._collectMode = None
374 self._compressMode = None
375 self.repositoryType = repositoryType
376 self.repositoryPath = repositoryPath
377 self.collectMode = collectMode
378 self.compressMode = compressMode
379
385
387 """
388 Informal string representation for class instance.
389 """
390 return self.__repr__()
391
393 """Equals operator, iplemented in terms of original Python 2 compare operator."""
394 return self.__cmp__(other) == 0
395
397 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
398 return self.__cmp__(other) < 0
399
401 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
402 return self.__cmp__(other) > 0
403
433
435 """
436 Property target used to set the repository type.
437 There is no validation; this value is kept around just for reference.
438 """
439 self._repositoryType = value
440
442 """
443 Property target used to get the repository type.
444 """
445 return self._repositoryType
446
448 """
449 Property target used to set the repository path.
450 The value must be an absolute path if it is not C{None}.
451 It does not have to exist on disk at the time of assignment.
452 @raise ValueError: If the value is not an absolute path.
453 @raise ValueError: If the value cannot be encoded properly.
454 """
455 if value is not None:
456 if not os.path.isabs(value):
457 raise ValueError("Repository path must be an absolute path.")
458 self._repositoryPath = encodePath(value)
459
461 """
462 Property target used to get the repository path.
463 """
464 return self._repositoryPath
465
467 """
468 Property target used to set the collect mode.
469 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
470 @raise ValueError: If the value is not valid.
471 """
472 if value is not None:
473 if value not in VALID_COLLECT_MODES:
474 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
475 self._collectMode = value
476
478 """
479 Property target used to get the collect mode.
480 """
481 return self._collectMode
482
484 """
485 Property target used to set the compress mode.
486 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
487 @raise ValueError: If the value is not valid.
488 """
489 if value is not None:
490 if value not in VALID_COMPRESS_MODES:
491 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
492 self._compressMode = value
493
495 """
496 Property target used to get the compress mode.
497 """
498 return self._compressMode
499
500 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
501 repositoryPath = property(_getRepositoryPath, _setRepositoryPath, None, doc="Path to the repository to collect.")
502 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
503 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
504
512
513 """
514 Class representing Subversion configuration.
515
516 Subversion configuration is used for backing up Subversion repositories.
517
518 The following restrictions exist on data in this class:
519
520 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
521 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
522 - The repositories list must be a list of C{Repository} objects.
523 - The repositoryDirs list must be a list of C{RepositoryDir} objects.
524
525 For the two lists, validation is accomplished through the
526 L{util.ObjectTypeList} list implementation that overrides common list
527 methods and transparently ensures that each element has the correct type.
528
529 @note: Lists within this class are "unordered" for equality comparisons.
530
531 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
532 collectMode, compressMode, repositories
533 """
534
535 - def __init__(self, collectMode=None, compressMode=None, repositories=None, repositoryDirs=None):
536 """
537 Constructor for the C{SubversionConfig} class.
538
539 @param collectMode: Default collect mode.
540 @param compressMode: Default compress mode.
541 @param repositories: List of Subversion repositories to back up.
542 @param repositoryDirs: List of Subversion parent directories to back up.
543
544 @raise ValueError: If one of the values is invalid.
545 """
546 self._collectMode = None
547 self._compressMode = None
548 self._repositories = None
549 self._repositoryDirs = None
550 self.collectMode = collectMode
551 self.compressMode = compressMode
552 self.repositories = repositories
553 self.repositoryDirs = repositoryDirs
554
560
562 """
563 Informal string representation for class instance.
564 """
565 return self.__repr__()
566
568 """Equals operator, iplemented in terms of original Python 2 compare operator."""
569 return self.__cmp__(other) == 0
570
572 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
573 return self.__cmp__(other) < 0
574
576 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
577 return self.__cmp__(other) > 0
578
609
611 """
612 Property target used to set the collect mode.
613 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
614 @raise ValueError: If the value is not valid.
615 """
616 if value is not None:
617 if value not in VALID_COLLECT_MODES:
618 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
619 self._collectMode = value
620
622 """
623 Property target used to get the collect mode.
624 """
625 return self._collectMode
626
628 """
629 Property target used to set the compress mode.
630 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
631 @raise ValueError: If the value is not valid.
632 """
633 if value is not None:
634 if value not in VALID_COMPRESS_MODES:
635 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
636 self._compressMode = value
637
639 """
640 Property target used to get the compress mode.
641 """
642 return self._compressMode
643
645 """
646 Property target used to set the repositories list.
647 Either the value must be C{None} or each element must be a C{Repository}.
648 @raise ValueError: If the value is not a C{Repository}
649 """
650 if value is None:
651 self._repositories = None
652 else:
653 try:
654 saved = self._repositories
655 self._repositories = ObjectTypeList(Repository, "Repository")
656 self._repositories.extend(value)
657 except Exception as e:
658 self._repositories = saved
659 raise e
660
662 """
663 Property target used to get the repositories list.
664 """
665 return self._repositories
666
668 """
669 Property target used to set the repositoryDirs list.
670 Either the value must be C{None} or each element must be a C{Repository}.
671 @raise ValueError: If the value is not a C{Repository}
672 """
673 if value is None:
674 self._repositoryDirs = None
675 else:
676 try:
677 saved = self._repositoryDirs
678 self._repositoryDirs = ObjectTypeList(RepositoryDir, "RepositoryDir")
679 self._repositoryDirs.extend(value)
680 except Exception as e:
681 self._repositoryDirs = saved
682 raise e
683
685 """
686 Property target used to get the repositoryDirs list.
687 """
688 return self._repositoryDirs
689
690 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
691 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
692 repositories = property(_getRepositories, _setRepositories, None, doc="List of Subversion repositories to back up.")
693 repositoryDirs = property(_getRepositoryDirs, _setRepositoryDirs, None, doc="List of Subversion parent directories to back up.")
694
695
696
697
698
699
700 @total_ordering
701 -class LocalConfig(object):
702
703 """
704 Class representing this extension's configuration document.
705
706 This is not a general-purpose configuration object like the main Cedar
707 Backup configuration object. Instead, it just knows how to parse and emit
708 Subversion-specific configuration values. Third parties who need to read
709 and write configuration related to this extension should access it through
710 the constructor, C{validate} and C{addConfig} methods.
711
712 @note: Lists within this class are "unordered" for equality comparisons.
713
714 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
715 subversion, validate, addConfig
716 """
717
718 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
719 """
720 Initializes a configuration object.
721
722 If you initialize the object without passing either C{xmlData} or
723 C{xmlPath} then configuration will be empty and will be invalid until it
724 is filled in properly.
725
726 No reference to the original XML data or original path is saved off by
727 this class. Once the data has been parsed (successfully or not) this
728 original information is discarded.
729
730 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
731 method will be called (with its default arguments) against configuration
732 after successfully parsing any passed-in XML. Keep in mind that even if
733 C{validate} is C{False}, it might not be possible to parse the passed-in
734 XML document if lower-level validations fail.
735
736 @note: It is strongly suggested that the C{validate} option always be set
737 to C{True} (the default) unless there is a specific need to read in
738 invalid configuration from disk.
739
740 @param xmlData: XML data representing configuration.
741 @type xmlData: String data.
742
743 @param xmlPath: Path to an XML file on disk.
744 @type xmlPath: Absolute path to a file on disk.
745
746 @param validate: Validate the document after parsing it.
747 @type validate: Boolean true/false.
748
749 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
750 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
751 @raise ValueError: If the parsed configuration document is not valid.
752 """
753 self._subversion = None
754 self.subversion = None
755 if xmlData is not None and xmlPath is not None:
756 raise ValueError("Use either xmlData or xmlPath, but not both.")
757 if xmlData is not None:
758 self._parseXmlData(xmlData)
759 if validate:
760 self.validate()
761 elif xmlPath is not None:
762 with open(xmlPath) as f:
763 xmlData = f.read()
764 self._parseXmlData(xmlData)
765 if validate:
766 self.validate()
767
769 """
770 Official string representation for class instance.
771 """
772 return "LocalConfig(%s)" % (self.subversion)
773
775 """
776 Informal string representation for class instance.
777 """
778 return self.__repr__()
779
781 """Equals operator, iplemented in terms of original Python 2 compare operator."""
782 return self.__cmp__(other) == 0
783
785 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
786 return self.__cmp__(other) < 0
787
789 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
790 return self.__cmp__(other) > 0
791
793 """
794 Original Python 2 comparison operator.
795 Lists within this class are "unordered" for equality comparisons.
796 @param other: Other object to compare to.
797 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
798 """
799 if other is None:
800 return 1
801 if self.subversion != other.subversion:
802 if self.subversion < other.subversion:
803 return -1
804 else:
805 return 1
806 return 0
807
809 """
810 Property target used to set the subversion configuration value.
811 If not C{None}, the value must be a C{SubversionConfig} object.
812 @raise ValueError: If the value is not a C{SubversionConfig}
813 """
814 if value is None:
815 self._subversion = None
816 else:
817 if not isinstance(value, SubversionConfig):
818 raise ValueError("Value must be a C{SubversionConfig} object.")
819 self._subversion = value
820
822 """
823 Property target used to get the subversion configuration value.
824 """
825 return self._subversion
826
827 subversion = property(_getSubversion, _setSubversion, None, "Subversion configuration in terms of a C{SubversionConfig} object.")
828
830 """
831 Validates configuration represented by the object.
832
833 Subversion configuration must be filled in. Within that, the collect
834 mode and compress mode are both optional, but the list of repositories
835 must contain at least one entry.
836
837 Each repository must contain a repository path, and then must be either
838 able to take collect mode and compress mode configuration from the parent
839 C{SubversionConfig} object, or must set each value on its own.
840
841 @raise ValueError: If one of the validations fails.
842 """
843 if self.subversion is None:
844 raise ValueError("Subversion section is required.")
845 if ((self.subversion.repositories is None or len(self.subversion.repositories) < 1) and
846 (self.subversion.repositoryDirs is None or len(self.subversion.repositoryDirs) <1)):
847 raise ValueError("At least one Subversion repository must be configured.")
848 if self.subversion.repositories is not None:
849 for repository in self.subversion.repositories:
850 if repository.repositoryPath is None:
851 raise ValueError("Each repository must set a repository path.")
852 if self.subversion.collectMode is None and repository.collectMode is None:
853 raise ValueError("Collect mode must either be set in parent section or individual repository.")
854 if self.subversion.compressMode is None and repository.compressMode is None:
855 raise ValueError("Compress mode must either be set in parent section or individual repository.")
856 if self.subversion.repositoryDirs is not None:
857 for repositoryDir in self.subversion.repositoryDirs:
858 if repositoryDir.directoryPath is None:
859 raise ValueError("Each repository directory must set a directory path.")
860 if self.subversion.collectMode is None and repositoryDir.collectMode is None:
861 raise ValueError("Collect mode must either be set in parent section or repository directory.")
862 if self.subversion.compressMode is None and repositoryDir.compressMode is None:
863 raise ValueError("Compress mode must either be set in parent section or repository directory.")
864
866 """
867 Adds a <subversion> configuration section as the next child of a parent.
868
869 Third parties should use this function to write configuration related to
870 this extension.
871
872 We add the following fields to the document::
873
874 collectMode //cb_config/subversion/collectMode
875 compressMode //cb_config/subversion/compressMode
876
877 We also add groups of the following items, one list element per
878 item::
879
880 repository //cb_config/subversion/repository
881 repository_dir //cb_config/subversion/repository_dir
882
883 @param xmlDom: DOM tree as from C{impl.createDocument()}.
884 @param parentNode: Parent that the section should be appended to.
885 """
886 if self.subversion is not None:
887 sectionNode = addContainerNode(xmlDom, parentNode, "subversion")
888 addStringNode(xmlDom, sectionNode, "collect_mode", self.subversion.collectMode)
889 addStringNode(xmlDom, sectionNode, "compress_mode", self.subversion.compressMode)
890 if self.subversion.repositories is not None:
891 for repository in self.subversion.repositories:
892 LocalConfig._addRepository(xmlDom, sectionNode, repository)
893 if self.subversion.repositoryDirs is not None:
894 for repositoryDir in self.subversion.repositoryDirs:
895 LocalConfig._addRepositoryDir(xmlDom, sectionNode, repositoryDir)
896
898 """
899 Internal method to parse an XML string into the object.
900
901 This method parses the XML document into a DOM tree (C{xmlDom}) and then
902 calls a static method to parse the subversion configuration section.
903
904 @param xmlData: XML data to be parsed
905 @type xmlData: String data
906
907 @raise ValueError: If the XML cannot be successfully parsed.
908 """
909 (xmlDom, parentNode) = createInputDom(xmlData)
910 self._subversion = LocalConfig._parseSubversion(parentNode)
911
912 @staticmethod
914 """
915 Parses a subversion configuration section.
916
917 We read the following individual fields::
918
919 collectMode //cb_config/subversion/collect_mode
920 compressMode //cb_config/subversion/compress_mode
921
922 We also read groups of the following item, one list element per
923 item::
924
925 repositories //cb_config/subversion/repository
926 repository_dirs //cb_config/subversion/repository_dir
927
928 The repositories are parsed by L{_parseRepositories}, and the repository
929 dirs are parsed by L{_parseRepositoryDirs}.
930
931 @param parent: Parent node to search beneath.
932
933 @return: C{SubversionConfig} object or C{None} if the section does not exist.
934 @raise ValueError: If some filled-in value is invalid.
935 """
936 subversion = None
937 section = readFirstChild(parent, "subversion")
938 if section is not None:
939 subversion = SubversionConfig()
940 subversion.collectMode = readString(section, "collect_mode")
941 subversion.compressMode = readString(section, "compress_mode")
942 subversion.repositories = LocalConfig._parseRepositories(section)
943 subversion.repositoryDirs = LocalConfig._parseRepositoryDirs(section)
944 return subversion
945
946 @staticmethod
948 """
949 Reads a list of C{Repository} objects from immediately beneath the parent.
950
951 We read the following individual fields::
952
953 repositoryType type
954 repositoryPath abs_path
955 collectMode collect_mode
956 compressMode compess_mode
957
958 The type field is optional, and its value is kept around only for
959 reference.
960
961 @param parent: Parent node to search beneath.
962
963 @return: List of C{Repository} objects or C{None} if none are found.
964 @raise ValueError: If some filled-in value is invalid.
965 """
966 lst = []
967 for entry in readChildren(parent, "repository"):
968 if isElement(entry):
969 repository = Repository()
970 repository.repositoryType = readString(entry, "type")
971 repository.repositoryPath = readString(entry, "abs_path")
972 repository.collectMode = readString(entry, "collect_mode")
973 repository.compressMode = readString(entry, "compress_mode")
974 lst.append(repository)
975 if lst == []:
976 lst = None
977 return lst
978
979 @staticmethod
981 """
982 Adds a repository container as the next child of a parent.
983
984 We add the following fields to the document::
985
986 repositoryType repository/type
987 repositoryPath repository/abs_path
988 collectMode repository/collect_mode
989 compressMode repository/compress_mode
990
991 The <repository> node itself is created as the next child of the parent
992 node. This method only adds one repository node. The parent must loop
993 for each repository in the C{SubversionConfig} object.
994
995 If C{repository} is C{None}, this method call will be a no-op.
996
997 @param xmlDom: DOM tree as from C{impl.createDocument()}.
998 @param parentNode: Parent that the section should be appended to.
999 @param repository: Repository to be added to the document.
1000 """
1001 if repository is not None:
1002 sectionNode = addContainerNode(xmlDom, parentNode, "repository")
1003 addStringNode(xmlDom, sectionNode, "type", repository.repositoryType)
1004 addStringNode(xmlDom, sectionNode, "abs_path", repository.repositoryPath)
1005 addStringNode(xmlDom, sectionNode, "collect_mode", repository.collectMode)
1006 addStringNode(xmlDom, sectionNode, "compress_mode", repository.compressMode)
1007
1008 @staticmethod
1010 """
1011 Reads a list of C{RepositoryDir} objects from immediately beneath the parent.
1012
1013 We read the following individual fields::
1014
1015 repositoryType type
1016 directoryPath abs_path
1017 collectMode collect_mode
1018 compressMode compess_mode
1019
1020 We also read groups of the following items, one list element per
1021 item::
1022
1023 relativeExcludePaths exclude/rel_path
1024 excludePatterns exclude/pattern
1025
1026 The exclusions are parsed by L{_parseExclusions}.
1027
1028 The type field is optional, and its value is kept around only for
1029 reference.
1030
1031 @param parent: Parent node to search beneath.
1032
1033 @return: List of C{RepositoryDir} objects or C{None} if none are found.
1034 @raise ValueError: If some filled-in value is invalid.
1035 """
1036 lst = []
1037 for entry in readChildren(parent, "repository_dir"):
1038 if isElement(entry):
1039 repositoryDir = RepositoryDir()
1040 repositoryDir.repositoryType = readString(entry, "type")
1041 repositoryDir.directoryPath = readString(entry, "abs_path")
1042 repositoryDir.collectMode = readString(entry, "collect_mode")
1043 repositoryDir.compressMode = readString(entry, "compress_mode")
1044 (repositoryDir.relativeExcludePaths, repositoryDir.excludePatterns) = LocalConfig._parseExclusions(entry)
1045 lst.append(repositoryDir)
1046 if lst == []:
1047 lst = None
1048 return lst
1049
1050 @staticmethod
1052 """
1053 Reads exclusions data from immediately beneath the parent.
1054
1055 We read groups of the following items, one list element per item::
1056
1057 relative exclude/rel_path
1058 patterns exclude/pattern
1059
1060 If there are none of some pattern (i.e. no relative path items) then
1061 C{None} will be returned for that item in the tuple.
1062
1063 @param parentNode: Parent node to search beneath.
1064
1065 @return: Tuple of (relative, patterns) exclusions.
1066 """
1067 section = readFirstChild(parentNode, "exclude")
1068 if section is None:
1069 return (None, None)
1070 else:
1071 relative = readStringList(section, "rel_path")
1072 patterns = readStringList(section, "pattern")
1073 return (relative, patterns)
1074
1075 @staticmethod
1077 """
1078 Adds a repository dir container as the next child of a parent.
1079
1080 We add the following fields to the document::
1081
1082 repositoryType repository_dir/type
1083 directoryPath repository_dir/abs_path
1084 collectMode repository_dir/collect_mode
1085 compressMode repository_dir/compress_mode
1086
1087 We also add groups of the following items, one list element per item::
1088
1089 relativeExcludePaths dir/exclude/rel_path
1090 excludePatterns dir/exclude/pattern
1091
1092 The <repository_dir> node itself is created as the next child of the
1093 parent node. This method only adds one repository node. The parent must
1094 loop for each repository dir in the C{SubversionConfig} object.
1095
1096 If C{repositoryDir} is C{None}, this method call will be a no-op.
1097
1098 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1099 @param parentNode: Parent that the section should be appended to.
1100 @param repositoryDir: Repository dir to be added to the document.
1101 """
1102 if repositoryDir is not None:
1103 sectionNode = addContainerNode(xmlDom, parentNode, "repository_dir")
1104 addStringNode(xmlDom, sectionNode, "type", repositoryDir.repositoryType)
1105 addStringNode(xmlDom, sectionNode, "abs_path", repositoryDir.directoryPath)
1106 addStringNode(xmlDom, sectionNode, "collect_mode", repositoryDir.collectMode)
1107 addStringNode(xmlDom, sectionNode, "compress_mode", repositoryDir.compressMode)
1108 if ((repositoryDir.relativeExcludePaths is not None and repositoryDir.relativeExcludePaths != []) or
1109 (repositoryDir.excludePatterns is not None and repositoryDir.excludePatterns != [])):
1110 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1111 if repositoryDir.relativeExcludePaths is not None:
1112 for relativePath in repositoryDir.relativeExcludePaths:
1113 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1114 if repositoryDir.excludePatterns is not None:
1115 for pattern in repositoryDir.excludePatterns:
1116 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127 -def executeAction(configPath, options, config):
1128 """
1129 Executes the Subversion backup action.
1130
1131 @param configPath: Path to configuration file on disk.
1132 @type configPath: String representing a path on disk.
1133
1134 @param options: Program command-line options.
1135 @type options: Options object.
1136
1137 @param config: Program configuration.
1138 @type config: Config object.
1139
1140 @raise ValueError: Under many generic error conditions
1141 @raise IOError: If a backup could not be written for some reason.
1142 """
1143 logger.debug("Executing Subversion extended action.")
1144 if config.options is None or config.collect is None:
1145 raise ValueError("Cedar Backup configuration is not properly filled in.")
1146 local = LocalConfig(xmlPath=configPath)
1147 todayIsStart = isStartOfWeek(config.options.startingDay)
1148 fullBackup = options.full or todayIsStart
1149 logger.debug("Full backup flag is [%s]", fullBackup)
1150 if local.subversion.repositories is not None:
1151 for repository in local.subversion.repositories:
1152 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1153 if local.subversion.repositoryDirs is not None:
1154 for repositoryDir in local.subversion.repositoryDirs:
1155 logger.debug("Working with repository directory [%s].", repositoryDir.directoryPath)
1156 for repositoryPath in _getRepositoryPaths(repositoryDir):
1157 repository = Repository(repositoryDir.repositoryType, repositoryPath,
1158 repositoryDir.collectMode, repositoryDir.compressMode)
1159 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1160 logger.info("Completed backing up Subversion repository directory [%s].", repositoryDir.directoryPath)
1161 logger.info("Executed the Subversion extended action successfully.")
1162
1176
1191
1193 """
1194 Gets the path to the revision file associated with a repository.
1195 @param config: Config object.
1196 @param repository: Repository object.
1197 @return: Absolute path to the revision file associated with the repository.
1198 """
1199 normalized = buildNormalizedPath(repository.repositoryPath)
1200 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1201 revisionPath = os.path.join(config.options.workingDir, filename)
1202 logger.debug("Revision file path is [%s]", revisionPath)
1203 return revisionPath
1204
1205 -def _getBackupPath(config, repositoryPath, compressMode, startRevision, endRevision):
1206 """
1207 Gets the backup file path (including correct extension) associated with a repository.
1208 @param config: Config object.
1209 @param repositoryPath: Path to the indicated repository
1210 @param compressMode: Compress mode to use for this repository.
1211 @param startRevision: Starting repository revision.
1212 @param endRevision: Ending repository revision.
1213 @return: Absolute path to the backup file associated with the repository.
1214 """
1215 normalizedPath = buildNormalizedPath(repositoryPath)
1216 filename = "svndump-%d:%d-%s.txt" % (startRevision, endRevision, normalizedPath)
1217 if compressMode == 'gzip':
1218 filename = "%s.gz" % filename
1219 elif compressMode == 'bzip2':
1220 filename = "%s.bz2" % filename
1221 backupPath = os.path.join(config.collect.targetDir, filename)
1222 logger.debug("Backup file path is [%s]", backupPath)
1223 return backupPath
1224
1238
1240 """
1241 Gets exclusions (file and patterns) associated with an repository directory.
1242
1243 The returned files value is a list of absolute paths to be excluded from the
1244 backup for a given directory. It is derived from the repository directory's
1245 relative exclude paths.
1246
1247 The returned patterns value is a list of patterns to be excluded from the
1248 backup for a given directory. It is derived from the repository directory's
1249 list of patterns.
1250
1251 @param repositoryDir: Repository directory object.
1252
1253 @return: Tuple (files, patterns) indicating what to exclude.
1254 """
1255 paths = []
1256 if repositoryDir.relativeExcludePaths is not None:
1257 for relativePath in repositoryDir.relativeExcludePaths:
1258 paths.append(os.path.join(repositoryDir.directoryPath, relativePath))
1259 patterns = []
1260 if repositoryDir.excludePatterns is not None:
1261 patterns.extend(repositoryDir.excludePatterns)
1262 logger.debug("Exclude paths: %s", paths)
1263 logger.debug("Exclude patterns: %s", patterns)
1264 return(paths, patterns)
1265
1267 """
1268 Backs up an individual Subversion repository.
1269
1270 This internal method wraps the public methods and adds some functionality
1271 to work better with the extended action itself.
1272
1273 @param config: Cedar Backup configuration.
1274 @param local: Local configuration
1275 @param todayIsStart: Indicates whether today is start of week
1276 @param fullBackup: Full backup flag
1277 @param repository: Repository to operate on
1278
1279 @raise ValueError: If some value is missing or invalid.
1280 @raise IOError: If there is a problem executing the Subversion dump.
1281 """
1282 logger.debug("Working with repository [%s]", repository.repositoryPath)
1283 logger.debug("Repository type is [%s]", repository.repositoryType)
1284 collectMode = _getCollectMode(local, repository)
1285 compressMode = _getCompressMode(local, repository)
1286 revisionPath = _getRevisionPath(config, repository)
1287 if not (fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart)):
1288 logger.debug("Repository will not be backed up, per collect mode.")
1289 return
1290 logger.debug("Repository meets criteria to be backed up today.")
1291 if collectMode != "incr" or fullBackup:
1292 startRevision = 0
1293 endRevision = getYoungestRevision(repository.repositoryPath)
1294 logger.debug("Using full backup, revision: (%d, %d).", startRevision, endRevision)
1295 else:
1296 if fullBackup:
1297 startRevision = 0
1298 endRevision = getYoungestRevision(repository.repositoryPath)
1299 else:
1300 startRevision = _loadLastRevision(revisionPath) + 1
1301 endRevision = getYoungestRevision(repository.repositoryPath)
1302 if startRevision > endRevision:
1303 logger.info("No need to back up repository [%s]; no new revisions.", repository.repositoryPath)
1304 return
1305 logger.debug("Using incremental backup, revision: (%d, %d).", startRevision, endRevision)
1306 backupPath = _getBackupPath(config, repository.repositoryPath, compressMode, startRevision, endRevision)
1307 with _getOutputFile(backupPath, compressMode) as outputFile:
1308 backupRepository(repository.repositoryPath, outputFile, startRevision, endRevision)
1309 if not os.path.exists(backupPath):
1310 raise IOError("Dump file [%s] does not seem to exist after backup completed." % backupPath)
1311 changeOwnership(backupPath, config.options.backupUser, config.options.backupGroup)
1312 if collectMode == "incr":
1313 _writeLastRevision(config, revisionPath, endRevision)
1314 logger.info("Completed backing up Subversion repository [%s].", repository.repositoryPath)
1315
1317 """
1318 Opens the output file used for saving the Subversion dump.
1319
1320 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1321 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1322 return an object from the normal C{open()} method.
1323
1324 @param backupPath: Path to file to open.
1325 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1326
1327 @return: Output file object, opened in binary mode for use with executeCommand()
1328 """
1329 if compressMode == "gzip":
1330 return GzipFile(backupPath, "wb")
1331 elif compressMode == "bzip2":
1332 return BZ2File(backupPath, "wb")
1333 else:
1334 return open(backupPath, "wb")
1335
1337 """
1338 Loads the indicated revision file from disk into an integer.
1339
1340 If we can't load the revision file successfully (either because it doesn't
1341 exist or for some other reason), then a revision of -1 will be returned -
1342 but the condition will be logged. This way, we err on the side of backing
1343 up too much, because anyone using this will presumably be adding 1 to the
1344 revision, so they don't duplicate any backups.
1345
1346 @param revisionPath: Path to the revision file on disk.
1347
1348 @return: Integer representing last backed-up revision, -1 on error or if none can be read.
1349 """
1350 if not os.path.isfile(revisionPath):
1351 startRevision = -1
1352 logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
1353 else:
1354 try:
1355 with open(revisionPath, "rb") as f:
1356 startRevision = pickle.load(f, fix_imports=True)
1357 logger.debug("Loaded revision file [%s] from disk: %d.", revisionPath, startRevision)
1358 except Exception as e:
1359 startRevision = -1
1360 logger.error("Failed loading revision file [%s] from disk: %s", revisionPath, e)
1361 return startRevision
1362
1364 """
1365 Writes the end revision to the indicated revision file on disk.
1366
1367 If we can't write the revision file successfully for any reason, we'll log
1368 the condition but won't throw an exception.
1369
1370 @param config: Config object.
1371 @param revisionPath: Path to the revision file on disk.
1372 @param endRevision: Last revision backed up on this run.
1373 """
1374 try:
1375 with open(revisionPath, "wb") as f:
1376 pickle.dump(endRevision, f, 0, fix_imports=True)
1377 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1378 logger.debug("Wrote new revision file [%s] to disk: %d.", revisionPath, endRevision)
1379 except Exception as e:
1380 logger.error("Failed to write revision file [%s] to disk: %s", revisionPath, e)
1381
1382
1383
1384
1385
1386
1387 -def backupRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1388 """
1389 Backs up an individual Subversion repository.
1390
1391 The starting and ending revision values control an incremental backup. If
1392 the starting revision is not passed in, then revision zero (the start of the
1393 repository) is assumed. If the ending revision is not passed in, then the
1394 youngest revision in the database will be used as the endpoint.
1395
1396 The backup data will be written into the passed-in back file. Normally,
1397 this would be an object as returned from C{open}, but it is possible to use
1398 something like a C{GzipFile} to write compressed output. The caller is
1399 responsible for closing the passed-in backup file.
1400
1401 @note: This function should either be run as root or as the owner of the
1402 Subversion repository.
1403
1404 @note: It is apparently I{not} a good idea to interrupt this function.
1405 Sometimes, this leaves the repository in a "wedged" state, which requires
1406 recovery using C{svnadmin recover}.
1407
1408 @param repositoryPath: Path to Subversion repository to back up
1409 @type repositoryPath: String path representing Subversion repository on disk.
1410
1411 @param backupFile: Python file object to use for writing backup.
1412 @type backupFile: Python file object as from C{open()} or C{file()}.
1413
1414 @param startRevision: Starting repository revision to back up (for incremental backups)
1415 @type startRevision: Integer value >= 0.
1416
1417 @param endRevision: Ending repository revision to back up (for incremental backups)
1418 @type endRevision: Integer value >= 0.
1419
1420 @raise ValueError: If some value is missing or invalid.
1421 @raise IOError: If there is a problem executing the Subversion dump.
1422 """
1423 if startRevision is None:
1424 startRevision = 0
1425 if endRevision is None:
1426 endRevision = getYoungestRevision(repositoryPath)
1427 if int(startRevision) < 0:
1428 raise ValueError("Start revision must be >= 0.")
1429 if int(endRevision) < 0:
1430 raise ValueError("End revision must be >= 0.")
1431 if startRevision > endRevision:
1432 raise ValueError("Start revision must be <= end revision.")
1433 args = [ "dump", "--quiet", "-r%s:%s" % (startRevision, endRevision), "--incremental", repositoryPath, ]
1434 command = resolveCommand(SVNADMIN_COMMAND)
1435 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
1436 if result != 0:
1437 raise IOError("Error [%d] executing Subversion dump for repository [%s]." % (result, repositoryPath))
1438 logger.debug("Completed dumping subversion repository [%s].", repositoryPath)
1439
1446 """
1447 Gets the youngest (newest) revision in a Subversion repository using C{svnlook}.
1448
1449 @note: This function should either be run as root or as the owner of the
1450 Subversion repository.
1451
1452 @param repositoryPath: Path to Subversion repository to look in.
1453 @type repositoryPath: String path representing Subversion repository on disk.
1454
1455 @return: Youngest revision as an integer.
1456
1457 @raise ValueError: If there is a problem parsing the C{svnlook} output.
1458 @raise IOError: If there is a problem executing the C{svnlook} command.
1459 """
1460 args = [ 'youngest', repositoryPath, ]
1461 command = resolveCommand(SVNLOOK_COMMAND)
1462 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=True)
1463 if result != 0:
1464 raise IOError("Error [%d] executing 'svnlook youngest' for repository [%s]." % (result, repositoryPath))
1465 if len(output) != 1:
1466 raise ValueError("Unable to parse 'svnlook youngest' output.")
1467 return int(output[0])
1468
1475
1476 """
1477 Class representing Subversion BDB (Berkeley Database) repository configuration.
1478 This object is deprecated. Use a simple L{Repository} instead.
1479 """
1480
1481 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1486
1492
1495
1496 """
1497 Class representing Subversion FSFS repository configuration.
1498 This object is deprecated. Use a simple L{Repository} instead.
1499 """
1500
1501 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1506
1512
1513
1514 -def backupBDBRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1515 """
1516 Backs up an individual Subversion BDB repository.
1517 This function is deprecated. Use L{backupRepository} instead.
1518 """
1519 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1520
1523 """
1524 Backs up an individual Subversion FSFS repository.
1525 This function is deprecated. Use L{backupRepository} instead.
1526 """
1527 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1528