1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides backup peer-related objects and utility functions.
40
41 @sort: LocalPeer, RemotePeer
42
43 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
44 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
45
46 @author: Kenneth J. Pronovici <pronovic@ieee.org>
47 """
48
49
50
51
52
53
54
55 import os
56 import logging
57 import shutil
58
59
60 from CedarBackup3.filesystem import FilesystemList
61 from CedarBackup3.util import resolveCommand, executeCommand, isRunningAsRoot
62 from CedarBackup3.util import splitCommandLine, encodePath
63 from CedarBackup3.config import VALID_FAILURE_MODES
64
65
66
67
68
69
70 logger = logging.getLogger("CedarBackup3.log.peer")
71
72 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
73 DEF_RSH_COMMAND = [ "/usr/bin/ssh", ]
74 DEF_CBACK_COMMAND = "/usr/bin/cback3"
75
76 DEF_COLLECT_INDICATOR = "cback.collect"
77 DEF_STAGE_INDICATOR = "cback.stage"
78
79 SU_COMMAND = [ "su" ]
87
88
89
90
91
92 """
93 Backup peer representing a local peer in a backup pool.
94
95 This is a class representing a local (non-network) peer in a backup pool.
96 Local peers are backed up by simple filesystem copy operations. A local
97 peer has associated with it a name (typically, but not necessarily, a
98 hostname) and a collect directory.
99
100 The public methods other than the constructor are part of a "backup peer"
101 interface shared with the C{RemotePeer} class.
102
103 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
104 _copyLocalDir, _copyLocalFile, name, collectDir
105 """
106
107
108
109
110
111 - def __init__(self, name, collectDir, ignoreFailureMode=None):
112 """
113 Initializes a local backup peer.
114
115 Note that the collect directory must be an absolute path, but does not
116 have to exist when the object is instantiated. We do a lazy validation
117 on this value since we could (potentially) be creating peer objects
118 before an ongoing backup completed.
119
120 @param name: Name of the backup peer
121 @type name: String, typically a hostname
122
123 @param collectDir: Path to the peer's collect directory
124 @type collectDir: String representing an absolute local path on disk
125
126 @param ignoreFailureMode: Ignore failure mode for this peer
127 @type ignoreFailureMode: One of VALID_FAILURE_MODES
128
129 @raise ValueError: If the name is empty.
130 @raise ValueError: If collect directory is not an absolute path.
131 """
132 self._name = None
133 self._collectDir = None
134 self._ignoreFailureMode = None
135 self.name = name
136 self.collectDir = collectDir
137 self.ignoreFailureMode = ignoreFailureMode
138
139
140
141
142
143
145 """
146 Property target used to set the peer name.
147 The value must be a non-empty string and cannot be C{None}.
148 @raise ValueError: If the value is an empty string or C{None}.
149 """
150 if value is None or len(value) < 1:
151 raise ValueError("Peer name must be a non-empty string.")
152 self._name = value
153
155 """
156 Property target used to get the peer name.
157 """
158 return self._name
159
161 """
162 Property target used to set the collect directory.
163 The value must be an absolute path and cannot be C{None}.
164 It does not have to exist on disk at the time of assignment.
165 @raise ValueError: If the value is C{None} or is not an absolute path.
166 @raise ValueError: If a path cannot be encoded properly.
167 """
168 if value is None or not os.path.isabs(value):
169 raise ValueError("Collect directory must be an absolute path.")
170 self._collectDir = encodePath(value)
171
173 """
174 Property target used to get the collect directory.
175 """
176 return self._collectDir
177
179 """
180 Property target used to set the ignoreFailure mode.
181 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
182 @raise ValueError: If the value is not valid.
183 """
184 if value is not None:
185 if value not in VALID_FAILURE_MODES:
186 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
187 self._ignoreFailureMode = value
188
190 """
191 Property target used to get the ignoreFailure mode.
192 """
193 return self._ignoreFailureMode
194
195 name = property(_getName, _setName, None, "Name of the peer.")
196 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
197 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
198
199
200
201
202
203
204 - def stagePeer(self, targetDir, ownership=None, permissions=None):
205 """
206 Stages data from the peer into the indicated local target directory.
207
208 The collect and target directories must both already exist before this
209 method is called. If passed in, ownership and permissions will be
210 applied to the files that are copied.
211
212 @note: The caller is responsible for checking that the indicator exists,
213 if they care. This function only stages the files within the directory.
214
215 @note: If you have user/group as strings, call the L{util.getUidGid} function
216 to get the associated uid/gid as an ownership tuple.
217
218 @param targetDir: Target directory to write data into
219 @type targetDir: String representing a directory on disk
220
221 @param ownership: Owner and group that the staged files should have
222 @type ownership: Tuple of numeric ids C{(uid, gid)}
223
224 @param permissions: Permissions that the staged files should have
225 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
226
227 @return: Number of files copied from the source directory to the target directory.
228
229 @raise ValueError: If collect directory is not a directory or does not exist
230 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
231 @raise ValueError: If a path cannot be encoded properly.
232 @raise IOError: If there were no files to stage (i.e. the directory was empty)
233 @raise IOError: If there is an IO error copying a file.
234 @raise OSError: If there is an OS error copying or changing permissions on a file
235 """
236 targetDir = encodePath(targetDir)
237 if not os.path.isabs(targetDir):
238 logger.debug("Target directory [%s] not an absolute path.", targetDir)
239 raise ValueError("Target directory must be an absolute path.")
240 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
241 logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir)
242 raise ValueError("Collect directory is not a directory or does not exist on disk.")
243 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
244 logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir)
245 raise ValueError("Target directory is not a directory or does not exist on disk.")
246 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
247 if count == 0:
248 raise IOError("Did not copy any files from local peer.")
249 return count
250
252 """
253 Checks the collect indicator in the peer's staging directory.
254
255 When a peer has completed collecting its backup files, it will write an
256 empty indicator file into its collect directory. This method checks to
257 see whether that indicator has been written. We're "stupid" here - if
258 the collect directory doesn't exist, you'll naturally get back C{False}.
259
260 If you need to, you can override the name of the collect indicator file
261 by passing in a different name.
262
263 @param collectIndicator: Name of the collect indicator file to check
264 @type collectIndicator: String representing name of a file in the collect directory
265
266 @return: Boolean true/false depending on whether the indicator exists.
267 @raise ValueError: If a path cannot be encoded properly.
268 """
269 collectIndicator = encodePath(collectIndicator)
270 if collectIndicator is None:
271 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
272 else:
273 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
274
276 """
277 Writes the stage indicator in the peer's staging directory.
278
279 When the master has completed collecting its backup files, it will write
280 an empty indicator file into the peer's collect directory. The presence
281 of this file implies that the staging process is complete.
282
283 If you need to, you can override the name of the stage indicator file by
284 passing in a different name.
285
286 @note: If you have user/group as strings, call the L{util.getUidGid}
287 function to get the associated uid/gid as an ownership tuple.
288
289 @param stageIndicator: Name of the indicator file to write
290 @type stageIndicator: String representing name of a file in the collect directory
291
292 @param ownership: Owner and group that the indicator file should have
293 @type ownership: Tuple of numeric ids C{(uid, gid)}
294
295 @param permissions: Permissions that the indicator file should have
296 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
297
298 @raise ValueError: If collect directory is not a directory or does not exist
299 @raise ValueError: If a path cannot be encoded properly.
300 @raise IOError: If there is an IO error creating the file.
301 @raise OSError: If there is an OS error creating or changing permissions on the file
302 """
303 stageIndicator = encodePath(stageIndicator)
304 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
305 logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir)
306 raise ValueError("Collect directory is not a directory or does not exist on disk.")
307 if stageIndicator is None:
308 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
309 else:
310 fileName = os.path.join(self.collectDir, stageIndicator)
311 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
312
313
314
315
316
317
318 @staticmethod
319 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
320 """
321 Copies files from the source directory to the target directory.
322
323 This function is not recursive. Only the files in the directory will be
324 copied. Ownership and permissions will be left at their default values
325 if new values are not specified. The source and target directories are
326 allowed to be soft links to a directory, but besides that soft links are
327 ignored.
328
329 @note: If you have user/group as strings, call the L{util.getUidGid}
330 function to get the associated uid/gid as an ownership tuple.
331
332 @param sourceDir: Source directory
333 @type sourceDir: String representing a directory on disk
334
335 @param targetDir: Target directory
336 @type targetDir: String representing a directory on disk
337
338 @param ownership: Owner and group that the copied files should have
339 @type ownership: Tuple of numeric ids C{(uid, gid)}
340
341 @param permissions: Permissions that the staged files should have
342 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
343
344 @return: Number of files copied from the source directory to the target directory.
345
346 @raise ValueError: If source or target is not a directory or does not exist.
347 @raise ValueError: If a path cannot be encoded properly.
348 @raise IOError: If there is an IO error copying the files.
349 @raise OSError: If there is an OS error copying or changing permissions on a files
350 """
351 filesCopied = 0
352 sourceDir = encodePath(sourceDir)
353 targetDir = encodePath(targetDir)
354 for fileName in os.listdir(sourceDir):
355 sourceFile = os.path.join(sourceDir, fileName)
356 targetFile = os.path.join(targetDir, fileName)
357 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
358 filesCopied += 1
359 return filesCopied
360
361 @staticmethod
362 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
363 """
364 Copies a source file to a target file.
365
366 If the source file is C{None} then the target file will be created or
367 overwritten as an empty file. If the target file is C{None}, this method
368 is a no-op. Attempting to copy a soft link or a directory will result in
369 an exception.
370
371 @note: If you have user/group as strings, call the L{util.getUidGid}
372 function to get the associated uid/gid as an ownership tuple.
373
374 @note: We will not overwrite a target file that exists when this method
375 is invoked. If the target already exists, we'll raise an exception.
376
377 @param sourceFile: Source file to copy
378 @type sourceFile: String representing a file on disk, as an absolute path
379
380 @param targetFile: Target file to create
381 @type targetFile: String representing a file on disk, as an absolute path
382
383 @param ownership: Owner and group that the copied should have
384 @type ownership: Tuple of numeric ids C{(uid, gid)}
385
386 @param permissions: Permissions that the staged files should have
387 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
388
389 @param overwrite: Indicates whether it's OK to overwrite the target file.
390 @type overwrite: Boolean true/false.
391
392 @raise ValueError: If the passed-in source file is not a regular file.
393 @raise ValueError: If a path cannot be encoded properly.
394 @raise IOError: If the target file already exists.
395 @raise IOError: If there is an IO error copying the file
396 @raise OSError: If there is an OS error copying or changing permissions on a file
397 """
398 targetFile = encodePath(targetFile)
399 sourceFile = encodePath(sourceFile)
400 if targetFile is None:
401 return
402 if not overwrite:
403 if os.path.exists(targetFile):
404 raise IOError("Target file [%s] already exists." % targetFile)
405 if sourceFile is None:
406 with open(targetFile, "w") as f:
407 f.write("")
408 else:
409 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
410 shutil.copy(sourceFile, targetFile)
411 else:
412 logger.debug("Source [%s] is not a regular file.", sourceFile)
413 raise ValueError("Source is not a regular file.")
414 if ownership is not None:
415 os.chown(targetFile, ownership[0], ownership[1])
416 if permissions is not None:
417 os.chmod(targetFile, permissions)
418
425
426
427
428
429
430 """
431 Backup peer representing a remote peer in a backup pool.
432
433 This is a class representing a remote (networked) peer in a backup pool.
434 Remote peers are backed up using an rcp-compatible copy command. A remote
435 peer has associated with it a name (which must be a valid hostname), a
436 collect directory, a working directory and a copy method (an rcp-compatible
437 command).
438
439 You can also set an optional local user value. This username will be used
440 as the local user for any remote copies that are required. It can only be
441 used if the root user is executing the backup. The root user will C{su} to
442 the local user and execute the remote copies as that user.
443
444 The copy method is associated with the peer and not with the actual request
445 to copy, because we can envision that each remote host might have a
446 different connect method.
447
448 The public methods other than the constructor are part of a "backup peer"
449 interface shared with the C{LocalPeer} class.
450
451 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
452 executeRemoteCommand, executeManagedAction, _getDirContents,
453 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir,
454 remoteUser, rcpCommand, rshCommand, cbackCommand
455 """
456
457
458
459
460
461 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None,
462 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None,
463 ignoreFailureMode=None):
464 """
465 Initializes a remote backup peer.
466
467 @note: If provided, each command will eventually be parsed into a list of
468 strings suitable for passing to C{util.executeCommand} in order to avoid
469 security holes related to shell interpolation. This parsing will be
470 done by the L{util.splitCommandLine} function. See the documentation for
471 that function for some important notes about its limitations.
472
473 @param name: Name of the backup peer
474 @type name: String, must be a valid DNS hostname
475
476 @param collectDir: Path to the peer's collect directory
477 @type collectDir: String representing an absolute path on the remote peer
478
479 @param workingDir: Working directory that can be used to create temporary files, etc.
480 @type workingDir: String representing an absolute path on the current host.
481
482 @param remoteUser: Name of the Cedar Backup user on the remote peer
483 @type remoteUser: String representing a username, valid via remote shell to the peer
484
485 @param localUser: Name of the Cedar Backup user on the current host
486 @type localUser: String representing a username, valid on the current host
487
488 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
489 @type rcpCommand: String representing a system command including required arguments
490
491 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
492 @type rshCommand: String representing a system command including required arguments
493
494 @param cbackCommand: A chack-compatible command to use for executing managed actions
495 @type cbackCommand: String representing a system command including required arguments
496
497 @param ignoreFailureMode: Ignore failure mode for this peer
498 @type ignoreFailureMode: One of VALID_FAILURE_MODES
499
500 @raise ValueError: If collect directory is not an absolute path
501 """
502 self._name = None
503 self._collectDir = None
504 self._workingDir = None
505 self._remoteUser = None
506 self._localUser = None
507 self._rcpCommand = None
508 self._rcpCommandList = None
509 self._rshCommand = None
510 self._rshCommandList = None
511 self._cbackCommand = None
512 self._ignoreFailureMode = None
513 self.name = name
514 self.collectDir = collectDir
515 self.workingDir = workingDir
516 self.remoteUser = remoteUser
517 self.localUser = localUser
518 self.rcpCommand = rcpCommand
519 self.rshCommand = rshCommand
520 self.cbackCommand = cbackCommand
521 self.ignoreFailureMode = ignoreFailureMode
522
523
524
525
526
527
529 """
530 Property target used to set the peer name.
531 The value must be a non-empty string and cannot be C{None}.
532 @raise ValueError: If the value is an empty string or C{None}.
533 """
534 if value is None or len(value) < 1:
535 raise ValueError("Peer name must be a non-empty string.")
536 self._name = value
537
539 """
540 Property target used to get the peer name.
541 """
542 return self._name
543
545 """
546 Property target used to set the collect directory.
547 The value must be an absolute path and cannot be C{None}.
548 It does not have to exist on disk at the time of assignment.
549 @raise ValueError: If the value is C{None} or is not an absolute path.
550 @raise ValueError: If the value cannot be encoded properly.
551 """
552 if value is not None:
553 if not os.path.isabs(value):
554 raise ValueError("Collect directory must be an absolute path.")
555 self._collectDir = encodePath(value)
556
558 """
559 Property target used to get the collect directory.
560 """
561 return self._collectDir
562
564 """
565 Property target used to set the working directory.
566 The value must be an absolute path and cannot be C{None}.
567 @raise ValueError: If the value is C{None} or is not an absolute path.
568 @raise ValueError: If the value cannot be encoded properly.
569 """
570 if value is not None:
571 if not os.path.isabs(value):
572 raise ValueError("Working directory must be an absolute path.")
573 self._workingDir = encodePath(value)
574
576 """
577 Property target used to get the working directory.
578 """
579 return self._workingDir
580
582 """
583 Property target used to set the remote user.
584 The value must be a non-empty string and cannot be C{None}.
585 @raise ValueError: If the value is an empty string or C{None}.
586 """
587 if value is None or len(value) < 1:
588 raise ValueError("Peer remote user must be a non-empty string.")
589 self._remoteUser = value
590
592 """
593 Property target used to get the remote user.
594 """
595 return self._remoteUser
596
598 """
599 Property target used to set the local user.
600 The value must be a non-empty string if it is not C{None}.
601 @raise ValueError: If the value is an empty string.
602 """
603 if value is not None:
604 if len(value) < 1:
605 raise ValueError("Peer local user must be a non-empty string.")
606 self._localUser = value
607
609 """
610 Property target used to get the local user.
611 """
612 return self._localUser
613
615 """
616 Property target to set the rcp command.
617
618 The value must be a non-empty string or C{None}. Its value is stored in
619 the two forms: "raw" as provided by the client, and "parsed" into a list
620 suitable for being passed to L{util.executeCommand} via
621 L{util.splitCommandLine}.
622
623 However, all the caller will ever see via the property is the actual
624 value they set (which includes seeing C{None}, even if we translate that
625 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
626 C{self._rcpCommandList} if we want the actual command list.
627
628 @raise ValueError: If the value is an empty string.
629 """
630 if value is None:
631 self._rcpCommand = None
632 self._rcpCommandList = DEF_RCP_COMMAND
633 else:
634 if len(value) >= 1:
635 self._rcpCommand = value
636 self._rcpCommandList = splitCommandLine(self._rcpCommand)
637 else:
638 raise ValueError("The rcp command must be a non-empty string.")
639
641 """
642 Property target used to get the rcp command.
643 """
644 return self._rcpCommand
645
647 """
648 Property target to set the rsh command.
649
650 The value must be a non-empty string or C{None}. Its value is stored in
651 the two forms: "raw" as provided by the client, and "parsed" into a list
652 suitable for being passed to L{util.executeCommand} via
653 L{util.splitCommandLine}.
654
655 However, all the caller will ever see via the property is the actual
656 value they set (which includes seeing C{None}, even if we translate that
657 internally to C{DEF_RSH_COMMAND}). Internally, we should always use
658 C{self._rshCommandList} if we want the actual command list.
659
660 @raise ValueError: If the value is an empty string.
661 """
662 if value is None:
663 self._rshCommand = None
664 self._rshCommandList = DEF_RSH_COMMAND
665 else:
666 if len(value) >= 1:
667 self._rshCommand = value
668 self._rshCommandList = splitCommandLine(self._rshCommand)
669 else:
670 raise ValueError("The rsh command must be a non-empty string.")
671
673 """
674 Property target used to get the rsh command.
675 """
676 return self._rshCommand
677
679 """
680 Property target to set the cback command.
681
682 The value must be a non-empty string or C{None}. Unlike the other
683 command, this value is only stored in the "raw" form provided by the
684 client.
685
686 @raise ValueError: If the value is an empty string.
687 """
688 if value is None:
689 self._cbackCommand = None
690 else:
691 if len(value) >= 1:
692 self._cbackCommand = value
693 else:
694 raise ValueError("The cback command must be a non-empty string.")
695
697 """
698 Property target used to get the cback command.
699 """
700 return self._cbackCommand
701
703 """
704 Property target used to set the ignoreFailure mode.
705 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
706 @raise ValueError: If the value is not valid.
707 """
708 if value is not None:
709 if value not in VALID_FAILURE_MODES:
710 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
711 self._ignoreFailureMode = value
712
714 """
715 Property target used to get the ignoreFailure mode.
716 """
717 return self._ignoreFailureMode
718
719 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
720 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
721 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
722 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
723 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
724 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
725 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.")
726 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.")
727 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
728
729
730
731
732
733
734 - def stagePeer(self, targetDir, ownership=None, permissions=None):
735 """
736 Stages data from the peer into the indicated local target directory.
737
738 The target directory must already exist before this method is called. If
739 passed in, ownership and permissions will be applied to the files that
740 are copied.
741
742 @note: The returned count of copied files might be inaccurate if some of
743 the copied files already existed in the staging directory prior to the
744 copy taking place. We don't clear the staging directory first, because
745 some extension might also be using it.
746
747 @note: If you have user/group as strings, call the L{util.getUidGid} function
748 to get the associated uid/gid as an ownership tuple.
749
750 @note: Unlike the local peer version of this method, an I/O error might
751 or might not be raised if the directory is empty. Since we're using a
752 remote copy method, we just don't have the fine-grained control over our
753 exceptions that's available when we can look directly at the filesystem,
754 and we can't control whether the remote copy method thinks an empty
755 directory is an error.
756
757 @param targetDir: Target directory to write data into
758 @type targetDir: String representing a directory on disk
759
760 @param ownership: Owner and group that the staged files should have
761 @type ownership: Tuple of numeric ids C{(uid, gid)}
762
763 @param permissions: Permissions that the staged files should have
764 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
765
766 @return: Number of files copied from the source directory to the target directory.
767
768 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
769 @raise ValueError: If a path cannot be encoded properly.
770 @raise IOError: If there were no files to stage (i.e. the directory was empty)
771 @raise IOError: If there is an IO error copying a file.
772 @raise OSError: If there is an OS error copying or changing permissions on a file
773 """
774 targetDir = encodePath(targetDir)
775 if not os.path.isabs(targetDir):
776 logger.debug("Target directory [%s] not an absolute path.", targetDir)
777 raise ValueError("Target directory must be an absolute path.")
778 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
779 logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir)
780 raise ValueError("Target directory is not a directory or does not exist on disk.")
781 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
782 self._rcpCommand, self._rcpCommandList,
783 self.collectDir, targetDir,
784 ownership, permissions)
785 if count == 0:
786 raise IOError("Did not copy any files from local peer.")
787 return count
788
790 """
791 Checks the collect indicator in the peer's staging directory.
792
793 When a peer has completed collecting its backup files, it will write an
794 empty indicator file into its collect directory. This method checks to
795 see whether that indicator has been written. If the remote copy command
796 fails, we return C{False} as if the file weren't there.
797
798 If you need to, you can override the name of the collect indicator file
799 by passing in a different name.
800
801 @note: Apparently, we can't count on all rcp-compatible implementations
802 to return sensible errors for some error conditions. As an example, the
803 C{scp} command in Debian 'woody' returns a zero (normal) status even when
804 it can't find a host or if the login or path is invalid. Because of
805 this, the implementation of this method is rather convoluted.
806
807 @param collectIndicator: Name of the collect indicator file to check
808 @type collectIndicator: String representing name of a file in the collect directory
809
810 @return: Boolean true/false depending on whether the indicator exists.
811 @raise ValueError: If a path cannot be encoded properly.
812 """
813 try:
814 if collectIndicator is None:
815 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
816 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
817 else:
818 collectIndicator = encodePath(collectIndicator)
819 sourceFile = os.path.join(self.collectDir, collectIndicator)
820 targetFile = os.path.join(self.workingDir, collectIndicator)
821 logger.debug("Fetch remote [%s] into [%s].", sourceFile, targetFile)
822 if os.path.exists(targetFile):
823 try:
824 os.remove(targetFile)
825 except:
826 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
827 try:
828 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
829 self._rcpCommand, self._rcpCommandList,
830 sourceFile, targetFile,
831 overwrite=False)
832 if os.path.exists(targetFile):
833 return True
834 else:
835 return False
836 except Exception as e:
837 logger.info("Failed looking for collect indicator: %s", e)
838 return False
839 finally:
840 if os.path.exists(targetFile):
841 try:
842 os.remove(targetFile)
843 except: pass
844
846 """
847 Writes the stage indicator in the peer's staging directory.
848
849 When the master has completed collecting its backup files, it will write
850 an empty indicator file into the peer's collect directory. The presence
851 of this file implies that the staging process is complete.
852
853 If you need to, you can override the name of the stage indicator file by
854 passing in a different name.
855
856 @note: If you have user/group as strings, call the L{util.getUidGid} function
857 to get the associated uid/gid as an ownership tuple.
858
859 @param stageIndicator: Name of the indicator file to write
860 @type stageIndicator: String representing name of a file in the collect directory
861
862 @raise ValueError: If a path cannot be encoded properly.
863 @raise IOError: If there is an IO error creating the file.
864 @raise OSError: If there is an OS error creating or changing permissions on the file
865 """
866 stageIndicator = encodePath(stageIndicator)
867 if stageIndicator is None:
868 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
869 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
870 else:
871 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
872 targetFile = os.path.join(self.collectDir, stageIndicator)
873 try:
874 if not os.path.exists(sourceFile):
875 with open(sourceFile, "w") as f:
876 f.write("")
877 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
878 self._rcpCommand, self._rcpCommandList,
879 sourceFile, targetFile)
880 finally:
881 if os.path.exists(sourceFile):
882 try:
883 os.remove(sourceFile)
884 except: pass
885
887 """
888 Executes a command on the peer via remote shell.
889
890 @param command: Command to execute
891 @type command: String command-line suitable for use with rsh.
892
893 @raise IOError: If there is an error executing the command on the remote peer.
894 """
895 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser,
896 self.name, self._rshCommand,
897 self._rshCommandList, command)
898
900 """
901 Executes a managed action on this peer.
902
903 @param action: Name of the action to execute.
904 @param fullBackup: Whether a full backup should be executed.
905
906 @raise IOError: If there is an error executing the action on the remote peer.
907 """
908 try:
909 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup)
910 self.executeRemoteCommand(command)
911 except IOError as e:
912 logger.info(e)
913 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
914
915
916
917
918
919
920 @staticmethod
921 - def _getDirContents(path):
922 """
923 Returns the contents of a directory in terms of a Set.
924
925 The directory's contents are read as a L{FilesystemList} containing only
926 files, and then the list is converted into a set object for later use.
927
928 @param path: Directory path to get contents for
929 @type path: String representing a path on disk
930
931 @return: Set of files in the directory
932 @raise ValueError: If path is not a directory or does not exist.
933 """
934 contents = FilesystemList()
935 contents.excludeDirs = True
936 contents.excludeLinks = True
937 contents.addDirContents(path)
938 return set(contents)
939
940 @staticmethod
941 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
942 sourceDir, targetDir, ownership=None, permissions=None):
943 """
944 Copies files from the source directory to the target directory.
945
946 This function is not recursive. Only the files in the directory will be
947 copied. Ownership and permissions will be left at their default values
948 if new values are not specified. Behavior when copying soft links from
949 the collect directory is dependent on the behavior of the specified rcp
950 command.
951
952 @note: The returned count of copied files might be inaccurate if some of
953 the copied files already existed in the staging directory prior to the
954 copy taking place. We don't clear the staging directory first, because
955 some extension might also be using it.
956
957 @note: If you have user/group as strings, call the L{util.getUidGid} function
958 to get the associated uid/gid as an ownership tuple.
959
960 @note: We don't have a good way of knowing exactly what files we copied
961 down from the remote peer, unless we want to parse the output of the rcp
962 command (ugh). We could change permissions on everything in the target
963 directory, but that's kind of ugly too. Instead, we use Python's set
964 functionality to figure out what files were added while we executed the
965 rcp command. This isn't perfect - for instance, it's not correct if
966 someone else is messing with the directory at the same time we're doing
967 the remote copy - but it's about as good as we're going to get.
968
969 @note: Apparently, we can't count on all rcp-compatible implementations
970 to return sensible errors for some error conditions. As an example, the
971 C{scp} command in Debian 'woody' returns a zero (normal) status even
972 when it can't find a host or if the login or path is invalid. We try
973 to work around this by issuing C{IOError} if we don't copy any files from
974 the remote host.
975
976 @param remoteUser: Name of the Cedar Backup user on the remote peer
977 @type remoteUser: String representing a username, valid via the copy command
978
979 @param localUser: Name of the Cedar Backup user on the current host
980 @type localUser: String representing a username, valid on the current host
981
982 @param remoteHost: Hostname of the remote peer
983 @type remoteHost: String representing a hostname, accessible via the copy command
984
985 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
986 @type rcpCommand: String representing a system command including required arguments
987
988 @param rcpCommandList: An rcp-compatible copy command to use for copying files
989 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
990
991 @param sourceDir: Source directory
992 @type sourceDir: String representing a directory on disk
993
994 @param targetDir: Target directory
995 @type targetDir: String representing a directory on disk
996
997 @param ownership: Owner and group that the copied files should have
998 @type ownership: Tuple of numeric ids C{(uid, gid)}
999
1000 @param permissions: Permissions that the staged files should have
1001 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1002
1003 @return: Number of files copied from the source directory to the target directory.
1004
1005 @raise ValueError: If source or target is not a directory or does not exist.
1006 @raise IOError: If there is an IO error copying the files.
1007 """
1008 beforeSet = RemotePeer._getDirContents(targetDir)
1009 if localUser is not None:
1010 try:
1011 if not isRunningAsRoot():
1012 raise IOError("Only root can remote copy as another user.")
1013 except AttributeError: pass
1014 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
1015 command = resolveCommand(SU_COMMAND)
1016 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1017 if result != 0:
1018 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
1019 else:
1020 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
1021 command = resolveCommand(rcpCommandList)
1022 result = executeCommand(command, [copySource, targetDir])[0]
1023 if result != 0:
1024 raise IOError("Error (%d) copying files from remote host." % result)
1025 afterSet = RemotePeer._getDirContents(targetDir)
1026 if len(afterSet) == 0:
1027 raise IOError("Did not copy any files from remote peer.")
1028 differenceSet = afterSet.difference(beforeSet)
1029 if len(differenceSet) == 0:
1030 raise IOError("Apparently did not copy any new files from remote peer.")
1031 for targetFile in differenceSet:
1032 if ownership is not None:
1033 os.chown(targetFile, ownership[0], ownership[1])
1034 if permissions is not None:
1035 os.chmod(targetFile, permissions)
1036 return len(differenceSet)
1037
1038 @staticmethod
1039 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
1040 rcpCommand, rcpCommandList,
1041 sourceFile, targetFile, ownership=None,
1042 permissions=None, overwrite=True):
1043 """
1044 Copies a remote source file to a target file.
1045
1046 @note: Internally, we have to go through and escape any spaces in the
1047 source path with double-backslash, otherwise things get screwed up. It
1048 doesn't seem to be required in the target path. I hope this is portable
1049 to various different rcp methods, but I guess it might not be (all I have
1050 to test with is OpenSSH).
1051
1052 @note: If you have user/group as strings, call the L{util.getUidGid} function
1053 to get the associated uid/gid as an ownership tuple.
1054
1055 @note: We will not overwrite a target file that exists when this method
1056 is invoked. If the target already exists, we'll raise an exception.
1057
1058 @note: Apparently, we can't count on all rcp-compatible implementations
1059 to return sensible errors for some error conditions. As an example, the
1060 C{scp} command in Debian 'woody' returns a zero (normal) status even when
1061 it can't find a host or if the login or path is invalid. We try to work
1062 around this by issuing C{IOError} the target file does not exist when
1063 we're done.
1064
1065 @param remoteUser: Name of the Cedar Backup user on the remote peer
1066 @type remoteUser: String representing a username, valid via the copy command
1067
1068 @param remoteHost: Hostname of the remote peer
1069 @type remoteHost: String representing a hostname, accessible via the copy command
1070
1071 @param localUser: Name of the Cedar Backup user on the current host
1072 @type localUser: String representing a username, valid on the current host
1073
1074 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1075 @type rcpCommand: String representing a system command including required arguments
1076
1077 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1078 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1079
1080 @param sourceFile: Source file to copy
1081 @type sourceFile: String representing a file on disk, as an absolute path
1082
1083 @param targetFile: Target file to create
1084 @type targetFile: String representing a file on disk, as an absolute path
1085
1086 @param ownership: Owner and group that the copied should have
1087 @type ownership: Tuple of numeric ids C{(uid, gid)}
1088
1089 @param permissions: Permissions that the staged files should have
1090 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1091
1092 @param overwrite: Indicates whether it's OK to overwrite the target file.
1093 @type overwrite: Boolean true/false.
1094
1095 @raise IOError: If the target file already exists.
1096 @raise IOError: If there is an IO error copying the file
1097 @raise OSError: If there is an OS error changing permissions on the file
1098 """
1099 if not overwrite:
1100 if os.path.exists(targetFile):
1101 raise IOError("Target file [%s] already exists." % targetFile)
1102 if localUser is not None:
1103 try:
1104 if not isRunningAsRoot():
1105 raise IOError("Only root can remote copy as another user.")
1106 except AttributeError: pass
1107 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
1108 command = resolveCommand(SU_COMMAND)
1109 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1110 if result != 0:
1111 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
1112 else:
1113 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
1114 command = resolveCommand(rcpCommandList)
1115 result = executeCommand(command, [copySource, targetFile])[0]
1116 if result != 0:
1117 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile))
1118 if not os.path.exists(targetFile):
1119 raise IOError("Apparently unable to copy file from remote host.")
1120 if ownership is not None:
1121 os.chown(targetFile, ownership[0], ownership[1])
1122 if permissions is not None:
1123 os.chmod(targetFile, permissions)
1124
1125 @staticmethod
1126 - def _pushLocalFile(remoteUser, localUser, remoteHost,
1127 rcpCommand, rcpCommandList,
1128 sourceFile, targetFile, overwrite=True):
1129 """
1130 Copies a local source file to a remote host.
1131
1132 @note: We will not overwrite a target file that exists when this method
1133 is invoked. If the target already exists, we'll raise an exception.
1134
1135 @note: Internally, we have to go through and escape any spaces in the
1136 source and target paths with double-backslash, otherwise things get
1137 screwed up. I hope this is portable to various different rcp methods,
1138 but I guess it might not be (all I have to test with is OpenSSH).
1139
1140 @note: If you have user/group as strings, call the L{util.getUidGid} function
1141 to get the associated uid/gid as an ownership tuple.
1142
1143 @param remoteUser: Name of the Cedar Backup user on the remote peer
1144 @type remoteUser: String representing a username, valid via the copy command
1145
1146 @param localUser: Name of the Cedar Backup user on the current host
1147 @type localUser: String representing a username, valid on the current host
1148
1149 @param remoteHost: Hostname of the remote peer
1150 @type remoteHost: String representing a hostname, accessible via the copy command
1151
1152 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1153 @type rcpCommand: String representing a system command including required arguments
1154
1155 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1156 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1157
1158 @param sourceFile: Source file to copy
1159 @type sourceFile: String representing a file on disk, as an absolute path
1160
1161 @param targetFile: Target file to create
1162 @type targetFile: String representing a file on disk, as an absolute path
1163
1164 @param overwrite: Indicates whether it's OK to overwrite the target file.
1165 @type overwrite: Boolean true/false.
1166
1167 @raise IOError: If there is an IO error copying the file
1168 @raise OSError: If there is an OS error changing permissions on the file
1169 """
1170 if not overwrite:
1171 if os.path.exists(targetFile):
1172 raise IOError("Target file [%s] already exists." % targetFile)
1173 if localUser is not None:
1174 try:
1175 if not isRunningAsRoot():
1176 raise IOError("Only root can remote copy as another user.")
1177 except AttributeError: pass
1178 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1179 command = resolveCommand(SU_COMMAND)
1180 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1181 if result != 0:
1182 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1183 else:
1184 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1185 command = resolveCommand(rcpCommandList)
1186 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1187 if result != 0:
1188 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1189
1190 @staticmethod
1191 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1192 """
1193 Executes a command on the peer via remote shell.
1194
1195 @param remoteUser: Name of the Cedar Backup user on the remote peer
1196 @type remoteUser: String representing a username, valid on the remote host
1197
1198 @param localUser: Name of the Cedar Backup user on the current host
1199 @type localUser: String representing a username, valid on the current host
1200
1201 @param remoteHost: Hostname of the remote peer
1202 @type remoteHost: String representing a hostname, accessible via the copy command
1203
1204 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
1205 @type rshCommand: String representing a system command including required arguments
1206
1207 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer
1208 @type rshCommandList: Command as a list to be passed to L{util.executeCommand}
1209
1210 @param remoteCommand: The command to be executed on the remote host
1211 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.)
1212
1213 @raise IOError: If there is an error executing the remote command
1214 """
1215 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand)
1216 if localUser is not None:
1217 try:
1218 if not isRunningAsRoot():
1219 raise IOError("Only root can remote shell as another user.")
1220 except AttributeError: pass
1221 command = resolveCommand(SU_COMMAND)
1222 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1223 if result != 0:
1224 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand))
1225 else:
1226 command = resolveCommand(rshCommandList)
1227 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0]
1228 if result != 0:
1229 raise IOError("Command failed [%s]" % (actualCommand))
1230
1231 @staticmethod
1233 """
1234 Builds a Cedar Backup command line for the named action.
1235
1236 @note: If the cback command is None, then DEF_CBACK_COMMAND is used.
1237
1238 @param cbackCommand: cback command to execute, including required options
1239 @param action: Name of the action to execute.
1240 @param fullBackup: Whether a full backup should be executed.
1241
1242 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand.
1243 @raise ValueError: If action is None.
1244 """
1245 if action is None:
1246 raise ValueError("Action cannot be None.")
1247 if cbackCommand is None:
1248 cbackCommand = DEF_CBACK_COMMAND
1249 if fullBackup:
1250 return "%s --full %s" % (cbackCommand, action)
1251 else:
1252 return "%s %s" % (cbackCommand, action)
1253