Source code for CedarBackup3.peer

# -*- coding: iso-8859-1 -*-
# vim: set ft=python ts=3 sw=3 expandtab:
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#              C E D A R
#          S O L U T I O N S       "Software done right."
#           S O F T W A R E
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Copyright (c) 2004-2008,2010,2015 Kenneth J. Pronovici.
# All rights reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License,
# Version 2, as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Copies of the GNU General Public License are available from
# the Free Software Foundation website, http://www.gnu.org/.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Author   : Kenneth J. Pronovici <pronovic@ieee.org>
# Language : Python 3 (>= 3.4)
# Project  : Cedar Backup, release 3
# Purpose  : Provides backup peer-related objects.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

########################################################################
# Module documentation
########################################################################

"""
Provides backup peer-related objects and utility functions.

Module Attributes
=================

Attributes:
   DEF_COLLECT_INDICATOR: Name of the default collect indicator file
   DEF_STAGE_INDICATOR: Name of the default stage indicator file

:author: Kenneth J. Pronovici <pronovic@ieee.org>
"""


########################################################################
# Imported modules
########################################################################

# System modules
import os
import logging
import shutil

# Cedar Backup modules
from CedarBackup3.filesystem import FilesystemList
from CedarBackup3.util import resolveCommand, executeCommand, isRunningAsRoot
from CedarBackup3.util import splitCommandLine, encodePath
from CedarBackup3.config import VALID_FAILURE_MODES


########################################################################
# Module-wide constants and variables
########################################################################

logger                  = logging.getLogger("CedarBackup3.log.peer")

DEF_RCP_COMMAND         = [ "/usr/bin/scp", "-B", "-q", "-C" ]
DEF_RSH_COMMAND         = [ "/usr/bin/ssh", ]
DEF_CBACK_COMMAND       = "/usr/bin/cback3"

DEF_COLLECT_INDICATOR   = "cback.collect"
DEF_STAGE_INDICATOR     = "cback.stage"

SU_COMMAND              = [ "su" ]


########################################################################
# LocalPeer class definition
########################################################################

[docs]class LocalPeer(object): ###################### # Class documentation ###################### """ Backup peer representing a local peer in a backup pool. This is a class representing a local (non-network) peer in a backup pool. Local peers are backed up by simple filesystem copy operations. A local peer has associated with it a name (typically, but not necessarily, a hostname) and a collect directory. The public methods other than the constructor are part of a "backup peer" interface shared with the ``RemotePeer`` class. """ ############## # Constructor ##############
[docs] def __init__(self, name, collectDir, ignoreFailureMode=None): """ Initializes a local backup peer. Note that the collect directory must be an absolute path, but does not have to exist when the object is instantiated. We do a lazy validation on this value since we could (potentially) be creating peer objects before an ongoing backup completed. Args: name: Name of the backup peer collectDir: Path to the peer's collect directory ignoreFailureMode: Ignore failure mode for this peer, one of ``VALID_FAILURE_MODES`` Raises: ValueError: If the name is empty ValueError: If collect directory is not an absolute path """ self._name = None self._collectDir = None self._ignoreFailureMode = None self.name = name self.collectDir = collectDir self.ignoreFailureMode = ignoreFailureMode
############# # Properties ############# def _setName(self, value): """ Property target used to set the peer name. The value must be a non-empty string and cannot be ``None``. Raises: ValueError: If the value is an empty string or ``None`` """ if value is None or len(value) < 1: raise ValueError("Peer name must be a non-empty string.") self._name = value def _getName(self): """ Property target used to get the peer name. """ return self._name def _setCollectDir(self, value): """ Property target used to set the collect directory. The value must be an absolute path and cannot be ``None``. It does not have to exist on disk at the time of assignment. Raises: ValueError: If the value is ``None`` or is not an absolute path ValueError: If a path cannot be encoded properly """ if value is None or not os.path.isabs(value): raise ValueError("Collect directory must be an absolute path.") self._collectDir = encodePath(value) def _getCollectDir(self): """ Property target used to get the collect directory. """ return self._collectDir def _setIgnoreFailureMode(self, value): """ Property target used to set the ignoreFailure mode. If not ``None``, the mode must be one of the values in :any:`VALID_FAILURE_MODES`. Raises: ValueError: If the value is not valid """ if value is not None: if value not in VALID_FAILURE_MODES: raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES) self._ignoreFailureMode = value def _getIgnoreFailureMode(self): """ Property target used to get the ignoreFailure mode. """ return self._ignoreFailureMode name = property(_getName, _setName, None, "Name of the peer.") collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.") ################# # Public methods #################
[docs] def stagePeer(self, targetDir, ownership=None, permissions=None): """ Stages data from the peer into the indicated local target directory. The collect and target directories must both already exist before this method is called. If passed in, ownership and permissions will be applied to the files that are copied. *Note:* The caller is responsible for checking that the indicator exists, if they care. This function only stages the files within the directory. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. Args: targetDir: Target directory to write data into ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` Returns: Number of files copied from the source directory to the target directory Raises: ValueError: If collect directory is not a directory or does not exist ValueError: If target directory is not a directory, does not exist or is not absolute ValueError: If a path cannot be encoded properly IOError: If there were no files to stage (i.e. the directory was empty) IOError: If there is an IO error copying a file OSError: If there is an OS error copying or changing permissions on a file """ targetDir = encodePath(targetDir) if not os.path.isabs(targetDir): logger.debug("Target directory [%s] not an absolute path.", targetDir) raise ValueError("Target directory must be an absolute path.") if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir) raise ValueError("Collect directory is not a directory or does not exist on disk.") if not os.path.exists(targetDir) or not os.path.isdir(targetDir): logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir) raise ValueError("Target directory is not a directory or does not exist on disk.") count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions) if count == 0: raise IOError("Did not copy any files from local peer.") return count
[docs] def checkCollectIndicator(self, collectIndicator=None): """ Checks the collect indicator in the peer's staging directory. When a peer has completed collecting its backup files, it will write an empty indicator file into its collect directory. This method checks to see whether that indicator has been written. We're "stupid" here - if the collect directory doesn't exist, you'll naturally get back ``False``. If you need to, you can override the name of the collect indicator file by passing in a different name. Args: collectIndicator: Name of the collect indicator file to check Returns: Boolean true/false depending on whether the indicator exists Raises: ValueError: If a path cannot be encoded properly """ collectIndicator = encodePath(collectIndicator) if collectIndicator is None: return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)) else: return os.path.exists(os.path.join(self.collectDir, collectIndicator))
[docs] def writeStageIndicator(self, stageIndicator=None, ownership=None, permissions=None): """ Writes the stage indicator in the peer's staging directory. When the master has completed collecting its backup files, it will write an empty indicator file into the peer's collect directory. The presence of this file implies that the staging process is complete. If you need to, you can override the name of the stage indicator file by passing in a different name. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. Args: stageIndicator: Name of the indicator file to write ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` Raises: ValueError: If collect directory is not a directory or does not exist ValueError: If a path cannot be encoded properly IOError: If there is an IO error creating the file OSError: If there is an OS error creating or changing permissions on the file """ stageIndicator = encodePath(stageIndicator) if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir) raise ValueError("Collect directory is not a directory or does not exist on disk.") if stageIndicator is None: fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) else: fileName = os.path.join(self.collectDir, stageIndicator) LocalPeer._copyLocalFile(None, fileName, ownership, permissions) # None for sourceFile results in an empty target
################## # Private methods ################## @staticmethod def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None): """ Copies files from the source directory to the target directory. This function is not recursive. Only the files in the directory will be copied. Ownership and permissions will be left at their default values if new values are not specified. The source and target directories are allowed to be soft links to a directory, but besides that soft links are ignored. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. Args: sourceDir: Source directory targetDir: Target directory ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` Returns: Number of files copied from the source directory to the target directory Raises: ValueError: If source or target is not a directory or does not exist ValueError: If a path cannot be encoded properly IOError: If there is an IO error copying the files OSError: If there is an OS error copying or changing permissions on a files """ filesCopied = 0 sourceDir = encodePath(sourceDir) targetDir = encodePath(targetDir) for fileName in os.listdir(sourceDir): sourceFile = os.path.join(sourceDir, fileName) targetFile = os.path.join(targetDir, fileName) LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions) filesCopied += 1 return filesCopied @staticmethod def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True): """ Copies a source file to a target file. If the source file is ``None`` then the target file will be created or overwritten as an empty file. If the target file is ``None``, this method is a no-op. Attempting to copy a soft link or a directory will result in an exception. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. *Note:* We will not overwrite a target file that exists when this method is invoked. If the target already exists, we'll raise an exception. Args: sourceFile: Source file to copy targetFile: Target file to create ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` overwrite: Indicates whether it's OK to overwrite the target file Raises: ValueError: If the passed-in source file is not a regular file ValueError: If a path cannot be encoded properly IOError: If the target file already exists IOError: If there is an IO error copying the file OSError: If there is an OS error copying or changing permissions on a file """ targetFile = encodePath(targetFile) sourceFile = encodePath(sourceFile) if targetFile is None: return if not overwrite: if os.path.exists(targetFile): raise IOError("Target file [%s] already exists." % targetFile) if sourceFile is None: with open(targetFile, "w") as f: f.write("") else: if os.path.isfile(sourceFile) and not os.path.islink(sourceFile): shutil.copy(sourceFile, targetFile) else: logger.debug("Source [%s] is not a regular file.", sourceFile) raise ValueError("Source is not a regular file.") if ownership is not None: os.chown(targetFile, ownership[0], ownership[1]) if permissions is not None: os.chmod(targetFile, permissions)
######################################################################## # RemotePeer class definition ########################################################################
[docs]class RemotePeer(object): ###################### # Class documentation ###################### """ Backup peer representing a remote peer in a backup pool. This is a class representing a remote (networked) peer in a backup pool. Remote peers are backed up using an rcp-compatible copy command. A remote peer has associated with it a name (which must be a valid hostname), a collect directory, a working directory and a copy method (an rcp-compatible command). You can also set an optional local user value. This username will be used as the local user for any remote copies that are required. It can only be used if the root user is executing the backup. The root user will ``su`` to the local user and execute the remote copies as that user. The copy method is associated with the peer and not with the actual request to copy, because we can envision that each remote host might have a different connect method. The public methods other than the constructor are part of a "backup peer" interface shared with the ``LocalPeer`` class. """ ############## # Constructor ##############
[docs] def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None, rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None, ignoreFailureMode=None): """ Initializes a remote backup peer. *Note:* If provided, each command will eventually be parsed into a list of strings suitable for passing to ``util.executeCommand`` in order to avoid security holes related to shell interpolation. This parsing will be done by the :any:`util.splitCommandLine` function. See the documentation for that function for some important notes about its limitations. Args: name: Name of the backup peer, a valid DNS name collectDir: Path to the peer's collect directory, absolute path workingDir: Working directory that can be used to create temporary files, etc, an absolute path remoteUser: Name of the Cedar Backup user on the remote peer localUser: Name of the Cedar Backup user on the current host rcpCommand: An rcp-compatible copy command to use for copying files from the peer rshCommand: An rsh-compatible copy command to use for remote shells to the peer cbackCommand: A chack-compatible command to use for executing managed actions ignoreFailureMode: Ignore failure mode for this peer, one of ``VALID_FAILURE_MODES`` Raises: ValueError: If collect directory is not an absolute path """ self._name = None self._collectDir = None self._workingDir = None self._remoteUser = None self._localUser = None self._rcpCommand = None self._rcpCommandList = None self._rshCommand = None self._rshCommandList = None self._cbackCommand = None self._ignoreFailureMode = None self.name = name self.collectDir = collectDir self.workingDir = workingDir self.remoteUser = remoteUser self.localUser = localUser self.rcpCommand = rcpCommand self.rshCommand = rshCommand self.cbackCommand = cbackCommand self.ignoreFailureMode = ignoreFailureMode
############# # Properties ############# def _setName(self, value): """ Property target used to set the peer name. The value must be a non-empty string and cannot be ``None``. Raises: ValueError: If the value is an empty string or ``None`` """ if value is None or len(value) < 1: raise ValueError("Peer name must be a non-empty string.") self._name = value def _getName(self): """ Property target used to get the peer name. """ return self._name def _setCollectDir(self, value): """ Property target used to set the collect directory. The value must be an absolute path and cannot be ``None``. It does not have to exist on disk at the time of assignment. Raises: ValueError: If the value is ``None`` or is not an absolute path ValueError: If the value cannot be encoded properly """ if value is not None: if not os.path.isabs(value): raise ValueError("Collect directory must be an absolute path.") self._collectDir = encodePath(value) def _getCollectDir(self): """ Property target used to get the collect directory. """ return self._collectDir def _setWorkingDir(self, value): """ Property target used to set the working directory. The value must be an absolute path and cannot be ``None``. Raises: ValueError: If the value is ``None`` or is not an absolute path ValueError: If the value cannot be encoded properly """ if value is not None: if not os.path.isabs(value): raise ValueError("Working directory must be an absolute path.") self._workingDir = encodePath(value) def _getWorkingDir(self): """ Property target used to get the working directory. """ return self._workingDir def _setRemoteUser(self, value): """ Property target used to set the remote user. The value must be a non-empty string and cannot be ``None``. Raises: ValueError: If the value is an empty string or ``None`` """ if value is None or len(value) < 1: raise ValueError("Peer remote user must be a non-empty string.") self._remoteUser = value def _getRemoteUser(self): """ Property target used to get the remote user. """ return self._remoteUser def _setLocalUser(self, value): """ Property target used to set the local user. The value must be a non-empty string if it is not ``None``. Raises: ValueError: If the value is an empty string """ if value is not None: if len(value) < 1: raise ValueError("Peer local user must be a non-empty string.") self._localUser = value def _getLocalUser(self): """ Property target used to get the local user. """ return self._localUser def _setRcpCommand(self, value): """ Property target to set the rcp command. The value must be a non-empty string or ``None``. Its value is stored in the two forms: "raw" as provided by the client, and "parsed" into a list suitable for being passed to :any:`util.executeCommand` via :any:`util.splitCommandLine`. However, all the caller will ever see via the property is the actual value they set (which includes seeing ``None``, even if we translate that internally to ``DEF_RCP_COMMAND``). Internally, we should always use ``self._rcpCommandList`` if we want the actual command list. Raises: ValueError: If the value is an empty string """ if value is None: self._rcpCommand = None self._rcpCommandList = DEF_RCP_COMMAND else: if len(value) >= 1: self._rcpCommand = value self._rcpCommandList = splitCommandLine(self._rcpCommand) else: raise ValueError("The rcp command must be a non-empty string.") def _getRcpCommand(self): """ Property target used to get the rcp command. """ return self._rcpCommand def _setRshCommand(self, value): """ Property target to set the rsh command. The value must be a non-empty string or ``None``. Its value is stored in the two forms: "raw" as provided by the client, and "parsed" into a list suitable for being passed to :any:`util.executeCommand` via :any:`util.splitCommandLine`. However, all the caller will ever see via the property is the actual value they set (which includes seeing ``None``, even if we translate that internally to ``DEF_RSH_COMMAND``). Internally, we should always use ``self._rshCommandList`` if we want the actual command list. Raises: ValueError: If the value is an empty string """ if value is None: self._rshCommand = None self._rshCommandList = DEF_RSH_COMMAND else: if len(value) >= 1: self._rshCommand = value self._rshCommandList = splitCommandLine(self._rshCommand) else: raise ValueError("The rsh command must be a non-empty string.") def _getRshCommand(self): """ Property target used to get the rsh command. """ return self._rshCommand def _setCbackCommand(self, value): """ Property target to set the cback command. The value must be a non-empty string or ``None``. Unlike the other command, this value is only stored in the "raw" form provided by the client. Raises: ValueError: If the value is an empty string """ if value is None: self._cbackCommand = None else: if len(value) >= 1: self._cbackCommand = value else: raise ValueError("The cback command must be a non-empty string.") def _getCbackCommand(self): """ Property target used to get the cback command. """ return self._cbackCommand def _setIgnoreFailureMode(self, value): """ Property target used to set the ignoreFailure mode. If not ``None``, the mode must be one of the values in :any:`VALID_FAILURE_MODES`. Raises: ValueError: If the value is not valid """ if value is not None: if value not in VALID_FAILURE_MODES: raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES) self._ignoreFailureMode = value def _getIgnoreFailureMode(self): """ Property target used to get the ignoreFailure mode. """ return self._ignoreFailureMode name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).") collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).") remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.") localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.") rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.") rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.") cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.") ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.") ################# # Public methods #################
[docs] def stagePeer(self, targetDir, ownership=None, permissions=None): """ Stages data from the peer into the indicated local target directory. The target directory must already exist before this method is called. If passed in, ownership and permissions will be applied to the files that are copied. *Note:* The returned count of copied files might be inaccurate if some of the copied files already existed in the staging directory prior to the copy taking place. We don't clear the staging directory first, because some extension might also be using it. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. *Note:* Unlike the local peer version of this method, an I/O error might or might not be raised if the directory is empty. Since we're using a remote copy method, we just don't have the fine-grained control over our exceptions that's available when we can look directly at the filesystem, and we can't control whether the remote copy method thinks an empty directory is an error. Args: targetDir: Target directory to write data into ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` Returns: Number of files copied from the source directory to the target directory Raises: ValueError: If target directory is not a directory, does not exist or is not absolute ValueError: If a path cannot be encoded properly IOError: If there were no files to stage (i.e. the directory was empty) IOError: If there is an IO error copying a file OSError: If there is an OS error copying or changing permissions on a file """ targetDir = encodePath(targetDir) if not os.path.isabs(targetDir): logger.debug("Target directory [%s] not an absolute path.", targetDir) raise ValueError("Target directory must be an absolute path.") if not os.path.exists(targetDir) or not os.path.isdir(targetDir): logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir) raise ValueError("Target directory is not a directory or does not exist on disk.") count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name, self._rcpCommand, self._rcpCommandList, self.collectDir, targetDir, ownership, permissions) if count == 0: raise IOError("Did not copy any files from local peer.") return count
[docs] def checkCollectIndicator(self, collectIndicator=None): """ Checks the collect indicator in the peer's staging directory. When a peer has completed collecting its backup files, it will write an empty indicator file into its collect directory. This method checks to see whether that indicator has been written. If the remote copy command fails, we return ``False`` as if the file weren't there. If you need to, you can override the name of the collect indicator file by passing in a different name. *Note:* Apparently, we can't count on all rcp-compatible implementations to return sensible errors for some error conditions. As an example, the ``scp`` command in Debian 'woody' returns a zero (normal) status even when it can't find a host or if the login or path is invalid. Because of this, the implementation of this method is rather convoluted. Args: collectIndicator: Name of the collect indicator file to check Returns: Boolean true/false depending on whether the indicator exists Raises: ValueError: If a path cannot be encoded properly """ try: if collectIndicator is None: sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR) targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR) else: collectIndicator = encodePath(collectIndicator) sourceFile = os.path.join(self.collectDir, collectIndicator) targetFile = os.path.join(self.workingDir, collectIndicator) logger.debug("Fetch remote [%s] into [%s].", sourceFile, targetFile) if os.path.exists(targetFile): try: os.remove(targetFile) except: raise Exception("Error: collect indicator [%s] already exists!" % targetFile) try: RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name, self._rcpCommand, self._rcpCommandList, sourceFile, targetFile, overwrite=False) if os.path.exists(targetFile): return True else: return False except Exception as e: logger.info("Failed looking for collect indicator: %s", e) return False finally: if os.path.exists(targetFile): try: os.remove(targetFile) except: pass
[docs] def writeStageIndicator(self, stageIndicator=None): """ Writes the stage indicator in the peer's staging directory. When the master has completed collecting its backup files, it will write an empty indicator file into the peer's collect directory. The presence of this file implies that the staging process is complete. If you need to, you can override the name of the stage indicator file by passing in a different name. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. Args: stageIndicator: Name of the indicator file to write Raises: ValueError: If a path cannot be encoded properly IOError: If there is an IO error creating the file OSError: If there is an OS error creating or changing permissions on the file """ stageIndicator = encodePath(stageIndicator) if stageIndicator is None: sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) else: sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) targetFile = os.path.join(self.collectDir, stageIndicator) try: if not os.path.exists(sourceFile): with open(sourceFile, "w") as f: f.write("") RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name, self._rcpCommand, self._rcpCommandList, sourceFile, targetFile) finally: if os.path.exists(sourceFile): try: os.remove(sourceFile) except: pass
[docs] def executeRemoteCommand(self, command): """ Executes a command on the peer via remote shell. Args: command: Command to execute Raises: IOError: If there is an error executing the command on the remote peer """ RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser, self.name, self._rshCommand, self._rshCommandList, command)
[docs] def executeManagedAction(self, action, fullBackup): """ Executes a managed action on this peer. Args: action: Name of the action to execute fullBackup: Whether a full backup should be executed Raises: IOError: If there is an error executing the action on the remote peer """ try: command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup) self.executeRemoteCommand(command) except IOError as e: logger.info(e) raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
################## # Private methods ################## @staticmethod def _getDirContents(path): """ Returns the contents of a directory in terms of a Set. The directory's contents are read as a :any:`FilesystemList` containing only files, and then the list is converted into a set object for later use. Args: path: Directory path to get contents for Returns: Set of files in the directory Raises: ValueError: If path is not a directory or does not exist """ contents = FilesystemList() contents.excludeDirs = True contents.excludeLinks = True contents.addDirContents(path) return set(contents) @staticmethod def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, sourceDir, targetDir, ownership=None, permissions=None): """ Copies files from the source directory to the target directory. This function is not recursive. Only the files in the directory will be copied. Ownership and permissions will be left at their default values if new values are not specified. Behavior when copying soft links from the collect directory is dependent on the behavior of the specified rcp command. *Note:* The returned count of copied files might be inaccurate if some of the copied files already existed in the staging directory prior to the copy taking place. We don't clear the staging directory first, because some extension might also be using it. *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. *Note:* We don't have a good way of knowing exactly what files we copied down from the remote peer, unless we want to parse the output of the rcp command (ugh). We could change permissions on everything in the target directory, but that's kind of ugly too. Instead, we use Python's set functionality to figure out what files were added while we executed the rcp command. This isn't perfect - for instance, it's not correct if someone else is messing with the directory at the same time we're doing the remote copy - but it's about as good as we're going to get. *Note:* Apparently, we can't count on all rcp-compatible implementations to return sensible errors for some error conditions. As an example, the ``scp`` command in Debian 'woody' returns a zero (normal) status even when it can't find a host or if the login or path is invalid. We try to work around this by issuing ``IOError`` if we don't copy any files from the remote host. Args: remoteUser: Name of the Cedar Backup user on the remote peer localUser: Name of the Cedar Backup user on the current host remoteHost: Hostname of the remote peer rcpCommand: An rcp-compatible copy command to use for copying files from the peer rcpCommandList: An rcp-compatible copy command to use for copying files, as for :any:`util.executeCommand` sourceDir: Source directory targetDir: Target directory ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` Returns: Number of files copied from the source directory to the target directory Raises: ValueError: If source or target is not a directory or does not exist IOError: If there is an IO error copying the files """ beforeSet = RemotePeer._getDirContents(targetDir) if localUser is not None: try: if not isRunningAsRoot(): raise IOError("Only root can remote copy as another user.") except AttributeError: pass actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir) command = resolveCommand(SU_COMMAND) result = executeCommand(command, [localUser, "-c", actualCommand])[0] if result != 0: raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser)) else: copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir) command = resolveCommand(rcpCommandList) result = executeCommand(command, [copySource, targetDir])[0] if result != 0: raise IOError("Error (%d) copying files from remote host." % result) afterSet = RemotePeer._getDirContents(targetDir) if len(afterSet) == 0: raise IOError("Did not copy any files from remote peer.") differenceSet = afterSet.difference(beforeSet) # files we added as part of copy if len(differenceSet) == 0: raise IOError("Apparently did not copy any new files from remote peer.") for targetFile in differenceSet: if ownership is not None: os.chown(targetFile, ownership[0], ownership[1]) if permissions is not None: os.chmod(targetFile, permissions) return len(differenceSet) @staticmethod def _copyRemoteFile(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, sourceFile, targetFile, ownership=None, permissions=None, overwrite=True): """ Copies a remote source file to a target file. *Note:* Internally, we have to go through and escape any spaces in the source path with double-backslash, otherwise things get screwed up. It doesn't seem to be required in the target path. I hope this is portable to various different rcp methods, but I guess it might not be (all I have to test with is OpenSSH). *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. *Note:* We will not overwrite a target file that exists when this method is invoked. If the target already exists, we'll raise an exception. *Note:* Apparently, we can't count on all rcp-compatible implementations to return sensible errors for some error conditions. As an example, the ``scp`` command in Debian 'woody' returns a zero (normal) status even when it can't find a host or if the login or path is invalid. We try to work around this by issuing ``IOError`` the target file does not exist when we're done. Args: remoteUser: Name of the Cedar Backup user on the remote peer remoteHost: Hostname of the remote peer localUser: Name of the Cedar Backup user on the current host rcpCommand: An rcp-compatible copy command to use for copying files from the peer rcpCommandList: An rcp-compatible copy command to use for copying files, as for :any:`util.executeCommand` sourceFile: Source file to copy targetFile: Target file to create ownership: Owner and group that files should have, tuple of numeric ``(uid, gid)`` permissions: Unix permissions mode that the staged files should have, in octal like ``0640`` overwrite: Indicates whether it's OK to overwrite the target file Raises: IOError: If the target file already exists IOError: If there is an IO error copying the file OSError: If there is an OS error changing permissions on the file """ if not overwrite: if os.path.exists(targetFile): raise IOError("Target file [%s] already exists." % targetFile) if localUser is not None: try: if not isRunningAsRoot(): raise IOError("Only root can remote copy as another user.") except AttributeError: pass actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile) command = resolveCommand(SU_COMMAND) result = executeCommand(command, [localUser, "-c", actualCommand])[0] if result != 0: raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser)) else: copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ ")) command = resolveCommand(rcpCommandList) result = executeCommand(command, [copySource, targetFile])[0] if result != 0: raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile)) if not os.path.exists(targetFile): raise IOError("Apparently unable to copy file from remote host.") if ownership is not None: os.chown(targetFile, ownership[0], ownership[1]) if permissions is not None: os.chmod(targetFile, permissions) @staticmethod def _pushLocalFile(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, sourceFile, targetFile, overwrite=True): """ Copies a local source file to a remote host. *Note:* We will not overwrite a target file that exists when this method is invoked. If the target already exists, we'll raise an exception. *Note:* Internally, we have to go through and escape any spaces in the source and target paths with double-backslash, otherwise things get screwed up. I hope this is portable to various different rcp methods, but I guess it might not be (all I have to test with is OpenSSH). *Note:* If you have user/group as strings, call the :any:`util.getUidGid` function to get the associated uid/gid as an ownership tuple. Args: remoteUser: Name of the Cedar Backup user on the remote peer localUser: Name of the Cedar Backup user on the current host remoteHost: Hostname of the remote peer rcpCommand: An rcp-compatible copy command to use for copying files from the peer rcpCommandList: An rcp-compatible copy command to use for copying files, as for :any:`util.executeCommand` sourceFile: Source file to copy targetFile: Target file to create overwrite: Indicates whether it's OK to overwrite the target file Raises: IOError: If there is an IO error copying the file OSError: If there is an OS error changing permissions on the file """ if not overwrite: if os.path.exists(targetFile): raise IOError("Target file [%s] already exists." % targetFile) if localUser is not None: try: if not isRunningAsRoot(): raise IOError("Only root can remote copy as another user.") except AttributeError: pass actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile) command = resolveCommand(SU_COMMAND) result = executeCommand(command, [localUser, "-c", actualCommand])[0] if result != 0: raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser)) else: copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ ")) command = resolveCommand(rcpCommandList) result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0] if result != 0: raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile)) @staticmethod def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand): """ Executes a command on the peer via remote shell. Args: remoteUser: Name of the Cedar Backup user on the remote peer localUser: Name of the Cedar Backup user on the current host remoteHost: Hostname of the remote peer rshCommand: An rsh-compatible copy command to use for remote shells to the peer rshCommandList: An rsh-compatible copy command to use for remote shells to the peer, as for :any:`util.executeCommand` remoteCommand: The command to be executed on the remote host, with no special shell characters Raises: IOError: If there is an error executing the remote command """ actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand) if localUser is not None: try: if not isRunningAsRoot(): raise IOError("Only root can remote shell as another user.") except AttributeError: pass command = resolveCommand(SU_COMMAND) result = executeCommand(command, [localUser, "-c", actualCommand])[0] if result != 0: raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand)) else: command = resolveCommand(rshCommandList) result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0] if result != 0: raise IOError("Command failed [%s]" % (actualCommand)) @staticmethod def _buildCbackCommand(cbackCommand, action, fullBackup): """ Builds a Cedar Backup command line for the named action. *Note:* If the cback command is None, then DEF_CBACK_COMMAND is used. Args: cbackCommand: cback command to execute, including required options action: Name of the action to execute fullBackup: Whether a full backup should be executed Returns: String suitable for passing to :any:`_executeRemoteCommand` as remoteCommand Raises: ValueError: If action is None """ if action is None: raise ValueError("Action cannot be None.") if cbackCommand is None: cbackCommand = DEF_CBACK_COMMAND if fullBackup: return "%s --full %s" % (cbackCommand, action) else: return "%s %s" % (cbackCommand, action)