1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Spans staged data among multiple discs
41
42 This is the Cedar Backup span tool. It is intended for use by people who stage
43 more data than can fit on a single disc. It allows a user to split staged data
44 among more than one disc. It can't be an extension because it requires user
45 input when switching media.
46
47 Most configuration is taken from the Cedar Backup configuration file,
48 specifically the store section. A few pieces of configuration are taken
49 directly from the user.
50
51 @author: Kenneth J. Pronovici <pronovic@ieee.org>
52 """
53
54
55
56
57
58
59 import sys
60 import os
61 import logging
62 import tempfile
63
64
65 from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT
66 from CedarBackup2.util import displayBytes, convertSize, mount, unmount
67 from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES
68 from CedarBackup2.config import Config
69 from CedarBackup2.filesystem import BackupFileList, compareDigestMaps, normalizeDir
70 from CedarBackup2.cli import Options, setupLogging, setupPathResolver
71 from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE
72 from CedarBackup2.actions.constants import STORE_INDICATOR
73 from CedarBackup2.actions.util import createWriter
74 from CedarBackup2.actions.store import writeIndicatorFile
75 from CedarBackup2.actions.util import findDailyDirs
76 from CedarBackup2.util import Diagnostics
77
78
79
80
81
82
83 logger = logging.getLogger("CedarBackup2.log.tools.span")
84
85
86
87
88
89
91
92 """
93 Tool-specific command-line options.
94
95 Most of the cback command-line options are exactly what we need here --
96 logfile path, permissions, verbosity, etc. However, we need to make a few
97 tweaks since we don't accept any actions.
98
99 Also, a few extra command line options that we accept are really ignored
100 underneath. I just don't care about that for a tool like this.
101 """
102
104 """
105 Validates command-line options represented by the object.
106 There are no validations here, because we don't use any actions.
107 @raise ValueError: If one of the validations fails.
108 """
109 pass
110
111
112
113
114
115
116
117
118
119
121 """
122 Implements the command-line interface for the C{cback-span} script.
123
124 Essentially, this is the "main routine" for the cback-span script. It does
125 all of the argument processing for the script, and then also implements the
126 tool functionality.
127
128 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not
129 easy to refactor this code to make it reusable and also readable, so I've
130 decided to just live with the duplication.
131
132 A different error code is returned for each type of failure:
133
134 - C{1}: The Python interpreter version is < 2.5
135 - C{2}: Error processing command-line arguments
136 - C{3}: Error configuring logging
137 - C{4}: Error parsing indicated configuration file
138 - C{5}: Backup was interrupted with a CTRL-C or similar
139 - C{6}: Error executing other parts of the script
140
141 @note: This script uses print rather than logging to the INFO level, because
142 it is interactive. Underlying Cedar Backup functionality uses the logging
143 mechanism exclusively.
144
145 @return: Error code as described above.
146 """
147 try:
148 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 5]:
149 sys.stderr.write("Python version 2.5 or greater required.\n")
150 return 1
151 except:
152
153 sys.stderr.write("Python version 2.5 or greater required.\n")
154 return 1
155
156 try:
157 options = SpanOptions(argumentList=sys.argv[1:])
158 except Exception, e:
159 _usage()
160 sys.stderr.write(" *** Error: %s\n" % e)
161 return 2
162
163 if options.help:
164 _usage()
165 return 0
166 if options.version:
167 _version()
168 return 0
169 if options.diagnostics:
170 _diagnostics()
171 return 0
172
173 try:
174 logfile = setupLogging(options)
175 except Exception, e:
176 sys.stderr.write("Error setting up logging: %s\n" % e)
177 return 3
178
179 logger.info("Cedar Backup 'span' utility run started.")
180 logger.info("Options were [%s]" % options)
181 logger.info("Logfile is [%s]" % logfile)
182
183 if options.config is None:
184 logger.debug("Using default configuration file.")
185 configPath = DEFAULT_CONFIG
186 else:
187 logger.debug("Using user-supplied configuration file.")
188 configPath = options.config
189
190 try:
191 logger.info("Configuration path is [%s]" % configPath)
192 config = Config(xmlPath=configPath)
193 setupPathResolver(config)
194 except Exception, e:
195 logger.error("Error reading or handling configuration: %s" % e)
196 logger.info("Cedar Backup 'span' utility run completed with status 4.")
197 return 4
198
199 if options.stacktrace:
200 _executeAction(options, config)
201 else:
202 try:
203 _executeAction(options, config)
204 except KeyboardInterrupt:
205 logger.error("Backup interrupted.")
206 logger.info("Cedar Backup 'span' utility run completed with status 5.")
207 return 5
208 except Exception, e:
209 logger.error("Error executing backup: %s" % e)
210 logger.info("Cedar Backup 'span' utility run completed with status 6.")
211 return 6
212
213 logger.info("Cedar Backup 'span' utility run completed with status 0.")
214 return 0
215
216
217
218
219
220
221
222
223
224
226 """
227 Prints usage information for the cback script.
228 @param fd: File descriptor used to print information.
229 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
230 """
231 fd.write("\n")
232 fd.write(" Usage: cback-span [switches]\n")
233 fd.write("\n")
234 fd.write(" Cedar Backup 'span' tool.\n")
235 fd.write("\n")
236 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n")
237 fd.write(" It is a utility, not an extension, and requires user interaction.\n")
238 fd.write("\n")
239 fd.write(" The following switches are accepted, mostly to set up underlying\n")
240 fd.write(" Cedar Backup functionality:\n")
241 fd.write("\n")
242 fd.write(" -h, --help Display this usage/help listing\n")
243 fd.write(" -V, --version Display version information\n")
244 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n")
245 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG)
246 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE)
247 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1]))
248 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE)
249 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n")
250 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n")
251 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n")
252 fd.write("\n")
253
254
255
256
257
258
260 """
261 Prints version information for the cback script.
262 @param fd: File descriptor used to print information.
263 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
264 """
265 fd.write("\n")
266 fd.write(" Cedar Backup 'span' tool.\n")
267 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE))
268 fd.write("\n")
269 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL))
270 fd.write(" See CREDITS for a list of included code and other contributors.\n")
271 fd.write(" This is free software; there is NO warranty. See the\n")
272 fd.write(" GNU General Public License version 2 for copying conditions.\n")
273 fd.write("\n")
274 fd.write(" Use the --help option for usage information.\n")
275 fd.write("\n")
276
277
278
279
280
281
283 """
284 Prints runtime diagnostics information.
285 @param fd: File descriptor used to print information.
286 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
287 """
288 fd.write("\n")
289 fd.write("Diagnostics:\n")
290 fd.write("\n")
291 Diagnostics().printDiagnostics(fd=fd, prefix=" ")
292 fd.write("\n")
293
294
295
296
297
298
300 """
301 Implements the guts of the cback-span tool.
302
303 @param options: Program command-line options.
304 @type options: SpanOptions object.
305
306 @param config: Program configuration.
307 @type config: Config object.
308
309 @raise Exception: Under many generic error conditions
310 """
311 print ""
312 print "================================================"
313 print " Cedar Backup 'span' tool"
314 print "================================================"
315 print ""
316 print "This the Cedar Backup span tool. It is used to split up staging"
317 print "data when that staging data does not fit onto a single disc."
318 print ""
319 print "This utility operates using Cedar Backup configuration. Configuration"
320 print "specifies which staging directory to look at and which writer device"
321 print "and media type to use."
322 print ""
323 if not _getYesNoAnswer("Continue?", default="Y"):
324 return
325 print "==="
326
327 print ""
328 print "Cedar Backup store configuration looks like this:"
329 print ""
330 print " Source Directory...: %s" % config.store.sourceDir
331 print " Media Type.........: %s" % config.store.mediaType
332 print " Device Type........: %s" % config.store.deviceType
333 print " Device Path........: %s" % config.store.devicePath
334 print " Device SCSI ID.....: %s" % config.store.deviceScsiId
335 print " Drive Speed........: %s" % config.store.driveSpeed
336 print " Check Data Flag....: %s" % config.store.checkData
337 print " No Eject Flag......: %s" % config.store.noEject
338 print ""
339 if not _getYesNoAnswer("Is this OK?", default="Y"):
340 return
341 print "==="
342
343 (writer, mediaCapacity) = _getWriter(config)
344
345 print ""
346 print "Please wait, indexing the source directory (this may take a while)..."
347 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir)
348 print "==="
349
350 print ""
351 print "The following daily staging directories have not yet been written to disc:"
352 print ""
353 for dailyDir in dailyDirs:
354 print " %s" % dailyDir
355
356 totalSize = fileList.totalSize()
357 print ""
358 print "The total size of the data in these directories is %s." % displayBytes(totalSize)
359 print ""
360 if not _getYesNoAnswer("Continue?", default="Y"):
361 return
362 print "==="
363
364 print ""
365 print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity)
366
367 print ""
368 print "Since estimates are not perfect and there is some uncertainly in"
369 print "media capacity calculations, it is good to have a \"cushion\","
370 print "a percentage of capacity to set aside. The cushion reduces the"
371 print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining."
372 print ""
373 cushion = _getFloat("What cushion percentage?", default=4.5)
374 print "==="
375
376 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity
377 minimumDiscs = (totalSize/realCapacity) + 1
378 print ""
379 print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity))
380 print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize))
381 print ""
382 if not _getYesNoAnswer("Continue?", default="Y"):
383 return
384 print "==="
385
386 happy = False
387 while not happy:
388 print ""
389 print "Which algorithm do you want to use to span your data across"
390 print "multiple discs?"
391 print ""
392 print "The following algorithms are available:"
393 print ""
394 print " first....: The \"first-fit\" algorithm"
395 print " best.....: The \"best-fit\" algorithm"
396 print " worst....: The \"worst-fit\" algorithm"
397 print " alternate: The \"alternate-fit\" algorithm"
398 print ""
399 print "If you don't like the results you will have a chance to try a"
400 print "different one later."
401 print ""
402 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate", ])
403 print "==="
404
405 print ""
406 print "Please wait, generating file lists (this may take a while)..."
407 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm)
408 print "==="
409
410 print ""
411 print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm
412 print "into %d discs." % len(spanSet)
413 print ""
414 counter = 0
415 for item in spanSet:
416 counter += 1
417 print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList),
418 displayBytes(item.size), item.utilization)
419 print ""
420 if _getYesNoAnswer("Accept this solution?", default="Y"):
421 happy = True
422 print "==="
423
424 counter = 0
425 for spanItem in spanSet:
426 counter += 1
427 if counter == 1:
428 print ""
429 _getReturn("Please place the first disc in your backup device.\nPress return when ready.")
430 print "==="
431 else:
432 print ""
433 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
434 print "==="
435 _writeDisc(config, writer, spanItem)
436
437 _writeStoreIndicator(config, dailyDirs)
438
439 print ""
440 print "Completed writing all discs."
441
442
443
444
445
446
448 """
449 Returns a list of all daily staging directories that have not yet been
450 stored.
451
452 The store indicator file C{cback.store} will be written to a daily staging
453 directory once that directory is written to disc. So, this function looks
454 at each daily staging directory within the configured staging directory, and
455 returns a list of those which do not contain the indicator file.
456
457 Returned is a tuple containing two items: a list of daily staging
458 directories, and a BackupFileList containing all files among those staging
459 directories.
460
461 @param stagingDir: Configured staging directory
462
463 @return: Tuple (staging dirs, backup file list)
464 """
465 results = findDailyDirs(stagingDir, STORE_INDICATOR)
466 fileList = BackupFileList()
467 for item in results:
468 fileList.addDirContents(item)
469 return (results, fileList)
470
471
472
473
474
475
487
488
489
490
491
492
503
504
505
506
507
508
522
524 """
525 Initialize an ISO image for a span item.
526 @param config: Cedar Backup configuration
527 @param writer: Writer to use
528 @param spanItem: Span item to write
529 """
530 complete = False
531 while not complete:
532 try:
533 print "Initializing image..."
534 writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir)
535 for path in spanItem.fileList:
536 graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1))
537 writer.addImageEntry(path, graftPoint)
538 complete = True
539 except KeyboardInterrupt, e:
540 raise e
541 except Exception, e:
542 logger.error("Failed to initialize image: %s" % e)
543 if not _getYesNoAnswer("Retry initialization step?", default="Y"):
544 raise e
545 print "Ok, attempting retry."
546 print "==="
547 print "Completed initializing image."
548
550 """
551 Writes a ISO image for a span item.
552 @param config: Cedar Backup configuration
553 @param writer: Writer to use
554 """
555 complete = False
556 while not complete:
557 try:
558 print "Writing image to disc..."
559 writer.writeImage()
560 complete = True
561 except KeyboardInterrupt, e:
562 raise e
563 except Exception, e:
564 logger.error("Failed to write image: %s" % e)
565 if not _getYesNoAnswer("Retry this step?", default="Y"):
566 raise e
567 print "Ok, attempting retry."
568 _getReturn("Please replace media if needed.\nPress return when ready.")
569 print "==="
570 print "Completed writing image."
571
573 """
574 Run a consistency check on an ISO image for a span item.
575 @param config: Cedar Backup configuration
576 @param writer: Writer to use
577 @param spanItem: Span item to write
578 """
579 if config.store.checkData:
580 complete = False
581 while not complete:
582 try:
583 print "Running consistency check..."
584 _consistencyCheck(config, spanItem.fileList)
585 complete = True
586 except KeyboardInterrupt, e:
587 raise e
588 except Exception, e:
589 logger.error("Consistency check failed: %s" % e)
590 if not _getYesNoAnswer("Retry the consistency check?", default="Y"):
591 raise e
592 if _getYesNoAnswer("Rewrite the disc first?", default="N"):
593 print "Ok, attempting retry."
594 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
595 print "==="
596 _discWriteImage(config, writer)
597 else:
598 print "Ok, attempting retry."
599 print "==="
600 print "Completed consistency check."
601
602
603
604
605
606
608 """
609 Runs a consistency check against media in the backup device.
610
611 The function mounts the device at a temporary mount point in the working
612 directory, and then compares the passed-in file list's digest map with the
613 one generated from the disc. The two lists should be identical.
614
615 If no exceptions are thrown, there were no problems with the consistency
616 check.
617
618 @warning: The implementation of this function is very UNIX-specific.
619
620 @param config: Config object.
621 @param fileList: BackupFileList whose contents to check against
622
623 @raise ValueError: If the check fails
624 @raise IOError: If there is a problem working with the media.
625 """
626 logger.debug("Running consistency check.")
627 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
628 try:
629 mount(config.store.devicePath, mountPoint, "iso9660")
630 discList = BackupFileList()
631 discList.addDirContents(mountPoint)
632 sourceList = BackupFileList()
633 sourceList.extend(fileList)
634 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint))
635 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir))
636 compareDigestMaps(sourceListDigest, discListDigest, verbose=True)
637 logger.info("Consistency check completed. No problems found.")
638 finally:
639 unmount(mountPoint, True, 5, 1)
640
641
642
643
644
645
647 """
648 Get a yes/no answer from the user.
649 The default will be placed at the end of the prompt.
650 A "Y" or "y" is considered yes, anything else no.
651 A blank (empty) response results in the default.
652 @param prompt: Prompt to show.
653 @param default: Default to set if the result is blank
654 @return: Boolean true/false corresponding to Y/N
655 """
656 if default == "Y":
657 prompt = "%s [Y/n]: " % prompt
658 else:
659 prompt = "%s [y/N]: " % prompt
660 answer = raw_input(prompt)
661 if answer in [ None, "", ]:
662 answer = default
663 if answer[0] in [ "Y", "y", ]:
664 return True
665 else:
666 return False
667
669 """
670 Get a particular choice from the user.
671 The default will be placed at the end of the prompt.
672 The function loops until getting a valid choice.
673 A blank (empty) response results in the default.
674 @param prompt: Prompt to show.
675 @param default: Default to set if the result is None or blank.
676 @param validChoices: List of valid choices (strings)
677 @return: Valid choice from user.
678 """
679 prompt = "%s [%s]: " % (prompt, default)
680 answer = raw_input(prompt)
681 if answer in [ None, "", ]:
682 answer = default
683 while answer not in validChoices:
684 print "Choice must be one of %s" % validChoices
685 answer = raw_input(prompt)
686 return answer
687
689 """
690 Get a floating point number from the user.
691 The default will be placed at the end of the prompt.
692 The function loops until getting a valid floating point number.
693 A blank (empty) response results in the default.
694 @param prompt: Prompt to show.
695 @param default: Default to set if the result is None or blank.
696 @return: Floating point number from user
697 """
698 prompt = "%s [%.2f]: " % (prompt, default)
699 while True:
700 answer = raw_input(prompt)
701 if answer in [ None, "" ]:
702 return default
703 else:
704 try:
705 return float(answer)
706 except ValueError:
707 print "Enter a floating point number."
708
710 """
711 Get a return key from the user.
712 @param prompt: Prompt to show.
713 """
714 raw_input(prompt)
715
716
717
718
719
720
721 if __name__ == "__main__":
722 sys.exit(cli())
723