changeset
Separated repository configuration form backup configuration;
Fri, 26 Jan 2018 19:04:04 +0000
- author
- Tuomo Valkonen <tuomov@iki.fi>
- date
- Fri, 26 Jan 2018 19:04:04 +0000
- changeset 74
- 4f56142e7497
- parent 73
- 4f0e9cf8f230
- child 75
- 2a44b9649212
Separated repository configuration form backup configuration;
gave passphrase management to Repository object;
various fixes.
| backup.py | file | annotate | diff | comparison | revisions | |
| borgend.py | file | annotate | diff | comparison | revisions | |
| config.example.yaml | file | annotate | diff | comparison | revisions | |
| config.py | file | annotate | diff | comparison | revisions | |
| instance.py | file | annotate | diff | comparison | revisions | |
| repository.py | file | annotate | diff | comparison | revisions |
--- a/backup.py Fri Jan 26 10:35:00 2018 +0000 +++ b/backup.py Fri Jan 26 19:04:04 2018 +0000 @@ -5,7 +5,6 @@ import config import logging import time -import keyring import borgend import repository from enum import IntEnum @@ -107,82 +106,47 @@ class Backup(TerminableThread): def __decode_config(self, cfg): - loc0='backup target %d' % self.identifier + loc0='Backup %d' % self.identifier - self._name=config.check_string(cfg, 'name', 'Name', loc0) + self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) - self.logger=logger.getChild(self._name) + logger.debug("Configuring backup '%s'" % self.backup_name) - self.loc='backup target "%s"' % self._name + self.logger=logger.getChild(self.backup_name) + + loc="Backup '%s'" % self.backup_name reponame=config.check_string(cfg, 'repository', - 'Target repository', self.loc) + 'Target repository', loc) - self.repository=repository.get_controller(reponame) + self.repository=repository.find_repository(reponame) + if not self.repository: + raise Exception("Repository '%s' not configured" % reponame) self.archive_prefix=config.check_string(cfg, 'archive_prefix', - 'Archive prefix', self.loc) + 'Archive prefix', loc) self.archive_template=config.check_string(cfg, 'archive_template', - 'Archive template', self.loc) + 'Archive template', loc) self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', - 'Backup interval', self.loc, + 'Backup interval', loc, config.defaults['backup_interval']) self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', - 'Retry interval', self.loc, + 'Retry interval', loc, config.defaults['retry_interval']) - self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', self.loc) - - self.common_parameters=config.check_list_of_dicts(cfg, 'common_parameters', - 'Borg parameters', self.loc, - default=[]) - - self.create_parameters=config.check_list_of_dicts(cfg, 'create_parameters', - 'Create parameters', self.loc, - default=[]) - - self.prune_parameters=config.check_list_of_dicts(cfg, 'prune_parameters', - 'Prune parameters', self.loc, - default=[]) - - self.__keychain_account=config.check_string(cfg, 'keychain_account', - 'Keychain account', self.loc, - default='') - - self.__passphrase=None + self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) - if config.settings['extract_passphrases_at_startup']: - try: - self.extract_passphrase() - except Exception: - pass + self.borg_parameters=config.BorgParameters.from_config(cfg, loc) - def extract_passphrase(self): - acc=self.__keychain_account - if not self.__passphrase: - if acc and acc!='': - self.logger.debug('Requesting passphrase') - try: - pw=keyring.get_password("borg-backup", acc) - except Exception as err: - self.logger.error('Failed to retrieve passphrase') - raise err - else: - self.logger.debug('Received passphrase') - self.__passphrase=pw - else: - self.__passphrase=None - return self.__passphrase def __init__(self, identifier, cfg, scheduler): self.identifier=identifier - self.config=config self.__status_update_callback=None self.scheduler=scheduler - self.logger=None # setup up __decode_config once backup name is known + self.logger=None # setup up in __decode_config once backup name is known self.borg_instance=None self.thread_log=None @@ -197,7 +161,7 @@ self.__decode_config(cfg) - super().__init__(target = self.__main_thread, name = self._name) + super().__init__(target = self.__main_thread, name = self.backup_name) self.daemon=True def is_running(self): @@ -293,17 +257,19 @@ self.logger.debug('Borg result: %s' % str(res)) with self._cond: - if res is None: + if res is None and self.errors.ok(): self.logger.error('No result from borg despite no error in log') - if errors.ok(): - self.errors=self.errors.combine(Errors.ERRORS) + self.errors=Errors.ERRORS - def __do_launch(self, op, archive_or_repository, *args): - passphrase=self.extract_passphrase() + def __do_launch(self, op, archive_or_repository, + common_params, op_params, paths=[]): - inst=BorgInstance(op.operation, archive_or_repository, *args) - inst.launch(passphrase=passphrase) + inst=BorgInstance(op.operation, archive_or_repository, + common_params, op_params, paths) + + # Only the Repository object has access to the passphrase + self.repository.launch_borg_instance(inst) self.logger.debug('Creating listener threads') @@ -328,22 +294,25 @@ t_log.start() t_res.start() + def __launch(self, op): self.logger.debug("Launching '%s'" % str(op.operation)) + params=(config.borg_parameters + +self.repository.borg_parameters + +self.borg_parameters) + if op.operation==Operation.CREATE: - archive="%s::%s%s" % (self.repository.repository_name, + archive="%s::%s%s" % (self.repository.location, self.archive_prefix, self.archive_template) - self.__do_launch(op, archive, - self.common_parameters+self.create_parameters, - self.paths) + self.__do_launch(op, archive, params.common, + params.create, self.paths) elif op.operation==Operation.PRUNE: - self.__do_launch(op, self.repository.repository_name, - ([{'prefix': self.archive_prefix}] + - self.common_parameters + - self.prune_parameters)) + self.__do_launch(op, self.repository.location, params.common, + [{'prefix': self.archive_prefix}] + params.create) + else: raise NotImplementedError("Invalid operation '%s'" % str(op.operation)) @@ -403,7 +372,7 @@ if not self._terminate: self.__main_thread_queue_and_launch() except Exception as err: - self.logger.exception("Error with backup '%s'" % self._name) + self.logger.exception("Error with backup '%s'" % self.backup_name) self.errors=Errors.ERRORS self.state=State.INACTIVE @@ -445,7 +414,7 @@ self.__update_status() # Wait under scheduled wait - self.scheduler.wait_until(now+delay, self._cond, self._name) + self.scheduler.wait_until(now+delay, self._cond, self.backup_name) else: # Nothing scheduled - just wait self.logger.info("Waiting for manual scheduling") @@ -466,7 +435,7 @@ self.__update_status() res=self.repository.queue_action(self._cond, action=self.__launch_and_wait, - name=self._name) + name=self.backup_name) if not res and not self._terminate: self.logger.debug("Queueing aborted") self.scheduled_operation=None
--- a/borgend.py Fri Jan 26 10:35:00 2018 +0000 +++ b/borgend.py Fri Jan 26 19:04:04 2018 +0000 @@ -42,6 +42,7 @@ import config from scheduler import Scheduler from fifolog import FIFOHandler +from repository import Repository # # Argument processing @@ -80,6 +81,7 @@ # Parse args. Let argparse handle errors/exit if there are any args= do_args() tray = None + repos=[] backups=[] try: @@ -105,6 +107,13 @@ scheduler = Scheduler() scheduler.start() + repoconfigs=config.settings['repositories'] + + for i in range(len(repoconfigs)): + logger.info('Setting up repository %d' % i) + r=Repository(i, repoconfigs[i]) + repos.append(r) + backupconfigs=config.settings['backups'] for i in range(len(backupconfigs)): @@ -112,6 +121,9 @@ b=Backup(i, backupconfigs[i], scheduler) backups.append(b) + for r in repos: + r.start() + for b in backups: b.start() @@ -133,6 +145,9 @@ for b in backups: b.terminate() + for r in repos: + r.terminate() + if tray: tray.quit() else:
--- a/config.example.yaml Fri Jan 26 10:35:00 2018 +0000 +++ b/config.example.yaml Fri Jan 26 19:04:04 2018 +0000 @@ -2,77 +2,84 @@ # Borgend example configuration file # +# General parameters for borg borg: - executable: /usr/local/bin/borg - common_parameters: - create_parameters: - - exclude-from: $HOME/lib/borg-exclude-patterns.txt - prune_parameters: - - daily: 7 - - weekly: 50 + executable: /usr/local/bin/borg + common_parameters: + create_parameters: + - exclude-from: $HOME/lib/borg-exclude-patterns.txt + prune_parameters: + - daily: 7 + - weekly: 50 +# Repositories: configure here locations, passphrases, +# and general access parameters +repositories: + - name: myserver + location: ssh://myserver.invalid/~/storage/borg + keychain_account: borg-backup@mylaptop + # Borg parameters + common_parameters: + # Borg is installed on remote host at ~/bin, which might not be on path + - remote-path: ~/bin/borg + create_parameters: + - compression: lzma + - checkpoint-interval: 600 + + - name: backup1 + location: /Volumes/backup1/borg + keychain_account: borg-backup@mylaptop + +# Backups: configure here which files should be backed up, how frequently, and to +# which repositires. backups: - # 1. Most files in $HOME to ssh://myserver.invalid - - name: Home to 'myserver' - # Backup every 24 hours - backup_interval: 86400 - # Retry every 15 minutes if unable to connect / unfinished backup - retry_interval: 900 - repository: ssh://myserver.invalid/~/storage/borg - archive_prefix: 'all@mylaptop-' - archive_template: '{now:%Y-%m-%d_%H:%M:%S}' - keychain_account: borg-backup@mylaptop - paths: - - $HOME - common_parameters: - # Borg is installed on remote host at ~/bin, - # which might not be on path - - remote-path: ~/bin/borg - create_parameters: - - compression: lzma - - checkpoint-interval: 600 - - pattern: "- $HOME/Downloads/" - - pattern: "- $HOME/Library/Mail/V*/MailData/" - - pattern: "+ $HOME/Library/Mail/" - - pattern: "+ $HOME/Library/Mobile Documents/" - - pattern: "- $HOME/Library/" - - pattern: "- $HOME/.config/borg/security/" + # 1. Most files in $HOME to ssh://myserver.invalid + - name: Home to 'myserver' + # Backup every 24 hours + backup_interval: 86400 + # Retry every 15 minutes if unable to connect / unfinished backup + retry_interval: 900 + repository: myserver + archive_prefix: 'all@mylaptop-' + archive_template: '{now:%Y-%m-%d_%H:%M:%S}' + paths: + - $HOME + create_parameters: + - pattern: "- $HOME/Downloads/" + - pattern: "- $HOME/Library/Mail/V*/MailData/" + - pattern: "+ $HOME/Library/Mail/" + - pattern: "+ $HOME/Library/Mobile Documents/" + - pattern: "- $HOME/Library/" + - pattern: "- $HOME/.config/borg/security/" - # 2. A subset of files $HOME more frequently to ssh://myserver.invalid - - name: Work to 'myserver' + # 2. A subset of files $HOME more frequently to ssh://myserver.invalid + - name: Work to 'myserver' # Backup every 3 hours backup_interval: 10800 # Retry every 15 minutes if unable to connect / unfinished backup retry_interval: 900 - repository: ssh://myserver.invalid/~/storage/borg + repository: myserver archive_prefix: 'work@mylaptop-' archive_template: '{now:%Y-%m-%d_%H:%M:%S}' - keychain_account: borg-backup@mylaptop paths: - - $HOME/work - common_parameters: - # Borg is installed on remote host at ~/bin, - # which might not be on path - - remote-path: ~/bin/borg - create_parameters: - - compression: lzma - - checkpoint-interval: 600 + - $HOME/work - # 3. Manual backup to external hard drive - - name: Home to 'backup1' + # 3. Manual backup to external hard drive + - name: Home to 'backup1' # Manual backup backup_interval: 0 retry_interval: 0 - repository: /Volumes/backup1/borg + repository: backup1 archive_prefix: 'mylaptop-' archive_template: '{now:%Y-%m-%d_%H:%M:%S}' - _keychain_account: borg-backup@mylaptop paths: - - $HOME + - $HOME create_parameters: - - pattern: "- $HOME/Downloads/" - - pattern: "- $HOME/Library/Mail/V*/MailData/" - - pattern: "+ $HOME/Library/Mail/" - - pattern: "+ $HOME/Library/Mobile Documents/" - - pattern: "- $HOME/Library/" - - pattern: "- $HOME/.config/borg/security/" + - pattern: "- $HOME/Downloads/" + - pattern: "- $HOME/Library/Mail/V*/MailData/" + - pattern: "+ $HOME/Library/Mail/" + - pattern: "+ $HOME/Library/Mobile Documents/" + - pattern: "- $HOME/Library/" + - pattern: "- $HOME/.config/borg/security/" + +
--- a/config.py Fri Jan 26 10:35:00 2018 +0000 +++ b/config.py Fri Jan 26 19:04:04 2018 +0000 @@ -8,9 +8,9 @@ import os import string import logging -import borgend import platform from functools import reduce +import borgend logger=borgend.logger.getChild(__name__) @@ -59,6 +59,18 @@ def error(x): raise AssertionError(x) +def check_field(cfg, field, descr, loc, default, check): + if field in cfg: + tmp=cfg[field] + if not check(tmp): + error("%s is of invalid type for %s" % (field, loc)) + return tmp + else: + if default is not None: + return default + else: + error("%s is not configured for %s" % (field, loc)) + def check_bool(cfg, field, descr, loc, default=None): return check_field(cfg, field, descr, loc, default, lambda x: isinstance(x, bool)) @@ -99,28 +111,34 @@ return check_field(cfg, field, descr, loc, default, lambda x: isinstance(x, int) and x>=0) -def check_field(cfg, field, descr, loc, default, check): - if field in cfg: - tmp=cfg[field] - if not check(tmp): - error("%s is of invalid type for %s" % (field, loc)) - return tmp - else: - if default is not None: - return default - else: - error("%s is not configured for %s" % (field, loc)) # -# Conversion of config into command line +# Borg command line parameter configuration helper routines and classes # -def arglistify(args): - flatten=lambda l: [item for sublist in l for item in sublist] - if args is None: - return [] - else: - return flatten([['--' + key, str(d[key])] for d in args for key in d]) +class BorgParameters: + def __init__(self, common, create, prune): + self.common=common or [] + self.create=create or [] + self.prune=prune or [] + + def from_config(cfg, loc): + common=check_list_of_dicts(cfg, 'common_parameters', + 'Borg parameters', loc, default=[]) + + create=check_list_of_dicts(cfg, 'create_parameters', + 'Create parameters', loc, default=[]) + + prune=check_list_of_dicts(cfg, 'prune_parameters', + 'Prune parameters', loc, default=[]) + + return BorgParameters(common, create, prune) + + def __add__(self, other): + common=self.common+other.common + create=self.create+other.create + prune=self.prune+other.prune + return BorgParameters(common, create, prune) # # Load config on module load @@ -153,11 +171,6 @@ def check_and_set(cfg, field, loc, defa, fn): cfg[field]=fn(cfg, field, field, loc, defa[field]) -def check_parameters(cmd): - check_and_set(settings['borg'], cmd+'_parameters', - 'borg', defaults['borg'], - check_list_of_dicts) - check_and_set(settings, 'backup_interval', 'top-level', defaults, check_nonneg_int) check_and_set(settings, 'retry_interval', 'top-level', defaults, check_nonneg_int) check_and_set(settings, 'extract_passphrases_at_startup', 'top-level', defaults, check_nonneg_int) @@ -168,8 +181,5 @@ check_and_set(settings['borg'], 'executable', 'borg', defaults['borg'], check_string) - check_parameters('common') - check_parameters('create') - check_parameters('prune') + borg_parameters=BorgParameters.from_config(settings['borg'], "top-level") -
--- a/instance.py Fri Jan 26 10:35:00 2018 +0000 +++ b/instance.py Fri Jan 26 19:04:04 2018 +0000 @@ -6,8 +6,8 @@ import logging import os import borgend +from config import settings from subprocess import Popen, PIPE -from config import settings, arglistify logger=borgend.logger.getChild(__name__) @@ -19,25 +19,33 @@ 'list': ['--json'], } +# Conversion of config into command line +def arglistify(args): + flatten=lambda l: [item for sublist in l for item in sublist] + if args is None: + return [] + else: + return flatten([['--' + key, str(d[key])] for d in args for key in d]) + class BorgInstance: - def __init__(self, operation, archive_or_repository, args, argsl): + def __init__(self, operation, archive_or_repository, + common_params, op_params, paths): self.operation=operation; - self.args=args; self.archive_or_repository=archive_or_repository; - self.argsl=argsl; + self.common_params=common_params + self.op_params=op_params + self.paths=paths def construct_cmdline(self): cmd=([settings['borg']['executable']]+necessary_opts+ - arglistify(settings['borg']['common_parameters'])+ + arglistify(self.common_params)+ [self.operation]) - tmp1=self.operation+'_parameters' - if tmp1 in settings['borg']: - cmd=cmd+arglistify(settings['borg'][tmp1]) if self.operation in necessary_opts_for: cmd=cmd+necessary_opts_for[self.operation] - return cmd+arglistify(self.args)+[self.archive_or_repository]+self.argsl + return (cmd+arglistify(self.op_params) + +[self.archive_or_repository]+self.paths) def launch(self, passphrase=None): cmd=self.construct_cmdline()
--- a/repository.py Fri Jan 26 10:35:00 2018 +0000 +++ b/repository.py Fri Jan 26 19:04:04 2018 +0000 @@ -3,7 +3,9 @@ # import weakref +import keyring import borgend +import config from scheduler import QueueThread, QueuedEvent logger=borgend.logger.getChild(__name__) @@ -72,21 +74,68 @@ return ev._goodtogo -class Repository(FIFO): - def __init__(self, name): - super().__init__(name = 'RepositoryThread %s' % name) - self.repository_name=name - - -# TODO: Should use weak references but they give KeyError repositories=weakref.WeakValueDictionary() -def get_controller(name): +class Repository(FIFO): + def __decode_config(self, cfg): + loc0='Repository %d' % self.identifier + + self.repository_name=config.check_string(cfg, 'name', 'Name', loc0) + + logger.debug("Configuring repository '%s'" % self.repository_name) + + loc = 'Repository "%s"' + + self.logger=logger.getChild(self.repository_name) + + self.location=config.check_string(cfg, 'location', + 'Target repository location', loc) + + self.borg_parameters=config.BorgParameters.from_config(cfg, loc) + + self.__keychain_account=config.check_string(cfg, 'keychain_account', + 'Keychain account', loc, + default='') + + self.__passphrase=None + + if config.settings['extract_passphrases_at_startup']: + try: + self.extract_passphrase() + except Exception: + pass + + def __init__(self, identifier, cfg): + self.identifier=identifier + self.__decode_config(cfg) + super().__init__(name = 'RepositoryThread %s' % self.repository_name) + repositories[self.repository_name]=self + + def __extract_passphrase(self): + acc=self.__keychain_account + if not self.__passphrase: + if acc and acc!='': + self.logger.debug('Requesting passphrase') + try: + pw=keyring.get_password("borg-backup", acc) + except Exception as err: + self.logger.error('Failed to retrieve passphrase') + raise err + else: + self.logger.debug('Received passphrase') + self.__passphrase=pw + else: + self.__passphrase=None + return self.__passphrase + + def launch_borg_instance(self, inst): + passphrase=self.__extract_passphrase() + inst.launch(passphrase=passphrase) + +def find_repository(name): if name in repositories: - repo = repositories[name] + return repositories[name] else: - repo = Repository(name) - repo.start() - repositories[name] = repo - return repo + return None +
