comparison: backup.py
backup.py
- changeset 74
- 4f56142e7497
- parent 71
- a8a5ebb64e02
- child 76
- 4b08fca3ce34
equal
deleted
inserted
replaced
| 73:4f0e9cf8f230 | 74:4f56142e7497 |
|---|---|
| 3 # | 3 # |
| 4 | 4 |
| 5 import config | 5 import config |
| 6 import logging | 6 import logging |
| 7 import time | 7 import time |
| 8 import keyring | |
| 9 import borgend | 8 import borgend |
| 10 import repository | 9 import repository |
| 11 from enum import IntEnum | 10 from enum import IntEnum |
| 12 from instance import BorgInstance | 11 from instance import BorgInstance |
| 13 from threading import Thread, Lock, Condition | 12 from threading import Thread, Lock, Condition |
| 105 # | 104 # |
| 106 | 105 |
| 107 class Backup(TerminableThread): | 106 class Backup(TerminableThread): |
| 108 | 107 |
| 109 def __decode_config(self, cfg): | 108 def __decode_config(self, cfg): |
| 110 loc0='backup target %d' % self.identifier | 109 loc0='Backup %d' % self.identifier |
| 111 | 110 |
| 112 self._name=config.check_string(cfg, 'name', 'Name', loc0) | 111 self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) |
| 113 | 112 |
| 114 self.logger=logger.getChild(self._name) | 113 logger.debug("Configuring backup '%s'" % self.backup_name) |
| 115 | 114 |
| 116 self.loc='backup target "%s"' % self._name | 115 self.logger=logger.getChild(self.backup_name) |
| 116 | |
| 117 loc="Backup '%s'" % self.backup_name | |
| 117 | 118 |
| 118 reponame=config.check_string(cfg, 'repository', | 119 reponame=config.check_string(cfg, 'repository', |
| 119 'Target repository', self.loc) | 120 'Target repository', loc) |
| 120 | 121 |
| 121 self.repository=repository.get_controller(reponame) | 122 self.repository=repository.find_repository(reponame) |
| 123 if not self.repository: | |
| 124 raise Exception("Repository '%s' not configured" % reponame) | |
| 122 | 125 |
| 123 self.archive_prefix=config.check_string(cfg, 'archive_prefix', | 126 self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
| 124 'Archive prefix', self.loc) | 127 'Archive prefix', loc) |
| 125 | 128 |
| 126 self.archive_template=config.check_string(cfg, 'archive_template', | 129 self.archive_template=config.check_string(cfg, 'archive_template', |
| 127 'Archive template', self.loc) | 130 'Archive template', loc) |
| 128 | 131 |
| 129 self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | 132 self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', |
| 130 'Backup interval', self.loc, | 133 'Backup interval', loc, |
| 131 config.defaults['backup_interval']) | 134 config.defaults['backup_interval']) |
| 132 | 135 |
| 133 self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', | 136 self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', |
| 134 'Retry interval', self.loc, | 137 'Retry interval', loc, |
| 135 config.defaults['retry_interval']) | 138 config.defaults['retry_interval']) |
| 136 | 139 |
| 137 self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', self.loc) | 140 self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) |
| 138 | 141 |
| 139 self.common_parameters=config.check_list_of_dicts(cfg, 'common_parameters', | 142 self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
| 140 'Borg parameters', self.loc, | 143 |
| 141 default=[]) | |
| 142 | |
| 143 self.create_parameters=config.check_list_of_dicts(cfg, 'create_parameters', | |
| 144 'Create parameters', self.loc, | |
| 145 default=[]) | |
| 146 | |
| 147 self.prune_parameters=config.check_list_of_dicts(cfg, 'prune_parameters', | |
| 148 'Prune parameters', self.loc, | |
| 149 default=[]) | |
| 150 | |
| 151 self.__keychain_account=config.check_string(cfg, 'keychain_account', | |
| 152 'Keychain account', self.loc, | |
| 153 default='') | |
| 154 | |
| 155 self.__passphrase=None | |
| 156 | |
| 157 if config.settings['extract_passphrases_at_startup']: | |
| 158 try: | |
| 159 self.extract_passphrase() | |
| 160 except Exception: | |
| 161 pass | |
| 162 | |
| 163 def extract_passphrase(self): | |
| 164 acc=self.__keychain_account | |
| 165 if not self.__passphrase: | |
| 166 if acc and acc!='': | |
| 167 self.logger.debug('Requesting passphrase') | |
| 168 try: | |
| 169 pw=keyring.get_password("borg-backup", acc) | |
| 170 except Exception as err: | |
| 171 self.logger.error('Failed to retrieve passphrase') | |
| 172 raise err | |
| 173 else: | |
| 174 self.logger.debug('Received passphrase') | |
| 175 self.__passphrase=pw | |
| 176 else: | |
| 177 self.__passphrase=None | |
| 178 return self.__passphrase | |
| 179 | 144 |
| 180 def __init__(self, identifier, cfg, scheduler): | 145 def __init__(self, identifier, cfg, scheduler): |
| 181 self.identifier=identifier | 146 self.identifier=identifier |
| 182 self.config=config | |
| 183 self.__status_update_callback=None | 147 self.__status_update_callback=None |
| 184 self.scheduler=scheduler | 148 self.scheduler=scheduler |
| 185 self.logger=None # setup up __decode_config once backup name is known | 149 self.logger=None # setup up in __decode_config once backup name is known |
| 186 | 150 |
| 187 self.borg_instance=None | 151 self.borg_instance=None |
| 188 self.thread_log=None | 152 self.thread_log=None |
| 189 self.thread_res=None | 153 self.thread_res=None |
| 190 | 154 |
| 195 self.state=State.INACTIVE | 159 self.state=State.INACTIVE |
| 196 self.errors=Errors.OK | 160 self.errors=Errors.OK |
| 197 | 161 |
| 198 self.__decode_config(cfg) | 162 self.__decode_config(cfg) |
| 199 | 163 |
| 200 super().__init__(target = self.__main_thread, name = self._name) | 164 super().__init__(target = self.__main_thread, name = self.backup_name) |
| 201 self.daemon=True | 165 self.daemon=True |
| 202 | 166 |
| 203 def is_running(self): | 167 def is_running(self): |
| 204 with self._cond: | 168 with self._cond: |
| 205 running=self.__is_running_unlocked() | 169 running=self.__is_running_unlocked() |
| 291 res=self.borg_instance.read_result() | 255 res=self.borg_instance.read_result() |
| 292 | 256 |
| 293 self.logger.debug('Borg result: %s' % str(res)) | 257 self.logger.debug('Borg result: %s' % str(res)) |
| 294 | 258 |
| 295 with self._cond: | 259 with self._cond: |
| 296 if res is None: | 260 if res is None and self.errors.ok(): |
| 297 self.logger.error('No result from borg despite no error in log') | 261 self.logger.error('No result from borg despite no error in log') |
| 298 if errors.ok(): | 262 self.errors=Errors.ERRORS |
| 299 self.errors=self.errors.combine(Errors.ERRORS) | 263 |
| 300 | 264 |
| 301 | 265 def __do_launch(self, op, archive_or_repository, |
| 302 def __do_launch(self, op, archive_or_repository, *args): | 266 common_params, op_params, paths=[]): |
| 303 passphrase=self.extract_passphrase() | 267 |
| 304 | 268 inst=BorgInstance(op.operation, archive_or_repository, |
| 305 inst=BorgInstance(op.operation, archive_or_repository, *args) | 269 common_params, op_params, paths) |
| 306 inst.launch(passphrase=passphrase) | 270 |
| 271 # Only the Repository object has access to the passphrase | |
| 272 self.repository.launch_borg_instance(inst) | |
| 307 | 273 |
| 308 self.logger.debug('Creating listener threads') | 274 self.logger.debug('Creating listener threads') |
| 309 | 275 |
| 310 t_log=Thread(target=self.__log_listener) | 276 t_log=Thread(target=self.__log_listener) |
| 311 t_log.daemon=True | 277 t_log.daemon=True |
| 326 self.__update_status() | 292 self.__update_status() |
| 327 | 293 |
| 328 t_log.start() | 294 t_log.start() |
| 329 t_res.start() | 295 t_res.start() |
| 330 | 296 |
| 297 | |
| 331 def __launch(self, op): | 298 def __launch(self, op): |
| 332 self.logger.debug("Launching '%s'" % str(op.operation)) | 299 self.logger.debug("Launching '%s'" % str(op.operation)) |
| 333 | 300 |
| 301 params=(config.borg_parameters | |
| 302 +self.repository.borg_parameters | |
| 303 +self.borg_parameters) | |
| 304 | |
| 334 if op.operation==Operation.CREATE: | 305 if op.operation==Operation.CREATE: |
| 335 archive="%s::%s%s" % (self.repository.repository_name, | 306 archive="%s::%s%s" % (self.repository.location, |
| 336 self.archive_prefix, | 307 self.archive_prefix, |
| 337 self.archive_template) | 308 self.archive_template) |
| 338 | 309 |
| 339 self.__do_launch(op, archive, | 310 self.__do_launch(op, archive, params.common, |
| 340 self.common_parameters+self.create_parameters, | 311 params.create, self.paths) |
| 341 self.paths) | |
| 342 elif op.operation==Operation.PRUNE: | 312 elif op.operation==Operation.PRUNE: |
| 343 self.__do_launch(op, self.repository.repository_name, | 313 self.__do_launch(op, self.repository.location, params.common, |
| 344 ([{'prefix': self.archive_prefix}] + | 314 [{'prefix': self.archive_prefix}] + params.create) |
| 345 self.common_parameters + | 315 |
| 346 self.prune_parameters)) | |
| 347 else: | 316 else: |
| 348 raise NotImplementedError("Invalid operation '%s'" % str(op.operation)) | 317 raise NotImplementedError("Invalid operation '%s'" % str(op.operation)) |
| 349 | 318 |
| 350 # This must be called with self._cond held. | 319 # This must be called with self._cond held. |
| 351 def __launch_and_wait(self): | 320 def __launch_and_wait(self): |
| 401 assert(not self.current_operation) | 370 assert(not self.current_operation) |
| 402 self.__main_thread_wait_schedule() | 371 self.__main_thread_wait_schedule() |
| 403 if not self._terminate: | 372 if not self._terminate: |
| 404 self.__main_thread_queue_and_launch() | 373 self.__main_thread_queue_and_launch() |
| 405 except Exception as err: | 374 except Exception as err: |
| 406 self.logger.exception("Error with backup '%s'" % self._name) | 375 self.logger.exception("Error with backup '%s'" % self.backup_name) |
| 407 self.errors=Errors.ERRORS | 376 self.errors=Errors.ERRORS |
| 408 | 377 |
| 409 self.state=State.INACTIVE | 378 self.state=State.INACTIVE |
| 410 self.scheduled_operation=None | 379 self.scheduled_operation=None |
| 411 | 380 |
| 443 self.scheduled_operation=op | 412 self.scheduled_operation=op |
| 444 self.state=State.SCHEDULED | 413 self.state=State.SCHEDULED |
| 445 self.__update_status() | 414 self.__update_status() |
| 446 | 415 |
| 447 # Wait under scheduled wait | 416 # Wait under scheduled wait |
| 448 self.scheduler.wait_until(now+delay, self._cond, self._name) | 417 self.scheduler.wait_until(now+delay, self._cond, self.backup_name) |
| 449 else: | 418 else: |
| 450 # Nothing scheduled - just wait | 419 # Nothing scheduled - just wait |
| 451 self.logger.info("Waiting for manual scheduling") | 420 self.logger.info("Waiting for manual scheduling") |
| 452 | 421 |
| 453 self.state=State.INACTIVE | 422 self.state=State.INACTIVE |
| 464 self.logger.debug("Queuing") | 433 self.logger.debug("Queuing") |
| 465 self.state=State.QUEUED | 434 self.state=State.QUEUED |
| 466 self.__update_status() | 435 self.__update_status() |
| 467 res=self.repository.queue_action(self._cond, | 436 res=self.repository.queue_action(self._cond, |
| 468 action=self.__launch_and_wait, | 437 action=self.__launch_and_wait, |
| 469 name=self._name) | 438 name=self.backup_name) |
| 470 if not res and not self._terminate: | 439 if not res and not self._terminate: |
| 471 self.logger.debug("Queueing aborted") | 440 self.logger.debug("Queueing aborted") |
| 472 self.scheduled_operation=None | 441 self.scheduled_operation=None |
| 473 self.state=State.INACTIVE | 442 self.state=State.INACTIVE |
| 474 self.__update_status() | 443 self.__update_status() |
