Fri, 26 Jan 2018 19:04:04 +0000
Separated repository configuration form backup configuration;
gave passphrase management to Repository object;
various fixes.
|
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
|
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
2 | # Repository abstraction for queuing |
|
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
3 | # |
|
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
4 | |
| 54 | 5 | import weakref |
|
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
6 | import keyring |
| 64 | 7 | import borgend |
|
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
8 | import config |
| 54 | 9 | from scheduler import QueueThread, QueuedEvent |
| 10 | ||
| 64 | 11 | logger=borgend.logger.getChild(__name__) |
| 12 | ||
| 54 | 13 | class FIFOEvent(QueuedEvent): |
| 14 | def __init__(self, cond, name=None): | |
| 15 | self._goodtogo=False | |
| 16 | super().__init__(cond, name=name) | |
| 17 | ||
| 18 | def __lt__(self, other): | |
| 64 | 19 | return False |
| 54 | 20 | |
| 21 | class FIFO(QueueThread): | |
| 22 | def __init__(self, **kwargs): | |
| 23 | super().__init__(target = self._fifo_thread, **kwargs) | |
| 24 | ||
| 25 | def _fifo_thread(self): | |
| 26 | with self._cond: | |
| 27 | while not self._terminate: | |
| 28 | ev=self._list | |
| 29 | if ev: | |
| 30 | # We can only remove ev from the list when ev.cond allows | |
| 31 | with ev.cond: | |
| 64 | 32 | if not ev._goodtogo: |
| 33 | ev._goodtogo=True | |
| 34 | ev.cond.notifyAll() | |
| 54 | 35 | self._cond.wait() |
| 36 | ||
| 37 | # Termination cleanup | |
| 38 | ev=self._list | |
| 39 | while ev: | |
| 40 | # We can only remove ev from the list when ev.cond allows | |
| 41 | with ev.cond: | |
| 42 | ev.cond.notifyAll() | |
| 43 | ev=ev.next | |
| 44 | ||
| 45 | # cond has to be acquired on entry! | |
| 46 | def queue_action(self, cond, action=lambda: (), name=None): | |
| 47 | ev=FIFOEvent(cond, name=name) | |
|
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
48 | |
| 54 | 49 | with self._cond: |
| 50 | self._insert(ev) | |
| 51 | ||
| 64 | 52 | # This will release the lock on cond, allowing queue manager (scheduler) |
| 53 | # thread to notify us if we are already to be released | |
| 54 | logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN') | |
| 55 | ev.cond.wait() | |
| 54 | 56 | |
| 57 | try: | |
| 64 | 58 | if ev._goodtogo: |
| 59 | logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN') | |
| 60 | # | |
| 61 | # TODO: action() has to unlink on finish; so should maybe | |
| 62 | # use weak references to event. | |
| 63 | # Or we have to make action take all the time, so make the | |
| 64 | # stdout thread. | |
| 65 | # OR: Easiest to just move finish-waiting into __launch_check | |
| 66 | # instead of at the outer level of the main loop. | |
| 67 | # | |
| 54 | 68 | action() |
| 69 | finally: | |
| 70 | with self._cond: | |
| 71 | self._unlink(ev) | |
| 72 | # Let _fifo_thread proceed to next action | |
| 73 | self._cond.notify() | |
| 74 | ||
| 64 | 75 | return ev._goodtogo |
| 76 | ||
| 54 | 77 | repositories=weakref.WeakValueDictionary() |
| 78 | ||
|
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
79 | class Repository(FIFO): |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
80 | def __decode_config(self, cfg): |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
81 | loc0='Repository %d' % self.identifier |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
82 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
83 | self.repository_name=config.check_string(cfg, 'name', 'Name', loc0) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
84 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
85 | logger.debug("Configuring repository '%s'" % self.repository_name) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
86 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
87 | loc = 'Repository "%s"' |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
88 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
89 | self.logger=logger.getChild(self.repository_name) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
90 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
91 | self.location=config.check_string(cfg, 'location', |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
92 | 'Target repository location', loc) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
93 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
94 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
95 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
96 | self.__keychain_account=config.check_string(cfg, 'keychain_account', |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
97 | 'Keychain account', loc, |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
98 | default='') |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
99 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
100 | self.__passphrase=None |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
101 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
102 | if config.settings['extract_passphrases_at_startup']: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
103 | try: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
104 | self.extract_passphrase() |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
105 | except Exception: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
106 | pass |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
107 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
108 | def __init__(self, identifier, cfg): |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
109 | self.identifier=identifier |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
110 | self.__decode_config(cfg) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
111 | super().__init__(name = 'RepositoryThread %s' % self.repository_name) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
112 | repositories[self.repository_name]=self |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
113 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
114 | def __extract_passphrase(self): |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
115 | acc=self.__keychain_account |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
116 | if not self.__passphrase: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
117 | if acc and acc!='': |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
118 | self.logger.debug('Requesting passphrase') |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
119 | try: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
120 | pw=keyring.get_password("borg-backup", acc) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
121 | except Exception as err: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
122 | self.logger.error('Failed to retrieve passphrase') |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
123 | raise err |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
124 | else: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
125 | self.logger.debug('Received passphrase') |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
126 | self.__passphrase=pw |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
127 | else: |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
128 | self.__passphrase=None |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
129 | return self.__passphrase |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
130 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
131 | def launch_borg_instance(self, inst): |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
132 | passphrase=self.__extract_passphrase() |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
133 | inst.launch(passphrase=passphrase) |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
134 | |
|
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
135 | def find_repository(name): |
| 54 | 136 | if name in repositories: |
|
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
137 | return repositories[name] |
| 54 | 138 | else: |
|
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
139 | return None |
| 54 | 140 | |
|
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
141 |