Sun, 28 Jan 2018 19:27:34 +0000
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
2 | # Repository abstraction for queuing |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
3 | # |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
4 | |
54 | 5 | import weakref |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
6 | import keyring |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
7 | import logging |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
8 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
9 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
10 | from .scheduler import QueueThread, QueuedEvent |
54 | 11 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
12 | logger=logging.getLogger(__name__) |
64 | 13 | |
54 | 14 | class FIFOEvent(QueuedEvent): |
15 | def __init__(self, cond, name=None): | |
16 | self._goodtogo=False | |
17 | super().__init__(cond, name=name) | |
18 | ||
19 | def __lt__(self, other): | |
64 | 20 | return False |
54 | 21 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
22 | # This FIFO essentially a fancy semaphore: Each thread waits on its own |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
23 | # Condition, so can also be woken up from other executing threads. |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
24 | # If they are woken up by the FIFO, then a "good to go" flag is set; |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
25 | # and a specified action executed. Otherwise this is not done. |
54 | 26 | class FIFO(QueueThread): |
27 | def __init__(self, **kwargs): | |
28 | super().__init__(target = self._fifo_thread, **kwargs) | |
29 | ||
30 | def _fifo_thread(self): | |
31 | with self._cond: | |
32 | while not self._terminate: | |
33 | ev=self._list | |
34 | if ev: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
35 | # We have to release lock on self._cond before obtaining |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
36 | # one on ev.cond to avoid race conditions with |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
37 | # self.queue_acion |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
38 | self._cond.release() |
54 | 39 | with ev.cond: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
40 | # Just set "good to go" flag and notify the queued |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
41 | # thread. To keep blocking other thread, it is the |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
42 | # job of the queued thred to remove itself. |
64 | 43 | if not ev._goodtogo: |
44 | ev._goodtogo=True | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
45 | ev.cond.notify_all() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
46 | self._cond.acquire() |
54 | 47 | self._cond.wait() |
48 | ||
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
49 | self.logger.debug('Terminating') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
50 | |
54 | 51 | # Termination cleanup |
52 | ev=self._list | |
53 | while ev: | |
54 | # We can only remove ev from the list when ev.cond allows | |
55 | with ev.cond: | |
56 | ev.cond.notifyAll() | |
57 | ev=ev.next | |
58 | ||
59 | # cond has to be acquired on entry! | |
60 | def queue_action(self, cond, action=lambda: (), name=None): | |
61 | ev=FIFOEvent(cond, name=name) | |
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
62 | |
54 | 63 | with self._cond: |
64 | self._insert(ev) | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
65 | self._cond.notify() |
54 | 66 | |
64 | 67 | # This will release the lock on cond, allowing queue manager (scheduler) |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
68 | # thread to notify us if we are ready to be released |
64 | 69 | logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN') |
70 | ev.cond.wait() | |
54 | 71 | |
72 | try: | |
64 | 73 | if ev._goodtogo: |
74 | logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN') | |
54 | 75 | action() |
76 | finally: | |
77 | with self._cond: | |
78 | self._unlink(ev) | |
79 | # Let _fifo_thread proceed to next action | |
80 | self._cond.notify() | |
81 | ||
64 | 82 | return ev._goodtogo |
83 | ||
54 | 84 | repositories=weakref.WeakValueDictionary() |
85 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
86 | class Repository(FIFO): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
87 | def __decode_config(self, cfg): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
88 | loc0='Repository %d' % self.identifier |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
89 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
90 | self.repository_name=config.check_string(cfg, 'name', 'Name', loc0) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
91 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
92 | logger.debug("Configuring repository '%s'" % self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
93 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
94 | loc = 'Repository "%s"' |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
95 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
96 | self.logger=logger.getChild(self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
97 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
98 | self.location=config.check_string(cfg, 'location', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
99 | 'Target repository location', loc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
100 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
101 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
102 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
103 | self.__keychain_account=config.check_string(cfg, 'keychain_account', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
104 | 'Keychain account', loc, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
105 | default='') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
106 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
107 | self.__passphrase=None |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
108 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
109 | if config.settings['extract_passphrases_at_startup']: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
110 | try: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
111 | self.__extract_passphrase() |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
112 | except Exception: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
113 | pass |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
114 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
115 | def __init__(self, identifier, cfg): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
116 | self.identifier=identifier |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
117 | self.__decode_config(cfg) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
118 | super().__init__(name = 'RepositoryThread %s' % self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
119 | repositories[self.repository_name]=self |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
120 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
121 | def __extract_passphrase(self): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
122 | if not self.__passphrase: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
123 | acc=self.__keychain_account |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
124 | if acc and acc!='': |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
125 | self.logger.debug('Requesting passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
126 | try: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
127 | pw=keyring.get_password("borg-backup", acc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
128 | except Exception as err: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
129 | self.logger.error('Failed to retrieve passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
130 | raise err |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
131 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
132 | self.logger.debug('Received passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
133 | self.__passphrase=pw |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
134 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
135 | self.__passphrase=None |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
136 | return self.__passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
137 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
138 | def launch_borg_instance(self, inst): |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
139 | try: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
140 | self.logger.debug('launch_borg_instance: entering _cond') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
141 | with self._cond: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
142 | self.logger.debug('launch_borg_instance: entering __extract_passphrase') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
143 | passphrase=self.__extract_passphrase() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
144 | except Exception as err: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
145 | self.logger.error('Aborting operation due to failure to obtain passphrase') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
146 | raise err |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
147 | else: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
148 | inst.launch(passphrase=passphrase) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
149 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
150 | def find_repository(name): |
54 | 151 | if name in repositories: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
152 | return repositories[name] |
54 | 153 | else: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
154 | return None |
54 | 155 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
156 |