Tue, 14 Aug 2018 22:57:48 -0500
Pruning scheduling fix
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
2 | # Borgend by Tuomo Valkonen, 2018 |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
3 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
4 | # Repository abstraction for queuing. |
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
5 | # |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
6 | |
54 | 7 | import weakref |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
8 | import keyring |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
9 | import logging |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
10 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
11 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
12 | from .scheduler import QueueThread, QueuedEvent |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
101
diff
changeset
|
13 | from .exprotect import protect_noreturn |
54 | 14 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
15 | logger=logging.getLogger(__name__) |
64 | 16 | |
54 | 17 | class FIFOEvent(QueuedEvent): |
18 | def __init__(self, cond, name=None): | |
19 | self._goodtogo=False | |
20 | super().__init__(cond, name=name) | |
21 | ||
101
3068b0de12ee
Part 2 of handling macOS sleep/wake signal brokenness.
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
22 | def is_before(self, other, snapshot=None): |
64 | 23 | return False |
54 | 24 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
25 | # This FIFO essentially a fancy semaphore: Each thread waits on its own |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
26 | # Condition, so can also be woken up from other executing threads. |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
27 | # If they are woken up by the FIFO, then a "good to go" flag is set; |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
28 | # and a specified action executed. Otherwise this is not done. |
54 | 29 | class FIFO(QueueThread): |
30 | def __init__(self, **kwargs): | |
31 | super().__init__(target = self._fifo_thread, **kwargs) | |
32 | ||
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
101
diff
changeset
|
33 | @protect_noreturn |
54 | 34 | def _fifo_thread(self): |
35 | with self._cond: | |
36 | while not self._terminate: | |
37 | ev=self._list | |
38 | if ev: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
39 | # We have to release lock on self._cond before obtaining |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
40 | # one on ev.cond to avoid race conditions with |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
41 | # self.queue_acion |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
42 | self._cond.release() |
54 | 43 | with ev.cond: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
44 | # Just set "good to go" flag and notify the queued |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
45 | # thread. To keep blocking other thread, it is the |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
46 | # job of the queued thred to remove itself. |
64 | 47 | if not ev._goodtogo: |
48 | ev._goodtogo=True | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
49 | ev.cond.notify_all() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
50 | self._cond.acquire() |
54 | 51 | self._cond.wait() |
52 | ||
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
53 | self.logger.debug('Terminating') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
54 | |
54 | 55 | # Termination cleanup |
56 | ev=self._list | |
57 | while ev: | |
58 | # We can only remove ev from the list when ev.cond allows | |
59 | with ev.cond: | |
60 | ev.cond.notifyAll() | |
61 | ev=ev.next | |
62 | ||
63 | # cond has to be acquired on entry! | |
64 | def queue_action(self, cond, action=lambda: (), name=None): | |
65 | ev=FIFOEvent(cond, name=name) | |
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
66 | |
54 | 67 | with self._cond: |
68 | self._insert(ev) | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
69 | self._cond.notify() |
54 | 70 | |
64 | 71 | # This will release the lock on cond, allowing queue manager (scheduler) |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
72 | # thread to notify us if we are ready to be released |
64 | 73 | logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN') |
74 | ev.cond.wait() | |
54 | 75 | |
76 | try: | |
64 | 77 | if ev._goodtogo: |
78 | logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN') | |
54 | 79 | action() |
80 | finally: | |
81 | with self._cond: | |
82 | self._unlink(ev) | |
83 | # Let _fifo_thread proceed to next action | |
84 | self._cond.notify() | |
85 | ||
64 | 86 | return ev._goodtogo |
87 | ||
54 | 88 | repositories=weakref.WeakValueDictionary() |
89 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
90 | class Repository(FIFO): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
91 | def __decode_config(self, cfg): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
92 | loc0='Repository %d' % self.identifier |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
93 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
94 | self.repository_name=config.check_string(cfg, 'name', 'Name', loc0) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
95 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
96 | logger.debug("Configuring repository '%s'" % self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
97 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
98 | loc = 'Repository "%s"' |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
99 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
100 | self.logger=logger.getChild(self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
101 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
102 | self.location=config.check_string(cfg, 'location', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
103 | 'Target repository location', loc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
104 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
105 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
106 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
107 | self.__keychain_account=config.check_string(cfg, 'keychain_account', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
108 | 'Keychain account', loc, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
109 | default='') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
110 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
111 | self.__passphrase=None |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
112 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
113 | if config.settings['extract_passphrases_at_startup']: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
114 | try: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
115 | self.__extract_passphrase() |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
116 | except Exception: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
117 | pass |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
118 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
119 | def __init__(self, identifier, cfg): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
120 | self.identifier=identifier |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
121 | self.__decode_config(cfg) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
122 | super().__init__(name = 'RepositoryThread %s' % self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
123 | repositories[self.repository_name]=self |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
124 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
125 | def __extract_passphrase(self): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
126 | if not self.__passphrase: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
127 | acc=self.__keychain_account |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
128 | if acc and acc!='': |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
129 | self.logger.debug('Requesting passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
130 | try: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
131 | pw=keyring.get_password("borg-backup", acc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
132 | except Exception as err: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
133 | self.logger.error('Failed to retrieve passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
134 | raise err |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
135 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
136 | self.logger.debug('Received passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
137 | self.__passphrase=pw |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
138 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
139 | self.__passphrase=None |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
140 | return self.__passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
141 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
142 | def launch_borg_instance(self, inst): |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
143 | try: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
144 | self.logger.debug('launch_borg_instance: entering _cond') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
145 | with self._cond: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
146 | self.logger.debug('launch_borg_instance: entering __extract_passphrase') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
147 | passphrase=self.__extract_passphrase() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
148 | except Exception as err: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
149 | self.logger.error('Aborting operation due to failure to obtain passphrase') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
150 | raise err |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
151 | else: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
152 | inst.launch(passphrase=passphrase) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
153 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
154 | def find_repository(name): |
54 | 155 | if name in repositories: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
156 | return repositories[name] |
54 | 157 | else: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
158 | return None |
54 | 159 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
160 |