Sun, 28 Jan 2018 17:54:14 +0000
Can use logging.getLogger directly now after proper packageisation
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
2 | # Repository abstraction for queuing |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
3 | # |
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
4 | |
54 | 5 | import weakref |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
6 | import keyring |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
7 | import logging |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
8 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
9 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
10 | from .scheduler import QueueThread, QueuedEvent |
54 | 11 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
12 | logger=logging.getLogger(__name__) |
64 | 13 | |
54 | 14 | class FIFOEvent(QueuedEvent): |
15 | def __init__(self, cond, name=None): | |
16 | self._goodtogo=False | |
17 | super().__init__(cond, name=name) | |
18 | ||
19 | def __lt__(self, other): | |
64 | 20 | return False |
54 | 21 | |
22 | class FIFO(QueueThread): | |
23 | def __init__(self, **kwargs): | |
24 | super().__init__(target = self._fifo_thread, **kwargs) | |
25 | ||
26 | def _fifo_thread(self): | |
27 | with self._cond: | |
28 | while not self._terminate: | |
29 | ev=self._list | |
30 | if ev: | |
31 | # We can only remove ev from the list when ev.cond allows | |
32 | with ev.cond: | |
64 | 33 | if not ev._goodtogo: |
34 | ev._goodtogo=True | |
35 | ev.cond.notifyAll() | |
54 | 36 | self._cond.wait() |
37 | ||
38 | # Termination cleanup | |
39 | ev=self._list | |
40 | while ev: | |
41 | # We can only remove ev from the list when ev.cond allows | |
42 | with ev.cond: | |
43 | ev.cond.notifyAll() | |
44 | ev=ev.next | |
45 | ||
46 | # cond has to be acquired on entry! | |
47 | def queue_action(self, cond, action=lambda: (), name=None): | |
48 | ev=FIFOEvent(cond, name=name) | |
53
442c558bd632
Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
49 | |
54 | 50 | with self._cond: |
51 | self._insert(ev) | |
52 | ||
64 | 53 | # This will release the lock on cond, allowing queue manager (scheduler) |
54 | # thread to notify us if we are already to be released | |
55 | logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN') | |
56 | ev.cond.wait() | |
54 | 57 | |
58 | try: | |
64 | 59 | if ev._goodtogo: |
60 | logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN') | |
61 | # | |
62 | # TODO: action() has to unlink on finish; so should maybe | |
63 | # use weak references to event. | |
64 | # Or we have to make action take all the time, so make the | |
65 | # stdout thread. | |
66 | # OR: Easiest to just move finish-waiting into __launch_check | |
67 | # instead of at the outer level of the main loop. | |
68 | # | |
54 | 69 | action() |
70 | finally: | |
71 | with self._cond: | |
72 | self._unlink(ev) | |
73 | # Let _fifo_thread proceed to next action | |
74 | self._cond.notify() | |
75 | ||
64 | 76 | return ev._goodtogo |
77 | ||
54 | 78 | repositories=weakref.WeakValueDictionary() |
79 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
80 | class Repository(FIFO): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
81 | def __decode_config(self, cfg): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
82 | loc0='Repository %d' % self.identifier |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
83 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
84 | self.repository_name=config.check_string(cfg, 'name', 'Name', loc0) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
85 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
86 | logger.debug("Configuring repository '%s'" % self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
87 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
88 | loc = 'Repository "%s"' |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
89 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
90 | self.logger=logger.getChild(self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
91 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
92 | self.location=config.check_string(cfg, 'location', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
93 | 'Target repository location', loc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
94 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
95 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
96 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
97 | self.__keychain_account=config.check_string(cfg, 'keychain_account', |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
98 | 'Keychain account', loc, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
99 | default='') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
100 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
101 | self.__passphrase=None |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
102 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
103 | if config.settings['extract_passphrases_at_startup']: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
104 | try: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
105 | self.extract_passphrase() |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
106 | except Exception: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
107 | pass |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
108 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
109 | def __init__(self, identifier, cfg): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
110 | self.identifier=identifier |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
111 | self.__decode_config(cfg) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
112 | super().__init__(name = 'RepositoryThread %s' % self.repository_name) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
113 | repositories[self.repository_name]=self |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
114 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
115 | def __extract_passphrase(self): |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
116 | acc=self.__keychain_account |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
117 | if not self.__passphrase: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
118 | if acc and acc!='': |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
119 | self.logger.debug('Requesting passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
120 | try: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
121 | pw=keyring.get_password("borg-backup", acc) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
122 | except Exception as err: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
123 | self.logger.error('Failed to retrieve passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
124 | raise err |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
125 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
126 | self.logger.debug('Received passphrase') |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
127 | self.__passphrase=pw |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
128 | else: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
129 | self.__passphrase=None |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
130 | return self.__passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
131 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
132 | def launch_borg_instance(self, inst): |
75
2a44b9649212
UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
133 | with self._cond: |
2a44b9649212
UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
134 | passphrase=self.__extract_passphrase() |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
135 | inst.launch(passphrase=passphrase) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
136 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
137 | def find_repository(name): |
54 | 138 | if name in repositories: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
139 | return repositories[name] |
54 | 140 | else: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
141 | return None |
54 | 142 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
64
diff
changeset
|
143 |