borgend/repository.py

Sun, 28 Jan 2018 17:54:14 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sun, 28 Jan 2018 17:54:14 +0000
changeset 86
2fe66644c50d
parent 80
a409242121d5
child 87
a214d475aa28
permissions
-rw-r--r--

Can use logging.getLogger directly now after proper packageisation

53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Repository abstraction for queuing
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
5 import weakref
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
6 import keyring
86
2fe66644c50d Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents: 80
diff changeset
7 import logging
80
a409242121d5 Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents: 79
diff changeset
8
a409242121d5 Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents: 79
diff changeset
9 from . import config
a409242121d5 Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents: 79
diff changeset
10 from .scheduler import QueueThread, QueuedEvent
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
11
86
2fe66644c50d Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents: 80
diff changeset
12 logger=logging.getLogger(__name__)
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
13
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
14 class FIFOEvent(QueuedEvent):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
15 def __init__(self, cond, name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
16 self._goodtogo=False
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
17 super().__init__(cond, name=name)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
18
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
19 def __lt__(self, other):
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
20 return False
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
21
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
22 class FIFO(QueueThread):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
23 def __init__(self, **kwargs):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
24 super().__init__(target = self._fifo_thread, **kwargs)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
25
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
26 def _fifo_thread(self):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
27 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
28 while not self._terminate:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
29 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
30 if ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
31 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
32 with ev.cond:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
33 if not ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
34 ev._goodtogo=True
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
35 ev.cond.notifyAll()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
36 self._cond.wait()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
37
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
38 # Termination cleanup
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
39 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
40 while ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
41 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
42 with ev.cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
43 ev.cond.notifyAll()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
44 ev=ev.next
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
45
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
46 # cond has to be acquired on entry!
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
47 def queue_action(self, cond, action=lambda: (), name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
48 ev=FIFOEvent(cond, name=name)
53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
49
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
50 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
51 self._insert(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
52
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
53 # This will release the lock on cond, allowing queue manager (scheduler)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
54 # thread to notify us if we are already to be released
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
55 logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
56 ev.cond.wait()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
57
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
58 try:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
59 if ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
60 logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
61 #
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
62 # TODO: action() has to unlink on finish; so should maybe
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
63 # use weak references to event.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
64 # Or we have to make action take all the time, so make the
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
65 # stdout thread.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
66 # OR: Easiest to just move finish-waiting into __launch_check
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
67 # instead of at the outer level of the main loop.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
68 #
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
69 action()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
70 finally:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
71 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
72 self._unlink(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
73 # Let _fifo_thread proceed to next action
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
74 self._cond.notify()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
75
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
76 return ev._goodtogo
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
77
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
78 repositories=weakref.WeakValueDictionary()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
79
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
80 class Repository(FIFO):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
81 def __decode_config(self, cfg):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
82 loc0='Repository %d' % self.identifier
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
83
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
84 self.repository_name=config.check_string(cfg, 'name', 'Name', loc0)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
85
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
86 logger.debug("Configuring repository '%s'" % self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
87
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
88 loc = 'Repository "%s"'
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
89
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
90 self.logger=logger.getChild(self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
91
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
92 self.location=config.check_string(cfg, 'location',
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
93 'Target repository location', loc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
94
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
95 self.borg_parameters=config.BorgParameters.from_config(cfg, loc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
96
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
97 self.__keychain_account=config.check_string(cfg, 'keychain_account',
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
98 'Keychain account', loc,
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
99 default='')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
100
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
101 self.__passphrase=None
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
102
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
103 if config.settings['extract_passphrases_at_startup']:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
104 try:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
105 self.extract_passphrase()
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
106 except Exception:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
107 pass
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
108
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
109 def __init__(self, identifier, cfg):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
110 self.identifier=identifier
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
111 self.__decode_config(cfg)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
112 super().__init__(name = 'RepositoryThread %s' % self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
113 repositories[self.repository_name]=self
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
114
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
115 def __extract_passphrase(self):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
116 acc=self.__keychain_account
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
117 if not self.__passphrase:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
118 if acc and acc!='':
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
119 self.logger.debug('Requesting passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
120 try:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
121 pw=keyring.get_password("borg-backup", acc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
122 except Exception as err:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
123 self.logger.error('Failed to retrieve passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
124 raise err
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
125 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
126 self.logger.debug('Received passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
127 self.__passphrase=pw
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
128 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
129 self.__passphrase=None
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
130 return self.__passphrase
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
131
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
132 def launch_borg_instance(self, inst):
75
2a44b9649212 UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
133 with self._cond:
2a44b9649212 UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
134 passphrase=self.__extract_passphrase()
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
135 inst.launch(passphrase=passphrase)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
136
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
137 def find_repository(name):
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
138 if name in repositories:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
139 return repositories[name]
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
140 else:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
141 return None
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
142
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
143

mercurial