repository.py

Sun, 28 Jan 2018 00:11:18 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sun, 28 Jan 2018 00:11:18 +0000
changeset 76
4b08fca3ce34
parent 75
2a44b9649212
child 79
b075b3db3044
permissions
-rw-r--r--

Dreamtime scheduling: discount system sleep periods

53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Repository abstraction for queuing
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
5 import weakref
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
6 import keyring
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
7 import borgend
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
8 import config
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
9 from scheduler import QueueThread, QueuedEvent
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
10
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
11 logger=borgend.logger.getChild(__name__)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
12
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
13 class FIFOEvent(QueuedEvent):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
14 def __init__(self, cond, name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
15 self._goodtogo=False
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
16 super().__init__(cond, name=name)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
17
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
18 def __lt__(self, other):
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
19 return False
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
20
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
21 class FIFO(QueueThread):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
22 def __init__(self, **kwargs):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
23 super().__init__(target = self._fifo_thread, **kwargs)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
24
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
25 def _fifo_thread(self):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
26 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
27 while not self._terminate:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
28 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
29 if ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
30 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
31 with ev.cond:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
32 if not ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
33 ev._goodtogo=True
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
34 ev.cond.notifyAll()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
35 self._cond.wait()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
36
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
37 # Termination cleanup
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
38 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
39 while ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
40 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
41 with ev.cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
42 ev.cond.notifyAll()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
43 ev=ev.next
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
44
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
45 # cond has to be acquired on entry!
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
46 def queue_action(self, cond, action=lambda: (), name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
47 ev=FIFOEvent(cond, name=name)
53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
48
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
49 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
50 self._insert(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
51
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
52 # This will release the lock on cond, allowing queue manager (scheduler)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
53 # thread to notify us if we are already to be released
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
54 logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
55 ev.cond.wait()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
56
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
57 try:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
58 if ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
59 logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
60 #
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
61 # TODO: action() has to unlink on finish; so should maybe
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
62 # use weak references to event.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
63 # Or we have to make action take all the time, so make the
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
64 # stdout thread.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
65 # OR: Easiest to just move finish-waiting into __launch_check
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
66 # instead of at the outer level of the main loop.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
67 #
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
68 action()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
69 finally:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
70 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
71 self._unlink(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
72 # Let _fifo_thread proceed to next action
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
73 self._cond.notify()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
74
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
75 return ev._goodtogo
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
76
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
77 repositories=weakref.WeakValueDictionary()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
78
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
79 class Repository(FIFO):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
80 def __decode_config(self, cfg):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
81 loc0='Repository %d' % self.identifier
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
82
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
83 self.repository_name=config.check_string(cfg, 'name', 'Name', loc0)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
84
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
85 logger.debug("Configuring repository '%s'" % self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
86
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
87 loc = 'Repository "%s"'
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
88
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
89 self.logger=logger.getChild(self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
90
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
91 self.location=config.check_string(cfg, 'location',
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
92 'Target repository location', loc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
93
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
94 self.borg_parameters=config.BorgParameters.from_config(cfg, loc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
95
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
96 self.__keychain_account=config.check_string(cfg, 'keychain_account',
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
97 'Keychain account', loc,
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
98 default='')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
99
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
100 self.__passphrase=None
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
101
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
102 if config.settings['extract_passphrases_at_startup']:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
103 try:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
104 self.extract_passphrase()
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
105 except Exception:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
106 pass
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
107
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
108 def __init__(self, identifier, cfg):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
109 self.identifier=identifier
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
110 self.__decode_config(cfg)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
111 super().__init__(name = 'RepositoryThread %s' % self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
112 repositories[self.repository_name]=self
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
113
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
114 def __extract_passphrase(self):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
115 acc=self.__keychain_account
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
116 if not self.__passphrase:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
117 if acc and acc!='':
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
118 self.logger.debug('Requesting passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
119 try:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
120 pw=keyring.get_password("borg-backup", acc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
121 except Exception as err:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
122 self.logger.error('Failed to retrieve passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
123 raise err
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
124 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
125 self.logger.debug('Received passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
126 self.__passphrase=pw
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
127 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
128 self.__passphrase=None
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
129 return self.__passphrase
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
130
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
131 def launch_borg_instance(self, inst):
75
2a44b9649212 UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
132 with self._cond:
2a44b9649212 UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
133 passphrase=self.__extract_passphrase()
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
134 inst.launch(passphrase=passphrase)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
135
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
136 def find_repository(name):
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
137 if name in repositories:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
138 return repositories[name]
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
139 else:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
140 return None
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
141
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
142

mercurial