repository.py

Sun, 28 Jan 2018 11:01:45 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Sun, 28 Jan 2018 11:01:45 +0000
changeset 77
e8773133bf79
parent 75
2a44b9649212
child 79
b075b3db3044
permissions
-rw-r--r--

DreamTime.monotonic() no longer needs to call time.monotonic()
so is more "static" within wakeup periods, and so comparable to
time.monotonic() realisations. Also some PyObjC weirdness handling.

53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Repository abstraction for queuing
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
5 import weakref
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
6 import keyring
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
7 import borgend
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
8 import config
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
9 from scheduler import QueueThread, QueuedEvent
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
10
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
11 logger=borgend.logger.getChild(__name__)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
12
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
13 class FIFOEvent(QueuedEvent):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
14 def __init__(self, cond, name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
15 self._goodtogo=False
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
16 super().__init__(cond, name=name)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
17
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
18 def __lt__(self, other):
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
19 return False
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
20
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
21 class FIFO(QueueThread):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
22 def __init__(self, **kwargs):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
23 super().__init__(target = self._fifo_thread, **kwargs)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
24
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
25 def _fifo_thread(self):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
26 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
27 while not self._terminate:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
28 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
29 if ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
30 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
31 with ev.cond:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
32 if not ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
33 ev._goodtogo=True
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
34 ev.cond.notifyAll()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
35 self._cond.wait()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
36
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
37 # Termination cleanup
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
38 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
39 while ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
40 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
41 with ev.cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
42 ev.cond.notifyAll()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
43 ev=ev.next
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
44
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
45 # cond has to be acquired on entry!
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
46 def queue_action(self, cond, action=lambda: (), name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
47 ev=FIFOEvent(cond, name=name)
53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
48
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
49 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
50 self._insert(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
51
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
52 # This will release the lock on cond, allowing queue manager (scheduler)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
53 # thread to notify us if we are already to be released
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
54 logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
55 ev.cond.wait()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
56
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
57 try:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
58 if ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
59 logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
60 #
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
61 # TODO: action() has to unlink on finish; so should maybe
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
62 # use weak references to event.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
63 # Or we have to make action take all the time, so make the
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
64 # stdout thread.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
65 # OR: Easiest to just move finish-waiting into __launch_check
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
66 # instead of at the outer level of the main loop.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
67 #
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
68 action()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
69 finally:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
70 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
71 self._unlink(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
72 # Let _fifo_thread proceed to next action
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
73 self._cond.notify()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
74
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
75 return ev._goodtogo
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
76
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
77 repositories=weakref.WeakValueDictionary()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
78
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
79 class Repository(FIFO):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
80 def __decode_config(self, cfg):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
81 loc0='Repository %d' % self.identifier
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
82
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
83 self.repository_name=config.check_string(cfg, 'name', 'Name', loc0)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
84
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
85 logger.debug("Configuring repository '%s'" % self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
86
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
87 loc = 'Repository "%s"'
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
88
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
89 self.logger=logger.getChild(self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
90
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
91 self.location=config.check_string(cfg, 'location',
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
92 'Target repository location', loc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
93
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
94 self.borg_parameters=config.BorgParameters.from_config(cfg, loc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
95
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
96 self.__keychain_account=config.check_string(cfg, 'keychain_account',
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
97 'Keychain account', loc,
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
98 default='')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
99
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
100 self.__passphrase=None
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
101
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
102 if config.settings['extract_passphrases_at_startup']:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
103 try:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
104 self.extract_passphrase()
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
105 except Exception:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
106 pass
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
107
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
108 def __init__(self, identifier, cfg):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
109 self.identifier=identifier
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
110 self.__decode_config(cfg)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
111 super().__init__(name = 'RepositoryThread %s' % self.repository_name)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
112 repositories[self.repository_name]=self
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
113
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
114 def __extract_passphrase(self):
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
115 acc=self.__keychain_account
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
116 if not self.__passphrase:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
117 if acc and acc!='':
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
118 self.logger.debug('Requesting passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
119 try:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
120 pw=keyring.get_password("borg-backup", acc)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
121 except Exception as err:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
122 self.logger.error('Failed to retrieve passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
123 raise err
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
124 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
125 self.logger.debug('Received passphrase')
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
126 self.__passphrase=pw
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
127 else:
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
128 self.__passphrase=None
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
129 return self.__passphrase
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
130
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
131 def launch_borg_instance(self, inst):
75
2a44b9649212 UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
132 with self._cond:
2a44b9649212 UI refresh fix; added debug messages
Tuomo Valkonen <tuomov@iki.fi>
parents: 74
diff changeset
133 passphrase=self.__extract_passphrase()
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
134 inst.launch(passphrase=passphrase)
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
135
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
136 def find_repository(name):
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
137 if name in repositories:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
138 return repositories[name]
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
139 else:
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
140 return None
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
141
74
4f56142e7497 Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents: 64
diff changeset
142

mercurial