repository.py

Thu, 25 Jan 2018 22:44:10 +0000

author
Tuomo Valkonen <tuomov@iki.fi>
date
Thu, 25 Jan 2018 22:44:10 +0000
changeset 67
75e514746980
parent 64
6cfe6a89e810
child 74
4f56142e7497
permissions
-rw-r--r--

README Non-MacOS description fix

53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
1 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
2 # Repository abstraction for queuing
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
3 #
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
4
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
5 import weakref
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
6 import borgend
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
7 from scheduler import QueueThread, QueuedEvent
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
8
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
9 logger=borgend.logger.getChild(__name__)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
10
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
11 class FIFOEvent(QueuedEvent):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
12 def __init__(self, cond, name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
13 self._goodtogo=False
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
14 super().__init__(cond, name=name)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
15
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
16 def __lt__(self, other):
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
17 return False
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
18
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
19 class FIFO(QueueThread):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
20 def __init__(self, **kwargs):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
21 super().__init__(target = self._fifo_thread, **kwargs)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
22
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
23 def _fifo_thread(self):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
24 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
25 while not self._terminate:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
26 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
27 if ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
28 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
29 with ev.cond:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
30 if not ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
31 ev._goodtogo=True
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
32 ev.cond.notifyAll()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
33 self._cond.wait()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
34
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
35 # Termination cleanup
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
36 ev=self._list
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
37 while ev:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
38 # We can only remove ev from the list when ev.cond allows
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
39 with ev.cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
40 ev.cond.notifyAll()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
41 ev=ev.next
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
42
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
43 # cond has to be acquired on entry!
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
44 def queue_action(self, cond, action=lambda: (), name=None):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
45 ev=FIFOEvent(cond, name=name)
53
442c558bd632 Generalisation of scheduler thread to general queue threads
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff changeset
46
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
47 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
48 self._insert(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
49
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
50 # This will release the lock on cond, allowing queue manager (scheduler)
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
51 # thread to notify us if we are already to be released
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
52 logger.debug("%s:Queuing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
53 ev.cond.wait()
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
55 try:
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
56 if ev._goodtogo:
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
57 logger.debug("%s:Executing %s", self.name, ev.name or 'UNKNOWN')
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
58 #
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
59 # TODO: action() has to unlink on finish; so should maybe
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
60 # use weak references to event.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
61 # Or we have to make action take all the time, so make the
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
62 # stdout thread.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
63 # OR: Easiest to just move finish-waiting into __launch_check
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
64 # instead of at the outer level of the main loop.
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
65 #
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
66 action()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
67 finally:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
68 with self._cond:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
69 self._unlink(ev)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
70 # Let _fifo_thread proceed to next action
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
71 self._cond.notify()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
72
64
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
73 return ev._goodtogo
6cfe6a89e810 Repository queuing fixes
Tuomo Valkonen <tuomov@iki.fi>
parents: 54
diff changeset
74
54
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
75 class Repository(FIFO):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
76 def __init__(self, name):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
77 super().__init__(name = 'RepositoryThread %s' % name)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
78 self.repository_name=name
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
79
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
80
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
81 # TODO: Should use weak references but they give KeyError
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
82 repositories=weakref.WeakValueDictionary()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
83
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
84 def get_controller(name):
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
85 if name in repositories:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
86 repo = repositories[name]
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
87 else:
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
88 repo = Repository(name)
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
89 repo.start()
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
90 repositories[name] = repo
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
91 return repo
cfcaa5f6ba33 Basic repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents: 53
diff changeset
92

mercurial