Tue, 18 Sep 2018 18:10:03 -0500
Fixed error handling of launch of borg fails
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
2 | # Borgend by Tuomo Valkonen, 2018 |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
3 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
4 | # This file implements the scheduling, running, and borg output processing |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
5 | # for a specific configured backup. |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
6 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
7 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
8 | import logging |
5 | 9 | import time |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
10 | import datetime |
97 | 11 | import re |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
12 | from enum import IntEnum |
49 | 13 | from threading import Thread, Lock, Condition |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
14 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
15 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | from . import repository |
97 | 17 | from .dreamtime import MonotonicTime, DreamTime, RealTime |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
18 | from .instance import BorgInstance |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
19 | from .scheduler import TerminableThread |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
20 | from .exprotect import protect_noreturn |
2 | 21 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
85
diff
changeset
|
22 | _logger=logging.getLogger(__name__) |
31 | 23 | |
98 | 24 | JOIN_TIMEOUT=10 |
64 | 25 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
26 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
27 | # State and operation related helper classes |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
28 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
29 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
30 | class State(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
31 | # State |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
32 | INACTIVE=0 |
98 | 33 | PAUSED=1 |
34 | SCHEDULED=2 | |
35 | QUEUED=3 | |
36 | ACTIVE=4 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
37 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
38 | def __str__(self): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
39 | return _statestring[self] |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
40 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
41 | class Errors(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
42 | OK=0 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
43 | BUSY=1 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
44 | OFFLINE=2 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
45 | ERRORS=3 |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
46 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
47 | def combine(self, other): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
48 | return max(self, other) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
49 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
50 | def ok(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
51 | return self==self.OK |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
52 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
53 | def __str__(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
54 | return _errorstring[self] |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
55 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
56 | _errorstring={ |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
57 | Errors.OK: 'ok', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
58 | Errors.BUSY: 'busy', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
59 | Errors.OFFLINE: 'offline', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
60 | Errors.ERRORS: 'errors' |
6 | 61 | } |
62 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
63 | _statestring={ |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
64 | State.INACTIVE: 'inactive', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
65 | State.PAUSED: 'paused', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
66 | State.SCHEDULED: 'scheduled', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
67 | State.QUEUED: 'queued', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
68 | State.ACTIVE: 'active' |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
69 | } |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
70 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
71 | class Operation: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
72 | CREATE='create' |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
73 | PRUNE='prune' |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
74 | INFO='info' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
75 | LIST='list' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
76 | |
97 | 77 | def __init__(self, type, start_time, **kwargs): |
78 | self.type=type | |
79 | self.start_time=start_time | |
80 | self.finish_time=None | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
81 | self.detail=kwargs |
97 | 82 | self.errors=Errors.OK |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
83 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
84 | def when(self): |
97 | 85 | return self.start_time.realtime() |
86 | ||
87 | def ok(self): | |
88 | return self.errors.ok() | |
89 | ||
90 | def add_error(self, error): | |
91 | self.errors=self.errors.combine(error) | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
92 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
93 | def name(self): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
94 | if 'reason' in self.detail: |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
95 | return str(self.type) + '.' + self.detail['reason'] |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
96 | else: |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
97 | return str(self.type) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
98 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
99 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
100 | class Status(Operation): |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
101 | def __init__(self, backup, op=None): |
108
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
102 | op=backup.current_operation |
109
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
103 | errorsop=backup.current_operation |
97 | 104 | |
108
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
105 | if not op: |
97 | 106 | op=backup.scheduled_operation |
107 | ||
109
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
108 | if not errorsop: |
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
109 | errorsop=backup.previous_operation |
246190bfd501
Better error reporting logic.
Tuomo Valkonen <tuomov@iki.fi>
parents:
108
diff
changeset
|
110 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
111 | if op: |
97 | 112 | super().__init__(op.type, op.start_time, **op.detail) |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
113 | else: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
114 | super().__init__(None, None) |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
115 | |
108
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
116 | if errorsop: |
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
117 | self.errors=errorsop.errors |
6409aa77ddd5
Error status wasn't correctly transmitted to UI after recent changes.
Tuomo Valkonen <tuomov@iki.fi>
parents:
107
diff
changeset
|
118 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
119 | self.name=backup.name |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
120 | self.state=backup.state |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
121 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
122 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
123 | # Miscellaneous helper routines |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
124 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
125 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
126 | loglevel_translation={ |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
127 | 'CRITICAL': logging.CRITICAL, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
128 | 'ERROR': logging.ERROR, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
129 | 'WARNING': logging.WARNING, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
130 | 'DEBUG': logging.DEBUG, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
131 | 'INFO': logging.INFO |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
132 | } |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
133 | |
6 | 134 | def translate_loglevel(x): |
135 | if x in loglevel_translation: | |
136 | return loglevel_translation[x] | |
137 | else: | |
138 | return logging.ERROR | |
139 | ||
15 | 140 | def safe_get_int(t, x): |
141 | if x in t: | |
142 | tmp=t[x] | |
143 | if isinstance(tmp, int): | |
144 | return tmp | |
145 | return None | |
146 | ||
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
147 | def parse_borg_date(d, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
148 | try: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
149 | res=datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S.%f') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
150 | except Exception: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
151 | logger.exception('Unable parse date from borg: "%s"' % d) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
152 | res=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
153 | return res |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
154 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
155 | _checkpoint_str='.checkpoint' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
156 | _checkpoint_str_len=len(_checkpoint_str) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
157 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
158 | def is_checkpoint(name): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
159 | i=name.rfind(_checkpoint_str); |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
160 | return i>=0 and i+_checkpoint_str_len==len(name) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
161 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
162 | def get_archive_time(a, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
163 | if not 'name' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
164 | logger.error('Borg archive list entry missing name') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
165 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
166 | if is_checkpoint(a['name']): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
167 | logger.debug('Skipping checkpoint in archive list') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
168 | return None, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
169 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
170 | thistime=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
171 | if 'start' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
172 | thistime=parse_borg_date(a['start'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
173 | if not thistime and 'time' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
174 | thistime=parse_borg_date(a['time'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
175 | if not thistime: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
176 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
177 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
178 | return thistime, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
179 | |
97 | 180 | _prune_progress_re=re.compile(".*\(([0-9]+)/([0-9]+)\)$") |
181 | # Borg gives very little progress info in easy form, so try to extrat it | |
182 | def check_prune_status(msg): | |
183 | res=_prune_progress_re.match(msg) | |
184 | if res: | |
185 | c=res.groups() | |
186 | try: | |
187 | archive_no=int(c[0]) | |
188 | of_total=int(c[1]) | |
189 | except: | |
190 | pass | |
191 | else: | |
192 | return archive_no, of_total | |
193 | return None, None | |
194 | ||
195 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
196 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
197 | # The Backup class |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
198 | # |
15 | 199 | |
49 | 200 | class Backup(TerminableThread): |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
201 | |
2 | 202 | def __decode_config(self, cfg): |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
203 | loc0='Backup %d' % self.identifier |
2 | 204 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
205 | self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) |
2 | 206 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
207 | _logger.debug("Configuring backup '%s'" % self.backup_name) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
208 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
209 | self.logger=_logger.getChild(self.backup_name) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
210 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
211 | loc="Backup '%s'" % self.backup_name |
2 | 212 | |
54 | 213 | reponame=config.check_string(cfg, 'repository', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
214 | 'Target repository', loc) |
54 | 215 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
216 | self.repository=repository.find_repository(reponame) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
217 | if not self.repository: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
218 | raise Exception("Repository '%s' not configured" % reponame) |
2 | 219 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
220 | self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
221 | 'Archive prefix', loc) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
222 | |
2 | 223 | self.archive_template=config.check_string(cfg, 'archive_template', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
224 | 'Archive template', loc) |
2 | 225 | |
226 | self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
227 | 'Backup interval', loc, |
2 | 228 | config.defaults['backup_interval']) |
229 | ||
97 | 230 | self.prune_interval=config.check_nonneg_int(cfg, 'prune_interval', |
231 | 'Prune interval', loc, | |
232 | config.defaults['prune_interval']) | |
233 | ||
2 | 234 | self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
235 | 'Retry interval', loc, |
2 | 236 | config.defaults['retry_interval']) |
237 | ||
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
238 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
239 | scheduling=config.check_string(cfg, 'scheduling', |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
240 | 'Scheduling mode', loc, |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
241 | default="dreamtime") |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
242 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
243 | if scheduling=="dreamtime": |
97 | 244 | self.timeclass=DreamTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
245 | elif scheduling=="realtime": |
97 | 246 | self.timeclass=MonotonicTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
247 | elif scheduling=="manual": |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
248 | self.backup_interval=0 |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
249 | else: |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
250 | logging.error("Invalid time class '%s' for %s" % (scheduling, loc)) |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
251 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
252 | self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) |
32 | 253 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
254 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
17
diff
changeset
|
255 | |
2 | 256 | |
49 | 257 | def __init__(self, identifier, cfg, scheduler): |
2 | 258 | self.identifier=identifier |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
259 | self.__status_update_callback=None |
98 | 260 | self._pause=False |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
261 | self.scheduler=scheduler |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
262 | self.logger=None # setup up in __decode_config once backup name is known |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
263 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
264 | self.borg_instance=None |
7 | 265 | self.thread_log=None |
8 | 266 | self.thread_res=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
267 | |
97 | 268 | self.previous_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
269 | self.current_operation=None |
10 | 270 | self.scheduled_operation=None |
97 | 271 | self.previous_operation_of_type={} |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
272 | self.state=State.INACTIVE |
97 | 273 | self.timeclass=DreamTime |
116 | 274 | self.start_time=self.timeclass.now() |
49 | 275 | |
276 | self.__decode_config(cfg) | |
277 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
278 | super().__init__(target = self.__main_thread, name = self.backup_name) |
49 | 279 | self.daemon=True |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
280 | |
7 | 281 | def is_running(self): |
49 | 282 | with self._cond: |
8 | 283 | running=self.__is_running_unlocked() |
284 | return running | |
285 | ||
286 | def __is_running_unlocked(self): | |
287 | running=self.current_operation | |
288 | if not running: | |
289 | # Consistency check | |
290 | assert((not self.borg_instance and not self.thread_log and | |
291 | not self.thread_res)) | |
7 | 292 | return running |
293 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
294 | def __block_when_running(self): |
7 | 295 | running=self.is_running() |
296 | assert(not running) | |
2 | 297 | |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
298 | @protect_noreturn |
7 | 299 | def __log_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
300 | self.logger.debug('Log listener thread waiting for entries') |
7 | 301 | success=True |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
302 | for msg in iter(self.borg_instance.read_log, None): |
97 | 303 | self.logger.debug(str(msg)) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
304 | t=msg['type'] |
15 | 305 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
306 | errormsg=None |
15 | 307 | callback=None |
308 | ||
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
309 | if t=='progress_percent': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
310 | current=safe_get_int(msg, 'current') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
311 | total=safe_get_int(msg, 'total') |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
312 | operation_no=safe_get_int(msg, 'operation') |
15 | 313 | if current is not None and total is not None: |
49 | 314 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
315 | self.current_operation.detail['progress_current']=current |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
316 | self.current_operation.detail['progress_total']=total |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
317 | self.current_operation.detail['operation_no']=operation_no |
14
5a988a2c2624
UI: progress percentange support (Borg doesn't seem to be reporting) + error indicator in tray: B?
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
318 | status, callback=self.__status_unlocked() |
15 | 319 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
320 | elif t=='archive_progress': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
321 | original_size=safe_get_int(msg, 'original_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
322 | compressed_size=safe_get_int(msg, 'compressed_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
323 | deduplicated_size=safe_get_int(msg, 'deduplicated_size') |
15 | 324 | if original_size is not None and original_size is not None and deduplicated_size is not None: |
49 | 325 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
326 | self.current_operation.detail['original_size']=original_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
327 | self.current_operation.detail['compressed_size']=compressed_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
328 | self.current_operation.detail['deduplicated_size']=deduplicated_size |
15 | 329 | status, callback=self.__status_unlocked() |
330 | ||
331 | elif t=='progress_message': | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
332 | pass |
15 | 333 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
334 | elif t=='file_status': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
335 | pass |
15 | 336 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
337 | elif t=='log_message': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
338 | if 'levelname' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
339 | msg['levelname']='ERROR' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
340 | if 'message' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
341 | msg['message']='UNKNOWN' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
342 | if 'name' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
343 | msg['name']='borg' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
344 | lvl=translate_loglevel(msg['levelname']) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
345 | self.logger.log(lvl, msg['name'] + ': ' + msg['message']) |
71
a8a5ebb64e02
Changed retry timing to start form end of previous attempt instead of beginning
Tuomo Valkonen <tuomov@iki.fi>
parents:
70
diff
changeset
|
346 | if lvl>=logging.ERROR: |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
347 | errormsg=msg |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
348 | errors=Errors.ERRORS |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
349 | if ('msgid' in msg and |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
350 | (msg['msgid']=='LockTimeout' or # observed in reality |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
351 | msg['msgid']=='LockErrorT' or # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
352 | msg['msgid']=='LockErrorT')): # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
353 | errors=Errors.BUSY |
49 | 354 | with self._cond: |
97 | 355 | self.current_operation.add_error(errors) |
98 | 356 | # Don't notify of errors if we are terminating or pausing |
357 | if not self._terminate_or_pause(): | |
358 | status, callback=self.__status_unlocked() | |
97 | 359 | elif lvl==logging.INFO and self.current_operation.type==Operation.PRUNE: |
360 | # Borg gives very little progress info in easy form, so try to extrat it | |
361 | archive_number, of_total=check_prune_status(msg['message']) | |
362 | if archive_number!=None and of_total!=None: | |
363 | self.current_operation.detail['progress_current_secondary']=archive_number | |
364 | self.current_operation.detail['progress_total_secondary']=of_total | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
365 | status, callback=self.__status_unlocked() |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
366 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
367 | elif t=='question_prompt' or t=='question_prompt_retry': |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
368 | self.logger.error('Did not expect to receive question prompt from borg') |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
369 | with self._cond: |
97 | 370 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
371 | # TODO: terminate org? Send 'NO' reply? |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
372 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
373 | elif (t=='question_invalid_answer' or t=='question_accepted_default' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
374 | or t=='question_accepted_true' or t=='question_accepted_false' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
375 | or t=='question_env_answer'): |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
376 | pass |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
377 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
378 | else: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
379 | self.logger.debug('Unrecognised log entry %s' % str(status)) |
15 | 380 | |
381 | if callback: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
382 | callback(status, errorlog=errormsg) |
6 | 383 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
384 | self.logger.debug('Waiting for borg subprocess to terminate in log thread') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
385 | |
6 | 386 | self.borg_instance.wait() |
387 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
388 | self.logger.debug('Borg subprocess terminated; terminating log listener thread') |
7 | 389 | |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
390 | @protect_noreturn |
7 | 391 | def __result_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
392 | self.logger.debug('Result listener thread waiting for result') |
7 | 393 | |
394 | res=self.borg_instance.read_result() | |
395 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
396 | self.logger.debug('Borg result: %s' % str(res)) |
7 | 397 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
398 | if res is None: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
399 | with self._cond: |
97 | 400 | # Prune gives absolutely no result, so don't complain |
401 | if (self.current_operation.ok() and | |
402 | self.current_operation.type!=Operation.PRUNE): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
403 | self.logger.error('No result from borg despite no error in log') |
97 | 404 | self.current_operation.add_error(Errors.ERRORS) |
405 | elif self.current_operation.type==Operation.LIST: | |
406 | self.__handle_list_result(res) | |
407 | ||
408 | # All other results are discarded | |
409 | ||
410 | def __handle_list_result(self, res): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
411 | ok=True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
412 | latest=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
413 | if 'archives' in res and isinstance(res['archives'], list): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
414 | archives=res['archives'] |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
415 | for a in archives: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
416 | if not isinstance(a, dict): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
417 | self.logger.error('Borg archive list entry not a dictionary') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
418 | ok=False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
419 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
420 | thistime, this_ok=get_archive_time(a, self.logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
421 | if thistime and (not latest or thistime>latest): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
422 | latest=thistime |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
423 | ok=ok and this_ok |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
424 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
425 | logger.error('Borg archive list missing "archives" entry') |
7 | 426 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
427 | if not ok: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
428 | with self._cond: |
97 | 429 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
430 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
431 | if latest: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
432 | self.logger.info('borg info: Previous backup was on %s' % latest.isoformat()) |
97 | 433 | when=MonotonicTime.from_realtime(time.mktime(latest.timetuple())) |
434 | op=Operation(Operation.CREATE, when, reason='listed') | |
435 | op.finish_time=when | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
436 | with self._cond: |
97 | 437 | self.previous_operation_of_type[Operation.CREATE]=op |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
438 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
439 | self.logger.info('borg info: Could not discover a previous backup') |
7 | 440 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
441 | def __do_launch(self, op, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
442 | common_params, op_params, paths=[]): |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
443 | |
119
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
444 | # Set up current_operation here so errors can be added to it in |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
445 | # __main_thread if there is an exception |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
446 | self.current_operation=op |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
447 | # Update scheduled time to real starting time to schedule |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
448 | # next run relative to this |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
449 | self.current_operation.start_time=MonotonicTime.now() |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
450 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
451 | self.logger.debug('Creating BorgInstance') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
452 | |
97 | 453 | inst=BorgInstance(op.type, archive_or_repository, |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
454 | common_params, op_params, paths) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
455 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
456 | self.logger.debug('Launching BorgInstance via repository') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
457 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
458 | # Only the Repository object has access to the passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
459 | self.repository.launch_borg_instance(inst) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
460 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
461 | self.logger.debug('Creating listener threads') |
31 | 462 | |
7 | 463 | t_log=Thread(target=self.__log_listener) |
464 | t_log.daemon=True | |
2 | 465 | |
7 | 466 | t_res=Thread(target=self.__result_listener) |
467 | t_res.daemon=True | |
468 | ||
469 | self.thread_log=t_log | |
470 | self.thread_res=t_res | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
471 | self.borg_instance=inst |
64 | 472 | # Reset error status when starting a new operation |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
473 | self.__update_status(State.ACTIVE) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
474 | |
7 | 475 | t_log.start() |
476 | t_res.start() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
477 | |
49 | 478 | def __launch(self, op): |
115
df0de44d2c4b
Changed one .debug message to .info
Tuomo Valkonen <tuomov@iki.fi>
parents:
109
diff
changeset
|
479 | self.logger.info("Launching '%s'" % str(op.type)) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
480 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
481 | params=(config.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
482 | +self.repository.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
483 | +self.borg_parameters) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
484 | |
97 | 485 | if op.type==Operation.CREATE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
486 | archive="%s::%s%s" % (self.repository.location, |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
487 | self.archive_prefix, |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
488 | self.archive_template) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
489 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
490 | self.__do_launch(op, archive, params.common, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
491 | params.create, self.paths) |
97 | 492 | elif op.type==Operation.PRUNE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
493 | self.__do_launch(op, self.repository.location, params.common, |
97 | 494 | [{'prefix': self.archive_prefix}] + params.prune) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
495 | |
97 | 496 | elif op.type==Operation.LIST: |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
497 | self.__do_launch(op, self.repository.location, params.common, |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
498 | [{'prefix': self.archive_prefix}]) |
8 | 499 | else: |
97 | 500 | raise NotImplementedError("Invalid operation '%s'" % str(op.type)) |
2 | 501 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
502 | # This must be called with self._cond held. |
64 | 503 | def __launch_and_wait(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
504 | op=self.scheduled_operation |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
505 | if not op: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
506 | self.logger.debug("Queued operation aborted") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
507 | else: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
508 | self.scheduled_operation=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
509 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
510 | self.__launch(op) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
511 | |
64 | 512 | self.__wait_finish() |
513 | ||
98 | 514 | def _terminate_or_pause(self): |
515 | return self._terminate or self._pause | |
516 | ||
64 | 517 | def __wait_finish(self): |
518 | # Wait for main logger thread to terminate, or for us to be terminated | |
98 | 519 | while not self._terminate_or_pause() and self.thread_res.is_alive(): |
64 | 520 | self._cond.release() |
116 | 521 | # Maybe wait for borg instead? |
64 | 522 | self.thread_res.join(JOIN_TIMEOUT) |
523 | self._cond.acquire() | |
524 | ||
98 | 525 | # If terminate or pause has been signalled, let outer termination handler |
64 | 526 | # take care of things (Within this Backup class, it would be cleanest |
527 | # to raise an exception instead, but in most other places it's better | |
528 | # to just check self._terminate, so we don't complicate things with | |
529 | # an extra exception.) | |
98 | 530 | if self.thread_res.is_alive(): |
64 | 531 | return |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
532 | |
64 | 533 | self.logger.debug('Waiting for borg and log subprocesses to terminate') |
534 | ||
535 | self._cond.release() | |
536 | self.thread_log.join() | |
537 | self._cond.acquire() | |
538 | ||
539 | if not self.borg_instance.wait(): | |
540 | self.logger.error('Borg subprocess did not terminate') | |
119
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
541 | self.current_operation.add_error(Errors.ERRORS) |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
542 | |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
543 | self.thread_res=None |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
544 | self.thread_log=None |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
545 | self.borg_instance=None |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
546 | |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
547 | self.__mark_current_finished() |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
548 | |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
549 | def __mark_current_finished(self): |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
550 | current=self.current_operation |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
551 | |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
552 | assert(current) |
97 | 553 | |
554 | current.finish_time=MonotonicTime.now() | |
64 | 555 | |
97 | 556 | self.previous_operation_of_type[current.type]=current |
557 | self.previous_operation=current | |
558 | self.current_operation=None | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
559 | |
106
a7bdc239ef62
Added exeption protection decorators to callbacks.
Tuomo Valkonen <tuomov@iki.fi>
parents:
104
diff
changeset
|
560 | @protect_noreturn |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
561 | def __main_thread(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
562 | with self._cond: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
563 | while not self._terminate: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
564 | try: |
64 | 565 | assert(not self.current_operation) |
566 | self.__main_thread_wait_schedule() | |
100
b141bed9e718
macOS "sleep" signals / status are complete bollocks:
Tuomo Valkonen <tuomov@iki.fi>
parents:
98
diff
changeset
|
567 | if ((not self._terminate_or_pause()) and self.scheduled_operation |
98 | 568 | and self.scheduled_operation.start_time <= MonotonicTime.now()): |
64 | 569 | self.__main_thread_queue_and_launch() |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
570 | except Exception as err: |
98 | 571 | self.logger.exception("Exception in backup '%s'" % self.backup_name) |
119
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
572 | current=self.current_operation |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
573 | if current: |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
574 | current.add_error(Errors.ERRORS) |
e5f29271089d
Fixed error handling of launch of borg fails
Tuomo Valkonen <tuomov@iki.fi>
parents:
116
diff
changeset
|
575 | self.__mark_current_finished() |
98 | 576 | finally: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
577 | self.__cleanup() |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
578 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
579 | def __cleanup(self): |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
580 | thread_log=self.thread_log |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
581 | thread_res=self.thread_res |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
582 | borg_instance=self.borg_instance |
98 | 583 | self.scheduled_operation=None |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
584 | self.thread_log=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
585 | self.thread_res=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
586 | self.borg_instance=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
587 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
588 | self._cond.release() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
589 | try: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
590 | if borg_instance: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
591 | self.logger.debug("Terminating a borg instance") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
592 | borg_instance.terminate() |
8 | 593 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
594 | if thread_log: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
595 | self.logger.debug("Waiting for log thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
596 | thread_log.join() |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
597 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
598 | if thread_res: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
599 | self.logger.debug("Waiting for result thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
600 | thread_res.join() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
601 | finally: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
602 | self._cond.acquire() |
98 | 603 | self.current_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
604 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
605 | # Main thread/2. Schedule next operation if there is no manually |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
606 | # requested one |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
607 | def __main_thread_wait_schedule(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
608 | op=None |
98 | 609 | if self._pause: |
610 | self.logger.info("Waiting for resume to be signalled") | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
611 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
612 | self.__update_status(State.PAUSED) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
613 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
614 | self._cond.wait() |
98 | 615 | else: |
616 | if not self.scheduled_operation: | |
617 | op=self.__next_operation_unlocked() | |
618 | if op: | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
619 | self.scheduled_operation=op |
98 | 620 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
621 | self.__update_status(State.SCHEDULED) |
98 | 622 | |
623 | # Wait under scheduled wait | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
624 | eventname=op.name() + '@' + self.backup_name |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
625 | self.scheduler.wait_until(op.start_time, self._cond, eventname) |
98 | 626 | else: |
627 | # Nothing scheduled - just wait | |
628 | self.logger.info("Waiting for manual scheduling") | |
629 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
630 | self.__update_status(State.INACTIVE) |
98 | 631 | |
632 | self._cond.wait() | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
633 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
634 | # Main thread/3. If there is a scheduled operation (it might have been |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
635 | # changed manually from 'op' created in __main_thread_wait_schedule above), |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
636 | # queue it on the repository, and launch the operation once repository |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
637 | # available |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
638 | def __main_thread_queue_and_launch(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
639 | if self.scheduled_operation: |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
640 | |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
641 | self.__update_status(State.QUEUED) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
642 | |
64 | 643 | res=self.repository.queue_action(self._cond, |
644 | action=self.__launch_and_wait, | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
645 | name=self.backup_name) |
98 | 646 | if not res: |
64 | 647 | self.logger.debug("Queueing aborted") |
8 | 648 | |
649 | def __next_operation_unlocked(self): | |
97 | 650 | listop=self.__next_operation_list() |
651 | if listop: | |
652 | return listop | |
653 | ||
654 | create=self.__next_operation_type(Operation.CREATE, | |
655 | self.backup_interval, | |
656 | important=True, | |
657 | initial_reason='initial'); | |
658 | ||
659 | prune=self.__next_operation_type(Operation.PRUNE, | |
660 | self.prune_interval, | |
661 | important=False, | |
662 | initial_reason=None); | |
663 | ||
116 | 664 | if prune: |
665 | self.logger.debug("prune scheduled at %s " % prune.start_time.isoformat()) | |
666 | else: | |
667 | self.logger.debug("no prune scheduled") | |
668 | ||
97 | 669 | if not prune: |
670 | return create | |
671 | elif not create: | |
672 | return prune | |
673 | elif create.start_time < prune.start_time: | |
674 | return create | |
675 | else: | |
676 | return prune | |
677 | ||
678 | def __next_operation_list(self): | |
679 | reason='initial' | |
680 | # Unless manual backup has been chosen (backup_interval<=0), perform | |
681 | # repository listing if no previous create operation known, or if we | |
682 | # just pruned the repository | |
683 | if self.backup_interval<=0: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
684 | return None |
97 | 685 | elif (self.previous_operation and |
686 | self.previous_operation.type==Operation.PRUNE): | |
687 | tm=MonotonicTime.now() | |
688 | reason='post-prune' | |
689 | elif Operation.LIST in self.previous_operation_of_type: | |
690 | prev=self.previous_operation_of_type[Operation.LIST] | |
691 | if prev.ok(): | |
692 | return None | |
107
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
693 | if self.retry_interval<=0: |
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
694 | # Do not retry in case of errors if retry interval is <= 0 |
10 | 695 | return None |
97 | 696 | # Attempt after retry interval |
697 | tm=MonotonicTime.after_other(prev.finish_time, self.retry_interval) | |
698 | else: | |
699 | # Nothing has been attempted: run immediately | |
700 | tm=MonotonicTime.now() | |
701 | return Operation(Operation.LIST, tm, reason=reason) | |
702 | ||
703 | def __next_operation_type(self, optype, standard_interval, | |
704 | important=False, | |
705 | initial_reason=None): | |
706 | if optype not in self.previous_operation_of_type: | |
707 | # No previous operation exists; perform immediately | |
708 | # if important, otherwise after standard interval. | |
709 | # Do not perform if manual operation selected by | |
710 | # setting standard_interval<=0 | |
711 | if standard_interval<=0: | |
10 | 712 | return None |
713 | else: | |
97 | 714 | if important: |
715 | tm=MonotonicTime.now() | |
716 | else: | |
116 | 717 | tm=self.timeclass.after_other(self.start_time, standard_interval) |
97 | 718 | if initial_reason: |
719 | return Operation(optype, tm, reason=initial_reason) | |
96
de8ac6c470d8
Backup scheduling fixes in case of an initial backup
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
720 | else: |
97 | 721 | return Operation(optype, tm) |
5 | 722 | else: |
97 | 723 | # Previous operation has been performed; perform after |
724 | # retry interval if there were errors, otherwise after | |
725 | # standard interval. | |
726 | prev=self.previous_operation_of_type[optype] | |
727 | if not prev.ok(): | |
107
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
728 | # Do not retry in case of errors if retry interval is <= 0 |
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
729 | if self.retry_interval<=0: |
f63835e9f29e
Oops, a zero retry_interval check had gone missing in scheduling reorganisation.
Tuomo Valkonen <tuomov@iki.fi>
parents:
106
diff
changeset
|
730 | return None |
97 | 731 | tm=MonotonicTime.after_other(prev.start_time, |
732 | self.retry_interval) | |
733 | return Operation(optype, tm, reason='retry') | |
734 | elif standard_interval>0: | |
735 | tm=self.timeclass.after_other(prev.start_time, | |
736 | standard_interval) | |
737 | return Operation(optype, tm) | |
738 | else: | |
739 | # Manual operation is standard_interval is zero. | |
740 | return None | |
10 | 741 | |
742 | def __status_unlocked(self): | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
743 | callback=self.__status_update_callback |
97 | 744 | status=Status(self) |
10 | 745 | |
746 | return status, callback | |
747 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
748 | def __update_status(self, state): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
749 | self.logger.debug("Entering %s state", str(state)) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
750 | self.state=state |
49 | 751 | status, callback = self.__status_unlocked() |
752 | if callback: | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
753 | #self._cond.release() |
49 | 754 | try: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
755 | callback(status) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
756 | except Exception: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
757 | self.logger.exception("Status update error") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
758 | #finally: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
759 | # self._cond.acquire() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
760 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
761 | # |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
762 | # Interface functions |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
763 | # |
49 | 764 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
765 | def set_status_update_callback(self, callback): |
49 | 766 | with self._cond: |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
767 | self.__status_update_callback=callback |
10 | 768 | |
49 | 769 | def status(self): |
770 | with self._cond: | |
771 | res=self.__status_unlocked() | |
772 | return res[0] | |
10 | 773 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
774 | def create(self): |
97 | 775 | op=Operation(Operation.CREATE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
776 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
777 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
778 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
779 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
780 | def prune(self): |
97 | 781 | op=Operation(Operation.PRUNE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
782 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
783 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
784 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
785 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
786 | # TODO: Decide exact (manual) abort mechanism. Perhaps two stages |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
787 | def abort(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
788 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
789 | if self.borg_instance: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
790 | self.borg_instance.terminate() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
791 | |
98 | 792 | def is_paused(self): |
793 | with self._cond: | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
794 | paused=(self.state==State.PAUSED) |
98 | 795 | return paused |
796 | ||
797 | def pause(self): | |
798 | with self._cond: | |
799 | self.logger.debug('Pause signalled') | |
800 | self.scheduled_operation=None | |
801 | self._pause=True | |
802 | self._cond.notify() | |
803 | ||
804 | def resume(self): | |
805 | with self._cond: | |
806 | self.logger.debug('Resume signalled') | |
807 | self._pause=False | |
808 | self._cond.notify() | |
809 |