Sun, 04 Feb 2018 09:38:55 +0000
Added some debug log messages
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
1 | # |
89
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
2 | # Borgend by Tuomo Valkonen, 2018 |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
3 | # |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
4 | # This file implements the scheduling, running, and borg output processing |
51cc2e25af38
Added author information headers and content information to source files
Tuomo Valkonen <tuomov@iki.fi>
parents:
88
diff
changeset
|
5 | # for a specific configured backup. |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
6 | # |
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
7 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
8 | import logging |
5 | 9 | import time |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
10 | import datetime |
97 | 11 | import re |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
12 | from enum import IntEnum |
49 | 13 | from threading import Thread, Lock, Condition |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
14 | |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
15 | from . import config |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
16 | from . import repository |
97 | 17 | from .dreamtime import MonotonicTime, DreamTime, RealTime |
80
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
18 | from .instance import BorgInstance |
a409242121d5
Better package-like organisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
79
diff
changeset
|
19 | from .scheduler import TerminableThread |
2 | 20 | |
86
2fe66644c50d
Can use logging.getLogger directly now after proper packageisation
Tuomo Valkonen <tuomov@iki.fi>
parents:
85
diff
changeset
|
21 | _logger=logging.getLogger(__name__) |
31 | 22 | |
98 | 23 | JOIN_TIMEOUT=10 |
64 | 24 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
25 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
26 | # State and operation related helper classes |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
27 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
28 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
29 | class State(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
30 | # State |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
31 | INACTIVE=0 |
98 | 32 | PAUSED=1 |
33 | SCHEDULED=2 | |
34 | QUEUED=3 | |
35 | ACTIVE=4 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
36 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
37 | def __str__(self): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
38 | return _statestring[self] |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
39 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
40 | class Errors(IntEnum): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
41 | OK=0 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
42 | BUSY=1 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
43 | OFFLINE=2 |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
44 | ERRORS=3 |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
45 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
46 | def combine(self, other): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
47 | return max(self, other) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
48 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
49 | def ok(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
50 | return self==self.OK |
45
aa2a95dc6093
Further improvements to state reporting; indicate busyness if repository lock cannot be acquired
Tuomo Valkonen <tuomov@iki.fi>
parents:
38
diff
changeset
|
51 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
52 | def __str__(self): |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
53 | return _errorstring[self] |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
54 | |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
55 | _errorstring={ |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
56 | Errors.OK: 'ok', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
57 | Errors.BUSY: 'busy', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
58 | Errors.OFFLINE: 'offline', |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
59 | Errors.ERRORS: 'errors' |
6 | 60 | } |
61 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
62 | _statestring={ |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
63 | State.INACTIVE: 'inactive', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
64 | State.PAUSED: 'paused', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
65 | State.SCHEDULED: 'scheduled', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
66 | State.QUEUED: 'queued', |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
67 | State.ACTIVE: 'active' |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
68 | } |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
69 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
70 | class Operation: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
71 | CREATE='create' |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
72 | PRUNE='prune' |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
73 | INFO='info' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
74 | LIST='list' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
75 | |
97 | 76 | def __init__(self, type, start_time, **kwargs): |
77 | self.type=type | |
78 | self.start_time=start_time | |
79 | self.finish_time=None | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
80 | self.detail=kwargs |
97 | 81 | self.errors=Errors.OK |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
82 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
83 | def when(self): |
97 | 84 | return self.start_time.realtime() |
85 | ||
86 | def ok(self): | |
87 | return self.errors.ok() | |
88 | ||
89 | def add_error(self, error): | |
90 | self.errors=self.errors.combine(error) | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
91 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
92 | def name(self): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
93 | if 'reason' in self.detail: |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
94 | return str(self.type) + '.' + self.detail['reason'] |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
95 | else: |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
96 | return str(self.type) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
97 | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
98 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
99 | class Status(Operation): |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
100 | def __init__(self, backup, op=None): |
97 | 101 | op=None |
102 | errorsop=None | |
103 | ||
104 | if backup.current_operation: | |
105 | op=backup.current_operation | |
106 | elif backup.scheduled_operation: | |
107 | op=backup.scheduled_operation | |
108 | if backup.previous_operation: | |
109 | errorsop=backup.previous_operation | |
110 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
111 | if op: |
97 | 112 | super().__init__(op.type, op.start_time, **op.detail) |
113 | if not errorsop: | |
114 | errorsop=op | |
115 | self.errors=errorsop.errors | |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
116 | else: |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
117 | super().__init__(None, None) |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
118 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
119 | self.name=backup.name |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
120 | self.state=backup.state |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
121 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
122 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
123 | # Miscellaneous helper routines |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
124 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
125 | |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
126 | loglevel_translation={ |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
127 | 'CRITICAL': logging.CRITICAL, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
128 | 'ERROR': logging.ERROR, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
129 | 'WARNING': logging.WARNING, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
130 | 'DEBUG': logging.DEBUG, |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
131 | 'INFO': logging.INFO |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
132 | } |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
133 | |
6 | 134 | def translate_loglevel(x): |
135 | if x in loglevel_translation: | |
136 | return loglevel_translation[x] | |
137 | else: | |
138 | return logging.ERROR | |
139 | ||
15 | 140 | def safe_get_int(t, x): |
141 | if x in t: | |
142 | tmp=t[x] | |
143 | if isinstance(tmp, int): | |
144 | return tmp | |
145 | return None | |
146 | ||
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
147 | def parse_borg_date(d, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
148 | try: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
149 | res=datetime.datetime.strptime(d, '%Y-%m-%dT%H:%M:%S.%f') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
150 | except Exception: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
151 | logger.exception('Unable parse date from borg: "%s"' % d) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
152 | res=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
153 | return res |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
154 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
155 | _checkpoint_str='.checkpoint' |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
156 | _checkpoint_str_len=len(_checkpoint_str) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
157 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
158 | def is_checkpoint(name): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
159 | i=name.rfind(_checkpoint_str); |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
160 | return i>=0 and i+_checkpoint_str_len==len(name) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
161 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
162 | def get_archive_time(a, logger): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
163 | if not 'name' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
164 | logger.error('Borg archive list entry missing name') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
165 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
166 | if is_checkpoint(a['name']): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
167 | logger.debug('Skipping checkpoint in archive list') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
168 | return None, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
169 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
170 | thistime=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
171 | if 'start' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
172 | thistime=parse_borg_date(a['start'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
173 | if not thistime and 'time' in a: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
174 | thistime=parse_borg_date(a['time'], logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
175 | if not thistime: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
176 | return None, False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
177 | |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
178 | return thistime, True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
179 | |
97 | 180 | _prune_progress_re=re.compile(".*\(([0-9]+)/([0-9]+)\)$") |
181 | # Borg gives very little progress info in easy form, so try to extrat it | |
182 | def check_prune_status(msg): | |
183 | res=_prune_progress_re.match(msg) | |
184 | if res: | |
185 | c=res.groups() | |
186 | try: | |
187 | archive_no=int(c[0]) | |
188 | of_total=int(c[1]) | |
189 | except: | |
190 | pass | |
191 | else: | |
192 | return archive_no, of_total | |
193 | return None, None | |
194 | ||
195 | ||
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
196 | # |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
197 | # The Backup class |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
198 | # |
15 | 199 | |
49 | 200 | class Backup(TerminableThread): |
1
4cdc9c1f6b28
basic scheduler structure draft, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
diff
changeset
|
201 | |
2 | 202 | def __decode_config(self, cfg): |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
203 | loc0='Backup %d' % self.identifier |
2 | 204 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
205 | self.backup_name=config.check_string(cfg, 'name', 'Name', loc0) |
2 | 206 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
207 | _logger.debug("Configuring backup '%s'" % self.backup_name) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
208 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
209 | self.logger=_logger.getChild(self.backup_name) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
210 | |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
211 | loc="Backup '%s'" % self.backup_name |
2 | 212 | |
54 | 213 | reponame=config.check_string(cfg, 'repository', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
214 | 'Target repository', loc) |
54 | 215 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
216 | self.repository=repository.find_repository(reponame) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
217 | if not self.repository: |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
218 | raise Exception("Repository '%s' not configured" % reponame) |
2 | 219 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
220 | self.archive_prefix=config.check_string(cfg, 'archive_prefix', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
221 | 'Archive prefix', loc) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
222 | |
2 | 223 | self.archive_template=config.check_string(cfg, 'archive_template', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
224 | 'Archive template', loc) |
2 | 225 | |
226 | self.backup_interval=config.check_nonneg_int(cfg, 'backup_interval', | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
227 | 'Backup interval', loc, |
2 | 228 | config.defaults['backup_interval']) |
229 | ||
97 | 230 | self.prune_interval=config.check_nonneg_int(cfg, 'prune_interval', |
231 | 'Prune interval', loc, | |
232 | config.defaults['prune_interval']) | |
233 | ||
2 | 234 | self.retry_interval=config.check_nonneg_int(cfg, 'retry_interval', |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
235 | 'Retry interval', loc, |
2 | 236 | config.defaults['retry_interval']) |
237 | ||
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
238 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
239 | scheduling=config.check_string(cfg, 'scheduling', |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
240 | 'Scheduling mode', loc, |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
241 | default="dreamtime") |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
242 | |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
243 | if scheduling=="dreamtime": |
97 | 244 | self.timeclass=DreamTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
245 | elif scheduling=="realtime": |
97 | 246 | self.timeclass=MonotonicTime |
76
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
247 | elif scheduling=="manual": |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
248 | self.backup_interval=0 |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
249 | else: |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
250 | logging.error("Invalid time class '%s' for %s" % (scheduling, loc)) |
4b08fca3ce34
Dreamtime scheduling: discount system sleep periods
Tuomo Valkonen <tuomov@iki.fi>
parents:
74
diff
changeset
|
251 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
252 | self.paths=config.check_nonempty_list_of_strings(cfg, 'paths', 'Paths', loc) |
32 | 253 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
254 | self.borg_parameters=config.BorgParameters.from_config(cfg, loc) |
20
fdfbe5d7b677
Keychain support and random fixes
Tuomo Valkonen <tuomov@iki.fi>
parents:
17
diff
changeset
|
255 | |
2 | 256 | |
49 | 257 | def __init__(self, identifier, cfg, scheduler): |
2 | 258 | self.identifier=identifier |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
259 | self.__status_update_callback=None |
98 | 260 | self._pause=False |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
261 | self.scheduler=scheduler |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
262 | self.logger=None # setup up in __decode_config once backup name is known |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
263 | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
264 | self.borg_instance=None |
7 | 265 | self.thread_log=None |
8 | 266 | self.thread_res=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
267 | |
97 | 268 | self.previous_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
269 | self.current_operation=None |
10 | 270 | self.scheduled_operation=None |
97 | 271 | self.previous_operation_of_type={} |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
272 | self.state=State.INACTIVE |
97 | 273 | self.timeclass=DreamTime |
49 | 274 | |
275 | self.__decode_config(cfg) | |
276 | ||
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
277 | super().__init__(target = self.__main_thread, name = self.backup_name) |
49 | 278 | self.daemon=True |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
279 | |
7 | 280 | def is_running(self): |
49 | 281 | with self._cond: |
8 | 282 | running=self.__is_running_unlocked() |
283 | return running | |
284 | ||
285 | def __is_running_unlocked(self): | |
286 | running=self.current_operation | |
287 | if not running: | |
288 | # Consistency check | |
289 | assert((not self.borg_instance and not self.thread_log and | |
290 | not self.thread_res)) | |
7 | 291 | return running |
292 | ||
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
293 | def __block_when_running(self): |
7 | 294 | running=self.is_running() |
295 | assert(not running) | |
2 | 296 | |
7 | 297 | def __log_listener(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
298 | self.logger.debug('Log listener thread waiting for entries') |
7 | 299 | success=True |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
300 | for msg in iter(self.borg_instance.read_log, None): |
97 | 301 | self.logger.debug(str(msg)) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
302 | t=msg['type'] |
15 | 303 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
304 | errormsg=None |
15 | 305 | callback=None |
306 | ||
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
307 | if t=='progress_percent': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
308 | current=safe_get_int(msg, 'current') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
309 | total=safe_get_int(msg, 'total') |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
310 | operation_no=safe_get_int(msg, 'operation') |
15 | 311 | if current is not None and total is not None: |
49 | 312 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
313 | self.current_operation.detail['progress_current']=current |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
314 | self.current_operation.detail['progress_total']=total |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
315 | self.current_operation.detail['operation_no']=operation_no |
14
5a988a2c2624
UI: progress percentange support (Borg doesn't seem to be reporting) + error indicator in tray: B?
Tuomo Valkonen <tuomov@iki.fi>
parents:
12
diff
changeset
|
316 | status, callback=self.__status_unlocked() |
15 | 317 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
318 | elif t=='archive_progress': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
319 | original_size=safe_get_int(msg, 'original_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
320 | compressed_size=safe_get_int(msg, 'compressed_size') |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
321 | deduplicated_size=safe_get_int(msg, 'deduplicated_size') |
15 | 322 | if original_size is not None and original_size is not None and deduplicated_size is not None: |
49 | 323 | with self._cond: |
62
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
324 | self.current_operation.detail['original_size']=original_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
325 | self.current_operation.detail['compressed_size']=compressed_size |
b7d13b2ad67e
Turned Status and Operation into classes instead of dictionaries
Tuomo Valkonen <tuomov@iki.fi>
parents:
61
diff
changeset
|
326 | self.current_operation.detail['deduplicated_size']=deduplicated_size |
15 | 327 | status, callback=self.__status_unlocked() |
328 | ||
329 | elif t=='progress_message': | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
330 | pass |
15 | 331 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
332 | elif t=='file_status': |
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
333 | pass |
15 | 334 | |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
335 | elif t=='log_message': |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
336 | if 'levelname' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
337 | msg['levelname']='ERROR' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
338 | if 'message' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
339 | msg['message']='UNKNOWN' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
340 | if 'name' not in msg: |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
341 | msg['name']='borg' |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
342 | lvl=translate_loglevel(msg['levelname']) |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
343 | self.logger.log(lvl, msg['name'] + ': ' + msg['message']) |
71
a8a5ebb64e02
Changed retry timing to start form end of previous attempt instead of beginning
Tuomo Valkonen <tuomov@iki.fi>
parents:
70
diff
changeset
|
344 | if lvl>=logging.ERROR: |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
345 | errormsg=msg |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
346 | errors=Errors.ERRORS |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
347 | if ('msgid' in msg and |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
348 | (msg['msgid']=='LockTimeout' or # observed in reality |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
349 | msg['msgid']=='LockErrorT' or # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
350 | msg['msgid']=='LockErrorT')): # in docs |
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
351 | errors=Errors.BUSY |
49 | 352 | with self._cond: |
97 | 353 | self.current_operation.add_error(errors) |
98 | 354 | # Don't notify of errors if we are terminating or pausing |
355 | if not self._terminate_or_pause(): | |
356 | status, callback=self.__status_unlocked() | |
97 | 357 | elif lvl==logging.INFO and self.current_operation.type==Operation.PRUNE: |
358 | # Borg gives very little progress info in easy form, so try to extrat it | |
359 | archive_number, of_total=check_prune_status(msg['message']) | |
360 | if archive_number!=None and of_total!=None: | |
361 | self.current_operation.detail['progress_current_secondary']=archive_number | |
362 | self.current_operation.detail['progress_total_secondary']=of_total | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
363 | status, callback=self.__status_unlocked() |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
364 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
365 | elif t=='question_prompt' or t=='question_prompt_retry': |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
366 | self.logger.error('Did not expect to receive question prompt from borg') |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
367 | with self._cond: |
97 | 368 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
369 | # TODO: terminate org? Send 'NO' reply? |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
370 | |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
371 | elif (t=='question_invalid_answer' or t=='question_accepted_default' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
372 | or t=='question_accepted_true' or t=='question_accepted_false' |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
373 | or t=='question_env_answer'): |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
374 | pass |
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
375 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
376 | else: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
377 | self.logger.debug('Unrecognised log entry %s' % str(status)) |
15 | 378 | |
379 | if callback: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
380 | callback(status, errorlog=errormsg) |
6 | 381 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
382 | self.logger.debug('Waiting for borg subprocess to terminate in log thread') |
4
d72c4844e791
Better borg output processing and some logging
Tuomo Valkonen <tuomov@iki.fi>
parents:
3
diff
changeset
|
383 | |
6 | 384 | self.borg_instance.wait() |
385 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
386 | self.logger.debug('Borg subprocess terminated; terminating log listener thread') |
7 | 387 | |
388 | def __result_listener(self): | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
389 | self.logger.debug('Result listener thread waiting for result') |
7 | 390 | |
391 | res=self.borg_instance.read_result() | |
392 | ||
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
393 | self.logger.debug('Borg result: %s' % str(res)) |
7 | 394 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
395 | if res is None: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
396 | with self._cond: |
97 | 397 | # Prune gives absolutely no result, so don't complain |
398 | if (self.current_operation.ok() and | |
399 | self.current_operation.type!=Operation.PRUNE): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
400 | self.logger.error('No result from borg despite no error in log') |
97 | 401 | self.current_operation.add_error(Errors.ERRORS) |
402 | elif self.current_operation.type==Operation.LIST: | |
403 | self.__handle_list_result(res) | |
404 | ||
405 | # All other results are discarded | |
406 | ||
407 | def __handle_list_result(self, res): | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
408 | ok=True |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
409 | latest=None |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
410 | if 'archives' in res and isinstance(res['archives'], list): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
411 | archives=res['archives'] |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
412 | for a in archives: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
413 | if not isinstance(a, dict): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
414 | self.logger.error('Borg archive list entry not a dictionary') |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
415 | ok=False |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
416 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
417 | thistime, this_ok=get_archive_time(a, self.logger) |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
418 | if thistime and (not latest or thistime>latest): |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
419 | latest=thistime |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
420 | ok=ok and this_ok |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
421 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
422 | logger.error('Borg archive list missing "archives" entry') |
7 | 423 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
424 | if not ok: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
425 | with self._cond: |
97 | 426 | self.current_operation.add_error(Errors.ERRORS) |
88
dfd52898f175
Added dummy entries in log reader for question prompts from borg
Tuomo Valkonen <tuomov@iki.fi>
parents:
87
diff
changeset
|
427 | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
428 | if latest: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
429 | self.logger.info('borg info: Previous backup was on %s' % latest.isoformat()) |
97 | 430 | when=MonotonicTime.from_realtime(time.mktime(latest.timetuple())) |
431 | op=Operation(Operation.CREATE, when, reason='listed') | |
432 | op.finish_time=when | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
433 | with self._cond: |
97 | 434 | self.previous_operation_of_type[Operation.CREATE]=op |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
435 | else: |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
436 | self.logger.info('borg info: Could not discover a previous backup') |
7 | 437 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
438 | def __do_launch(self, op, archive_or_repository, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
439 | common_params, op_params, paths=[]): |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
440 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
441 | self.logger.debug('Creating BorgInstance') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
442 | |
97 | 443 | inst=BorgInstance(op.type, archive_or_repository, |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
444 | common_params, op_params, paths) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
445 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
446 | self.logger.debug('Launching BorgInstance via repository') |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
447 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
448 | # Only the Repository object has access to the passphrase |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
449 | self.repository.launch_borg_instance(inst) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
450 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
451 | self.logger.debug('Creating listener threads') |
31 | 452 | |
7 | 453 | t_log=Thread(target=self.__log_listener) |
454 | t_log.daemon=True | |
2 | 455 | |
7 | 456 | t_res=Thread(target=self.__result_listener) |
457 | t_res.daemon=True | |
458 | ||
459 | self.thread_log=t_log | |
460 | self.thread_res=t_res | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
461 | self.borg_instance=inst |
64 | 462 | self.current_operation=op |
463 | # Update scheduled time to real starting time to schedule | |
464 | # next run relative to this | |
97 | 465 | self.current_operation.start_time=MonotonicTime.now() |
64 | 466 | # Reset error status when starting a new operation |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
467 | self.__update_status(State.ACTIVE) |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
468 | |
7 | 469 | t_log.start() |
470 | t_res.start() | |
3
4cad934aa9ce
Can launch borg now; output not yet processed
Tuomo Valkonen <tuomov@iki.fi>
parents:
2
diff
changeset
|
471 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
472 | |
49 | 473 | def __launch(self, op): |
97 | 474 | self.logger.debug("Launching '%s'" % str(op.type)) |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
475 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
476 | params=(config.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
477 | +self.repository.borg_parameters |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
478 | +self.borg_parameters) |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
479 | |
97 | 480 | if op.type==Operation.CREATE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
481 | archive="%s::%s%s" % (self.repository.location, |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
482 | self.archive_prefix, |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
483 | self.archive_template) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
484 | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
485 | self.__do_launch(op, archive, params.common, |
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
486 | params.create, self.paths) |
97 | 487 | elif op.type==Operation.PRUNE: |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
488 | self.__do_launch(op, self.repository.location, params.common, |
97 | 489 | [{'prefix': self.archive_prefix}] + params.prune) |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
490 | |
97 | 491 | elif op.type==Operation.LIST: |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
492 | self.__do_launch(op, self.repository.location, params.common, |
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
493 | [{'prefix': self.archive_prefix}]) |
8 | 494 | else: |
97 | 495 | raise NotImplementedError("Invalid operation '%s'" % str(op.type)) |
2 | 496 | |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
497 | # This must be called with self._cond held. |
64 | 498 | def __launch_and_wait(self): |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
499 | op=self.scheduled_operation |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
500 | if not op: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
501 | self.logger.debug("Queued operation aborted") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
502 | else: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
503 | self.scheduled_operation=None |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
504 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
505 | self.__launch(op) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
506 | |
64 | 507 | self.__wait_finish() |
508 | ||
98 | 509 | def _terminate_or_pause(self): |
510 | return self._terminate or self._pause | |
511 | ||
64 | 512 | def __wait_finish(self): |
97 | 513 | current=self.current_operation |
514 | ||
64 | 515 | # Wait for main logger thread to terminate, or for us to be terminated |
98 | 516 | while not self._terminate_or_pause() and self.thread_res.is_alive(): |
64 | 517 | self._cond.release() |
518 | self.thread_res.join(JOIN_TIMEOUT) | |
519 | self._cond.acquire() | |
520 | ||
98 | 521 | # If terminate or pause has been signalled, let outer termination handler |
64 | 522 | # take care of things (Within this Backup class, it would be cleanest |
523 | # to raise an exception instead, but in most other places it's better | |
524 | # to just check self._terminate, so we don't complicate things with | |
525 | # an extra exception.) | |
98 | 526 | if self.thread_res.is_alive(): |
64 | 527 | return |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
528 | |
64 | 529 | self.logger.debug('Waiting for borg and log subprocesses to terminate') |
530 | ||
531 | self._cond.release() | |
532 | self.thread_log.join() | |
533 | self._cond.acquire() | |
534 | ||
535 | if not self.borg_instance.wait(): | |
536 | self.logger.error('Borg subprocess did not terminate') | |
97 | 537 | curent.add_error(Errors.ERRORS) |
538 | ||
539 | current.finish_time=MonotonicTime.now() | |
64 | 540 | |
97 | 541 | self.previous_operation_of_type[current.type]=current |
542 | self.previous_operation=current | |
543 | self.current_operation=None | |
64 | 544 | self.thread_res=None |
545 | self.thread_log=None | |
546 | self.borg_instance=None | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
547 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
548 | def __main_thread(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
549 | with self._cond: |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
550 | while not self._terminate: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
551 | try: |
64 | 552 | assert(not self.current_operation) |
553 | self.__main_thread_wait_schedule() | |
100
b141bed9e718
macOS "sleep" signals / status are complete bollocks:
Tuomo Valkonen <tuomov@iki.fi>
parents:
98
diff
changeset
|
554 | if ((not self._terminate_or_pause()) and self.scheduled_operation |
98 | 555 | and self.scheduled_operation.start_time <= MonotonicTime.now()): |
64 | 556 | self.__main_thread_queue_and_launch() |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
557 | except Exception as err: |
98 | 558 | self.logger.exception("Exception in backup '%s'" % self.backup_name) |
559 | finally: | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
560 | self.__cleanup() |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
561 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
562 | def __cleanup(self): |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
563 | thread_log=self.thread_log |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
564 | thread_res=self.thread_res |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
565 | borg_instance=self.borg_instance |
98 | 566 | self.scheduled_operation=None |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
567 | self.thread_log=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
568 | self.thread_res=None |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
569 | self.borg_instance=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
570 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
571 | self._cond.release() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
572 | try: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
573 | if borg_instance: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
574 | self.logger.debug("Terminating a borg instance") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
575 | borg_instance.terminate() |
8 | 576 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
577 | if thread_log: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
578 | self.logger.debug("Waiting for log thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
579 | thread_log.join() |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
580 | |
87
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
581 | if thread_res: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
582 | self.logger.debug("Waiting for result thread to terminate") |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
583 | thread_res.join() |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
584 | finally: |
a214d475aa28
Better recovery from errors; fixes to potential race conditions in scheduler and repository queue
Tuomo Valkonen <tuomov@iki.fi>
parents:
86
diff
changeset
|
585 | self._cond.acquire() |
98 | 586 | self.current_operation=None |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
587 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
588 | # Main thread/2. Schedule next operation if there is no manually |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
589 | # requested one |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
590 | def __main_thread_wait_schedule(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
591 | op=None |
98 | 592 | if self._pause: |
593 | self.logger.info("Waiting for resume to be signalled") | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
594 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
595 | self.__update_status(State.PAUSED) |
61
bc6c3d74e6ea
Made combination error-state into an error-state matrix, as well as Enum
Tuomo Valkonen <tuomov@iki.fi>
parents:
58
diff
changeset
|
596 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
597 | self._cond.wait() |
98 | 598 | else: |
599 | if not self.scheduled_operation: | |
600 | op=self.__next_operation_unlocked() | |
601 | if op: | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
602 | self.scheduled_operation=op |
98 | 603 | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
604 | self.__update_status(State.SCHEDULED) |
98 | 605 | |
606 | # Wait under scheduled wait | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
607 | eventname=op.name() + '@' + self.backup_name |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
608 | self.scheduler.wait_until(op.start_time, self._cond, eventname) |
98 | 609 | else: |
610 | # Nothing scheduled - just wait | |
611 | self.logger.info("Waiting for manual scheduling") | |
612 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
613 | self.__update_status(State.INACTIVE) |
98 | 614 | |
615 | self._cond.wait() | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
616 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
617 | # Main thread/3. If there is a scheduled operation (it might have been |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
618 | # changed manually from 'op' created in __main_thread_wait_schedule above), |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
619 | # queue it on the repository, and launch the operation once repository |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
620 | # available |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
621 | def __main_thread_queue_and_launch(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
622 | if self.scheduled_operation: |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
623 | |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
624 | self.__update_status(State.QUEUED) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
625 | |
64 | 626 | res=self.repository.queue_action(self._cond, |
627 | action=self.__launch_and_wait, | |
74
4f56142e7497
Separated repository configuration form backup configuration;
Tuomo Valkonen <tuomov@iki.fi>
parents:
71
diff
changeset
|
628 | name=self.backup_name) |
98 | 629 | if not res: |
64 | 630 | self.logger.debug("Queueing aborted") |
8 | 631 | |
632 | def __next_operation_unlocked(self): | |
97 | 633 | listop=self.__next_operation_list() |
634 | if listop: | |
635 | return listop | |
636 | ||
637 | create=self.__next_operation_type(Operation.CREATE, | |
638 | self.backup_interval, | |
639 | important=True, | |
640 | initial_reason='initial'); | |
641 | ||
642 | prune=self.__next_operation_type(Operation.PRUNE, | |
643 | self.prune_interval, | |
644 | important=False, | |
645 | initial_reason=None); | |
646 | ||
647 | if not prune: | |
648 | return create | |
649 | elif not create: | |
650 | return prune | |
651 | elif create.start_time < prune.start_time: | |
652 | return create | |
653 | else: | |
654 | return prune | |
655 | ||
656 | def __next_operation_list(self): | |
657 | reason='initial' | |
658 | # Unless manual backup has been chosen (backup_interval<=0), perform | |
659 | # repository listing if no previous create operation known, or if we | |
660 | # just pruned the repository | |
661 | if self.backup_interval<=0: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
662 | return None |
97 | 663 | elif (self.previous_operation and |
664 | self.previous_operation.type==Operation.PRUNE): | |
665 | tm=MonotonicTime.now() | |
666 | reason='post-prune' | |
667 | elif Operation.LIST in self.previous_operation_of_type: | |
668 | prev=self.previous_operation_of_type[Operation.LIST] | |
669 | if prev.ok(): | |
670 | return None | |
671 | if self.retry_interval==0: | |
85
56a000d15965
On startup, for better scheduling, obtain previous backup time with 'borg list'
Tuomo Valkonen <tuomov@iki.fi>
parents:
80
diff
changeset
|
672 | # Do not retry in case of errors if retry interval is zero |
10 | 673 | return None |
97 | 674 | # Attempt after retry interval |
675 | tm=MonotonicTime.after_other(prev.finish_time, self.retry_interval) | |
676 | else: | |
677 | # Nothing has been attempted: run immediately | |
678 | tm=MonotonicTime.now() | |
679 | return Operation(Operation.LIST, tm, reason=reason) | |
680 | ||
681 | def __next_operation_type(self, optype, standard_interval, | |
682 | important=False, | |
683 | initial_reason=None): | |
684 | if optype not in self.previous_operation_of_type: | |
685 | # No previous operation exists; perform immediately | |
686 | # if important, otherwise after standard interval. | |
687 | # Do not perform if manual operation selected by | |
688 | # setting standard_interval<=0 | |
689 | if standard_interval<=0: | |
10 | 690 | return None |
691 | else: | |
97 | 692 | if important: |
693 | tm=MonotonicTime.now() | |
694 | else: | |
695 | tm=self.timeclass.after(standard_interval) | |
696 | if initial_reason: | |
697 | return Operation(optype, tm, reason=initial_reason) | |
96
de8ac6c470d8
Backup scheduling fixes in case of an initial backup
Tuomo Valkonen <tuomov@iki.fi>
parents:
89
diff
changeset
|
698 | else: |
97 | 699 | return Operation(optype, tm) |
5 | 700 | else: |
97 | 701 | # Previous operation has been performed; perform after |
702 | # retry interval if there were errors, otherwise after | |
703 | # standard interval. | |
704 | prev=self.previous_operation_of_type[optype] | |
705 | if not prev.ok(): | |
706 | tm=MonotonicTime.after_other(prev.start_time, | |
707 | self.retry_interval) | |
708 | return Operation(optype, tm, reason='retry') | |
709 | elif standard_interval>0: | |
710 | tm=self.timeclass.after_other(prev.start_time, | |
711 | standard_interval) | |
712 | return Operation(optype, tm) | |
713 | else: | |
714 | # Manual operation is standard_interval is zero. | |
715 | return None | |
10 | 716 | |
717 | def __status_unlocked(self): | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
718 | callback=self.__status_update_callback |
97 | 719 | status=Status(self) |
10 | 720 | |
721 | return status, callback | |
722 | ||
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
723 | def __update_status(self, state): |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
724 | self.logger.debug("Entering %s state", str(state)) |
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
725 | self.state=state |
49 | 726 | status, callback = self.__status_unlocked() |
727 | if callback: | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
728 | #self._cond.release() |
49 | 729 | try: |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
730 | callback(status) |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
731 | except Exception: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
732 | self.logger.exception("Status update error") |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
733 | #finally: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
734 | # self._cond.acquire() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
735 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
736 | # |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
737 | # Interface functions |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
738 | # |
49 | 739 | |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
740 | def set_status_update_callback(self, callback): |
49 | 741 | with self._cond: |
21
c36e549a7f12
Errors as rumps notifications
Tuomo Valkonen <tuomov@iki.fi>
parents:
20
diff
changeset
|
742 | self.__status_update_callback=callback |
10 | 743 | |
49 | 744 | def status(self): |
745 | with self._cond: | |
746 | res=self.__status_unlocked() | |
747 | return res[0] | |
10 | 748 | |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
749 | def create(self): |
97 | 750 | op=Operation(Operation.CREATE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
751 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
752 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
753 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
754 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
755 | def prune(self): |
97 | 756 | op=Operation(Operation.PRUNE, MonotonicTime.now(), reason='manual') |
55
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
757 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
758 | self.scheduled_operation=op |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
759 | self._cond.notify() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
760 | |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
761 | # TODO: Decide exact (manual) abort mechanism. Perhaps two stages |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
762 | def abort(self): |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
763 | with self._cond: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
764 | if self.borg_instance: |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
765 | self.borg_instance.terminate() |
407af23d16bb
Improved backup main thread loop, etc.
Tuomo Valkonen <tuomov@iki.fi>
parents:
54
diff
changeset
|
766 | |
98 | 767 | def is_paused(self): |
768 | with self._cond: | |
104
d33e2d7dbeb1
Added some debug log messages
Tuomo Valkonen <tuomov@iki.fi>
parents:
100
diff
changeset
|
769 | paused=(self.state==State.PAUSED) |
98 | 770 | return paused |
771 | ||
772 | def pause(self): | |
773 | with self._cond: | |
774 | self.logger.debug('Pause signalled') | |
775 | self.scheduled_operation=None | |
776 | self._pause=True | |
777 | self._cond.notify() | |
778 | ||
779 | def resume(self): | |
780 | with self._cond: | |
781 | self.logger.debug('Resume signalled') | |
782 | self._pause=False | |
783 | self._cond.notify() | |
784 |